diff --git a/.env.example b/.env.example index 548e3c0ba..860d868fe 100644 --- a/.env.example +++ b/.env.example @@ -1,15 +1,36 @@ # MeshCore Analyzer — Environment Configuration -# Copy to .env and customize. All values have sensible defaults. +# New installation: copy to .env and customize. Existing installation: merge +# missing keys into your private .env; do not replace paths or credentials. # # This file is read by BOTH docker compose AND manage.sh — one source of truth. # manage.sh setup negotiates and updates only these production managed keys: # PROD_DATA_DIR, PROD_HTTP_PORT, PROD_HTTPS_PORT, PROD_MQTT_PORT, DISABLE_MOSQUITTO # Each environment keeps config + data together in one directory: -# ~/meshcore-data/config.json, meshcore.db, Caddyfile, theme.json -# ~/meshcore-staging-data/config.json, meshcore.db, Caddyfile +# ~/meshcore-data/config.json, Caddyfile, theme.json, queues and backups +# ~/meshcore-staging-data/config.json, Caddyfile, queues and backups +# With PostgreSQL selected, its separate persistent directory is not a logical backup. + +# --- Storage choice --- +# SQLite is the default and needs no database service or PostgreSQL credentials. +# setup offers PostgreSQL explicitly. Once installed, storage-selection.json in +# the shared state directory is authoritative; ordinary updates keep it. +# To change engines, use the verified offline storage switch in docs/storage.md. +# Optional PostgreSQL credentials/directories: merge .env.postgres.example. +# Do not replace an existing .env or regenerate existing database passwords. +# Optional container state subdirectory, shared by setup/server/ingestor: +# CORESCOPE_STATE_DIR=/app/data + +# --- Example deployment (docker-compose.example.yml only) --- +# For upgrades, point DATA_DIR at the existing state directory BEFORE bootstrap. +# PROD_DATA_DIR and STAGING_DATA_DIR below do not control the example variant. +DATA_DIR=./data +HTTP_PORT=80 +# CORESCOPE_IMAGE= +# Bundled Caddy forwards port 80 to the Go server on 3000 by default. If you +# explicitly set DISABLE_CADDY=true, your proxy/port mapping must use port 3000. # --- Production --- -# Data directory (database, theme, etc.) +# State directory (config, theme, queues and backups) # Default: ~/meshcore-data # Used by: docker compose, manage.sh PROD_DATA_DIR=~/meshcore-data diff --git a/.env.postgres.example b/.env.postgres.example new file mode 100644 index 000000000..d0bbe08a2 --- /dev/null +++ b/.env.postgres.example @@ -0,0 +1,21 @@ +# OPTIONAL PostgreSQL settings — merge into your existing private .env. +# Base Compose files need none of these. Add the matching .postgres.yml override. +# Use a different persistent directory for each production/staging instance. +# Generate a DIFFERENT value for each blank field: openssl rand -hex 32 +# Keep .env private. Hexadecimal passwords can be safely embedded in the +# generated container URLs; the initializer enforces at least 32 hex digits. +# These roles are created only when PostgreSQL's data directory is empty. +# Editing these values afterward does not change existing database passwords. +POSTGRES_ADMIN_PASSWORD= +CORESCOPE_OWNER_PASSWORD= +CORESCOPE_READER_PASSWORD= +CORESCOPE_WRITER_PASSWORD= +CORESCOPE_ACCOUNTS_PASSWORD= +CORESCOPE_CHANNELS_PASSWORD= +PROD_POSTGRES_DATA_DIR=~/corescope-postgres +STAGING_POSTGRES_DATA_DIR=~/corescope-staging-postgres +# docker-compose.example.yml uses this local default instead: +POSTGRES_DATA_DIR=./postgres-data +# Existing installations keep their recorded backend on update. Changing engines +# requires the offline storage switch in docs/storage.md. Never mount two database +# servers on one persistent directory. diff --git a/.eslintrc.json b/.eslintrc.json index 33197e2fc..a469ec574 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -12,6 +12,7 @@ "CACHE_INVALIDATE_MS": "readonly", "CLIENT_CONFIG": "readonly", "CLIENT_TTL": "readonly", + "CSAuth": "readonly", "ChannelColorPicker": "readonly", "ChannelColors": "readonly", "ChannelDecrypt": "readonly", @@ -22,6 +23,7 @@ "EXTERNAL_URLS": "readonly", "FAV_KEY": "readonly", "FilterUX": "readonly", + "fitColumnsToContent": "readonly", "GestureHints": "readonly", "HEALTH_THRESHOLDS": "readonly", "HashColor": "readonly", @@ -35,6 +37,7 @@ "MAX_HOP_DIST": "readonly", "MeshAudio": "readonly", "MeshConfigReady": "readonly", + "NodeHopAnalytics": "readonly", "PAYLOAD_COLORS": "readonly", "PAYLOAD_TYPES": "readonly", "PERF_SLOW_MS": "readonly", @@ -255,6 +258,7 @@ "pad3": "readonly", "pages": "readonly", "parseViewportHash": "readonly", + "pathHashSize": "readonly", "payloadTypeColor": "readonly", "payloadTypeName": "readonly", "process": "readonly", diff --git a/.github/agents/pr-reviewer.agent.md b/.github/agents/pr-reviewer.agent.md index 357fbca5c..9def873bb 100644 --- a/.github/agents/pr-reviewer.agent.md +++ b/.github/agents/pr-reviewer.agent.md @@ -28,7 +28,7 @@ You will pay special attention to the following areas during your review: - **Project Architecture**: Does the PR respect the single Node.js server + static frontend architecture? Are changes in the right place? ### 2. Testing and Validation -- **No commit without tests**: Is the backend logic change covered by unit tests? Is `test-packet-filter.js` or `test-aging.js` updated if necessary? +- **No commit without tests**: Is the backend logic change covered by unit tests? Is `tests/unit/test-packet-filter.js` or `tests/unit/test-aging.js` updated if necessary? - **Browser Validation**: Has the contributor confirmed the change works in a browser? Is there a screenshot for visual changes? - **Cache Busters**: If any `public/` assets (`.js`, `.css`) were modified, has the cache buster in `public/index.html` been bumped in the *same commit*? This is critical. diff --git a/.github/instructions/copilot.instructions.md b/.github/instructions/copilot.instructions.md index 357fbca5c..9def873bb 100644 --- a/.github/instructions/copilot.instructions.md +++ b/.github/instructions/copilot.instructions.md @@ -28,7 +28,7 @@ You will pay special attention to the following areas during your review: - **Project Architecture**: Does the PR respect the single Node.js server + static frontend architecture? Are changes in the right place? ### 2. Testing and Validation -- **No commit without tests**: Is the backend logic change covered by unit tests? Is `test-packet-filter.js` or `test-aging.js` updated if necessary? +- **No commit without tests**: Is the backend logic change covered by unit tests? Is `tests/unit/test-packet-filter.js` or `tests/unit/test-aging.js` updated if necessary? - **Browser Validation**: Has the contributor confirmed the change works in a browser? Is there a screenshot for visual changes? - **Cache Busters**: If any `public/` assets (`.js`, `.css`) were modified, has the cache buster in `public/index.html` been bumped in the *same commit*? This is critical. diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 6b68ff79c..c80b6688b 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -18,10 +18,22 @@ on: pull_request: branches: [master] workflow_dispatch: + inputs: + images_published: + description: 'Release fast path already published the tag images' + type: boolean + default: false + postgres_benchmark: + description: Run only the isolated PostgreSQL benchmark (no publish or deploy) + type: boolean + default: false + candidate_sha: + description: Full candidate commit for postgres_benchmark + type: string + default: '' permissions: contents: read - packages: write concurrency: group: ci-${{ github.event.pull_request.number || github.ref }} @@ -33,19 +45,38 @@ env: STAGING_SERVICE: staging-go STAGING_CONTAINER: corescope-staging-go -# Pipeline (sequential, fail-fast): -# go-test → e2e-test → build-and-publish → deploy → publish-badges +# Pipeline. The test jobs and the image build run side by side; only the GHCR +# push waits for all of them: +# changes ─┬─ go-test ─────────────────────────┐ +# ├─ race-test (ingestor changes only) ─┤ +# ├─ e2e-shard ×3 ── e2e-test (gate) ───┼─ build-and-publish → deploy, publish +# └─ image-check (two-arch build) ──────┘ # PRs stop after build-and-publish (no GHCR push). Master continues to deploy + badges. jobs: + postgres-benchmark: + name: Isolated PostgreSQL comparison + needs: [changes] + if: | + (github.event_name == 'workflow_dispatch' && inputs.postgres_benchmark) + || (github.event_name == 'pull_request' + && needs.changes.outputs.code == 'true' + && contains(github.event.pull_request.body, '[x] Run PostgreSQL comparison')) + permissions: + contents: read + uses: ./.github/workflows/postgres-benchmark.yml + with: + candidate_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || inputs.candidate_sha }} + # ─────────────────────────────────────────────────────────────── # 0. Change scope — decides whether the expensive jobs below run. # ─────────────────────────────────────────────────────────────── changes: name: "🔎 Change scope" - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 outputs: code: ${{ steps.scope.outputs.code }} + ingestor: ${{ steps.scope.outputs.ingestor }} steps: - name: Checkout code uses: actions/checkout@v5 @@ -56,11 +87,18 @@ jobs: id: scope run: | set -euo pipefail + if [ "${{ inputs.postgres_benchmark }}" = "true" ]; then + echo "code=false" >> "$GITHUB_OUTPUT" + echo "ingestor=false" >> "$GITHUB_OUTPUT" + echo "benchmark-only dispatch" + exit 0 + fi # Only pull requests are scoped. Pushes and dispatches always count as # code, so master keeps producing an :edge image and tag builds are # never skipped. if [ "${{ github.event_name }}" != "pull_request" ]; then echo "code=true" >> "$GITHUB_OUTPUT" + echo "ingestor=true" >> "$GITHUB_OUTPUT" echo "event=${{ github.event_name }} -> full pipeline" exit 0 fi @@ -68,6 +106,7 @@ jobs: # Empty diff means something is off; run everything rather than guess. if [ -z "$CHANGED" ]; then echo "code=true" >> "$GITHUB_OUTPUT" + echo "ingestor=true" >> "$GITHUB_OUTPUT" echo "empty diff -> full pipeline" exit 0 fi @@ -80,15 +119,125 @@ jobs: echo "code=false" >> "$GITHUB_OUTPUT" echo "-> documentation only, heavy jobs skip (and report Success)" fi + # The race detector is its own job, and it only earns its ten minutes + # when the code it inspects changed. A frontend or docs PR cannot + # introduce a data race in the ingestor. + if echo "$CHANGED" | grep -qE '^cmd/ingestor/.*[.]go$'; then + echo "ingestor=true" >> "$GITHUB_OUTPUT" + echo "-> ingestor Go files changed, race detector runs" + else + echo "ingestor=false" >> "$GITHUB_OUTPUT" + echo "-> no ingestor Go changes, race detector skips" + fi # ─────────────────────────────────────────────────────────────── - # 1. Go Build & Test + # 1a. Race detector (ingestor) — its own job on purpose # ─────────────────────────────────────────────────────────────── + # Runs beside "Go Build & Test" rather than inside it, so its ten-odd + # minutes overlap the E2E job instead of extending the critical path, and + # only when ingestor Go files changed. A frontend or documentation PR cannot + # introduce a data race here, and paying for the check on every PR is what + # gets a check switched off again. + # + # The server has -race in its own step already; this closes the same gap for + # the ingestor, which carries an atomic.Pointer snapshot (the region key + # set) whose safety is an argument until something checks it. + race-backend: + name: "Race detector (ingestor, ${{ matrix.backend }})" + strategy: + fail-fast: false + matrix: + backend: [sqlite, postgres] + runs-on: ubuntu-24.04 + needs: [changes] + if: needs.changes.outputs.ingestor == 'true' + services: &postgres_test_services + postgres: + image: ${{ matrix.backend == 'postgres' && 'postgres:18.6-alpine3.24' || '' }} + env: + POSTGRES_PASSWORD: "ci-${{ github.run_id }}-${{ github.run_attempt }}" + POSTGRES_INITDB_ARGS: "--encoding=UTF8 --locale=C" + ports: ["127.0.0.1:5432:5432"] + options: >- + --health-cmd "pg_isready -h 127.0.0.1 -U postgres" + --health-interval 5s --health-timeout 5s --health-retries 20 + env: + CORESCOPE_TEST_BACKEND: ${{ matrix.backend }} + CORESCOPE_DB_BACKEND: ${{ matrix.backend }} + CORESCOPE_TEST_POSTGRES_URL: ${{ matrix.backend == 'postgres' && format('postgresql://postgres:ci-{0}-{1}@127.0.0.1:5432/postgres?sslmode=disable', github.run_id, github.run_attempt) || '' }} + PGHOST: 127.0.0.1 + PGPORT: 5432 + PGUSER: postgres + PGPASSWORD: "ci-${{ github.run_id }}-${{ github.run_attempt }}" + PGSSLMODE: disable + steps: + - name: Checkout code + uses: actions/checkout@v5 + + - name: Install pinned PostgreSQL 18.6 clients + if: matrix.backend == 'postgres' + run: &install_postgres_clients bash scripts/install-postgres-ci.sh client + + - name: Set up Go 1.27.2 + uses: actions/setup-go@v6 + with: + go-version: '1.27.2' + # All modules: a -race build of a cgo package is the slowest thing in + # this pipeline, and setup-go caches ~/.cache/go-build as well as the + # module cache, so a key that misses on an unrelated module hurts most + # here. Same reasoning as go-test below. + cache-dependency-path: "**/go.sum" + + - name: go test -race + run: | + set -e -o pipefail + cd cmd/ingestor + go test -timeout 30m -race ./... + + race-test: + name: "🏁 Race detector (ingestor)" + runs-on: ubuntu-24.04 + needs: [race-backend, changes] + if: ${{ !cancelled() && needs.changes.outputs.ingestor == 'true' }} + steps: + - name: Require both native race matrices + run: | + result="${{ needs['race-backend'].result }}" + [ "$result" = success ] || { echo "::error::Race matrices did not all pass ($result)"; exit 1; } + go-test: name: "✅ Go Build & Test" - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 + needs: [go-backend, changes] + if: ${{ !cancelled() && needs.changes.outputs.code == 'true' }} + steps: + - name: Require both native storage matrices + run: | + result="${{ needs['go-backend'].result }}" + [ "$result" = success ] || { echo "::error::Storage matrices did not all pass ($result)"; exit 1; } + + # ─────────────────────────────────────────────────────────────── + # 1. Go Build & Test + # ─────────────────────────────────────────────────────────────── + go-backend: + name: "Go Build & Test (${{ matrix.backend }})" + strategy: + fail-fast: false + matrix: + backend: [sqlite, postgres] + runs-on: ubuntu-24.04 needs: [changes] if: needs.changes.outputs.code == 'true' + services: *postgres_test_services + env: + CORESCOPE_TEST_BACKEND: ${{ matrix.backend }} + CORESCOPE_DB_BACKEND: ${{ matrix.backend }} + CORESCOPE_TEST_POSTGRES_URL: ${{ matrix.backend == 'postgres' && format('postgresql://postgres:ci-{0}-{1}@127.0.0.1:5432/postgres?sslmode=disable', github.run_id, github.run_attempt) || '' }} + PGHOST: 127.0.0.1 + PGPORT: 5432 + PGUSER: postgres + PGPASSWORD: "ci-${{ github.run_id }}-${{ github.run_attempt }}" + PGSSLMODE: disable steps: - name: Checkout code uses: actions/checkout@v5 @@ -98,20 +247,28 @@ jobs: - name: Clean Go module cache run: rm -rf ~/go/pkg/mod 2>/dev/null || true - - name: Set up Go 1.22 + - name: Install pinned PostgreSQL 18.6 clients + if: matrix.backend == 'postgres' + run: *install_postgres_clients + + + - name: Set up Go 1.27.2 uses: actions/setup-go@v6 with: - go-version: '1.22' - cache-dependency-path: | - cmd/server/go.sum - cmd/ingestor/go.sum + go-version: '1.27.2' + # All modules: the offline SQLite importer is expensive to + # redo, and setup-go caches ~/.cache/go-build as well as the module + # cache, so a key that misses on an unrelated module hurts. + cache-dependency-path: "**/go.sum" + + - name: Test PostgreSQL benchmark controls + run: python3 -m unittest discover -s scripts/postgres-benchmark -p 'test_*.py' - name: Enforce gofmt + go vet (issue #1859) run: | set -e - # gofmt: every tracked *.go file except the misnamed Dockerfile.go, - # which is a Dockerfile (FROM ...) that gofmt cannot parse. - unformatted=$(gofmt -l $(git ls-files '*.go' | grep -vi 'Dockerfile.go')) + # Every tracked Go source file must be formatted. + unformatted=$(gofmt -l $(git ls-files '*.go')) if [ -n "$unformatted" ]; then echo "::error::gofmt required on these files:" echo "$unformatted" @@ -128,10 +285,12 @@ jobs: run: | set -e -o pipefail cd cmd/server - go build . + CGO_ENABLED=1 go build . # -race gates PR #1208's atomic.Pointer migration: the race-detector # is what makes path_inspect_atomic_race_test.go actually assert. go test -timeout 20m -race -coverprofile=server-coverage.out ./... 2>&1 | tee server-test.log + # The e2etest build (fake mailer + /__e2e/last-mail) has its own tests. + go test -tags e2etest -count=1 -run 'E2E' . echo "--- Go Server Coverage ---" go tool cover -func=server-coverage.out | tail -1 @@ -139,7 +298,7 @@ jobs: run: | set -e -o pipefail cd cmd/ingestor - go build . + CGO_ENABLED=1 go build . go test -timeout 20m -coverprofile=ingestor-coverage.out ./... 2>&1 | tee ingestor-test.log echo "--- Go Ingestor Coverage ---" go tool cover -func=ingestor-coverage.out | tail -1 @@ -151,90 +310,65 @@ jobs: go test ./... echo "--- Channel library tests passed ---" cd ../../cmd/decrypt - CGO_ENABLED=0 go build -ldflags="-s -w" -o corescope-decrypt . + CGO_ENABLED=1 go build -ldflags="-s -w" -o corescope-decrypt . go test ./... echo "--- Decrypt CLI tests passed ---" + - name: Test user-management modules (users, mailer) + run: | + set -e -o pipefail + cd internal/users && go test -race ./... + cd ../mailer && go test -race ./... + echo "--- users + mailer tests passed ---" + + - name: Test shared selection and native SQLite snapshot modules + if: matrix.backend == 'sqlite' + run: | + set -euo pipefail + cd internal/dbconfig && go test -race ./... + cd ../sqliteutil && go test -race ./... + + - name: Test migrate CLI + dbschema + run: | + set -e -o pipefail + # Neither module had CI test execution before the mattn/go-sqlite3 + # migration; both open the database, and dbschema owns the migrations. + if [ "$CORESCOPE_TEST_BACKEND" = postgres ]; then + (cd internal/pgutil && go test ./...) + fi + cd internal/dbschema + go test ./... + cd ../../cmd/migrate + CGO_ENABLED=1 go build -o /dev/null . + go test ./... + echo "--- migrate + dbschema tests passed ---" + + - name: Validate default and optional Compose models + if: matrix.backend == 'sqlite' + run: node scripts/test-storage-compose.js + - name: Verify Dockerfile COPY invariants (issue #1316) run: bash scripts/check-dockerfile-internal-pkgs.sh - name: Staging disk-monitor unit tests (issue #1684) run: bash scripts/staging/test-disk-monitor.sh + - name: QA SQL parameter-binding unit tests (issue #1977) + run: | + if [ "$CORESCOPE_TEST_BACKEND" = sqlite ] && ! command -v sqlite3 >/dev/null; then + sudo apt-get update -qq + sudo apt-get install -y sqlite3 + fi + bash qa/scripts/test-blacklist-sql.sh + - name: Lint CSS variables (issue #1128) run: | set -e node scripts/check-css-vars.js node scripts/test-check-css-vars.js - - name: Run JS unit tests (packet-filter) - run: | - set -e - node test-packet-filter.js - node test-packet-filter-time.js - node test-confidence-indicator.js - node test-1659-analytics-warmup.js - node test-channels-merge-1498-unit.js - node test-issue-1518-home-url.js - node test-channel-decrypt-insecure-context.js - node test-live-region-filter.js - node test-issue-1136-observer-iata-map.js - node test-channel-qr.js - node test-channel-qr-wiring.js - node test-channel-modal-ux.js - node test-channel-issue-1087.js - node test-issue-1409-no-encrypted-flood.js - node test-channel-issue-1101.js - node test-observer-iata-1188.js - node test-pull-to-reconnect-1091.js - node test-channel-fluid-layout.js - node test-issue-1279-p2-code-filter.js - node test-area-filter.js - node test-issue-1293-marker-shapes.js - node test-issue-1356-map-a11y.js - node test-issue-1360-pill-letter-count.js - node test-issue-1364-pill-no-clamp.js - node test-issue-1375-scope-stats-fetch.js - node test-issue-1361-cb-presets.js - node test-issue-1380-cb-sim-overlay.js - node test-issue-1380-cb-reset-button.js - node test-issue-1407-cb-preset-propagation.js - node test-issue-1412-customizer-no-override.js - node test-issue-1418-raw-hex-extraction.js - node test-issue-1418-edge-weights.js - node test-issue-1418-cb-preset-ramp.js - node test-issue-1418-spider-fan.js - node test-issue-1418-deeplink-hops-channels.js - node test-issue-1418-polish-review.js - node test-issue-1420-tile-providers.js - node test-issue-1614-tile-url-function.js - node test-issue-1438-marker-css-vars.js - node test-issue-1846-observers-width.js - node test-issue-1562-observers-summary.js - node test-issue-1509-nav-active-bg.js - node test-issue-1509-detect-preset.js - node test-live.js - node test-coverage-gate.js - node test-node-reach-coverage.js - node test-issue-1107-live-layout.js - node test-issue-1532-live-fullscreen.js - node test-issue-1619-feed-detail-card-draggable.js - node test-xss-escape-sinks.js - node test-preflight-xss-gate.js - node test-traces.js - node test-issue-1648-m4-emoji-scan.js - node test-issue-1753-copy-url-slash.js - node test-issue-1668-m3-typography.js - node test-mqtt-status-panel.js - node test-issue-1697-mqtt-mobile-e2e.js - node test-warmup-banner.js - node test-issue-1633-hide-1byte-hops.js - node test-issue-1668-m4-per-route.js - node test-a11y-axe-1668-selftest.js - node test-a11y-1716-rf-range-btn-active.js - node test-issue-1705-subpath-contrast.js - node test-issue-1770-mobile-row-clamp.js - node test-a11y-axe-routes-coverage.js + - name: Run JS unit tests + run: sh test-all.sh - name: 🛡️ Preflight XSS gate — actual --diff check (PR only) # The fixture self-test above (test-preflight-xss-gate.js) only @@ -274,7 +408,7 @@ jobs: echo "✅ All .proto files are syntactically valid" - name: Generate Go coverage badges - if: success() + if: success() && matrix.backend == 'sqlite' run: | mkdir -p .badges @@ -303,7 +437,7 @@ jobs: echo "| Ingestor | ${INGESTOR_COV}% |" >> $GITHUB_STEP_SUMMARY - name: Upload Go coverage badges - if: success() + if: success() && matrix.backend == 'sqlite' uses: actions/upload-artifact@v6 with: name: go-badges @@ -315,14 +449,39 @@ jobs: # ─────────────────────────────────────────────────────────────── # 2. Playwright E2E Tests (against Go server with fixture DB) # ─────────────────────────────────────────────────────────────── - e2e-test: - name: "🎭 Playwright E2E Tests" - needs: [go-test, changes] - runs-on: ubuntu-latest + # Three shards on three runners, each with its own server and fixture copy, + # so no shard shares a browser, a CPU or server state with another. One + # runner per shard rather than parallel suites on one runner: the gesture and + # animation suites are timing-sensitive and would flake under CPU contention. + # + # It needs only `changes`, not go-test: this job builds its own binaries and + # uses nothing go-test produces. build-and-publish still needs both, so + # nothing reaches GHCR unless every test job passed. + e2e-shard: + name: "🎭 Playwright E2E (${{ matrix.backend }}) shard ${{ matrix.shard }}/3" + needs: [changes] + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3] + backend: [sqlite, postgres] + env: + CORESCOPE_TEST_BACKEND: ${{ matrix.backend }} + CORESCOPE_DB_BACKEND: ${{ matrix.backend }} + CORESCOPE_TEST_POSTGRES_URL: ${{ matrix.backend == 'postgres' && format('postgresql://postgres:ci-{0}-{1}@127.0.0.1:5432/postgres?sslmode=disable', github.run_id, github.run_attempt) || '' }} + PGHOST: 127.0.0.1 + PGPORT: 5432 + PGUSER: postgres + PGPASSWORD: "ci-${{ github.run_id }}-${{ github.run_attempt }}" + PGSSLMODE: disable + SHARD: ${{ matrix.shard }} + E2E_SHARDS: 3 defaults: run: shell: bash - if: needs.changes.outputs.code == 'true' + if: needs.changes.outputs.code == 'true' && !(startsWith(github.ref, 'refs/tags/v') && inputs.images_published) + services: *postgres_test_services steps: - name: Checkout code uses: actions/checkout@v5 @@ -337,22 +496,36 @@ jobs: - name: Clean Go module cache run: rm -rf ~/go/pkg/mod 2>/dev/null || true - - name: Set up Go 1.22 + - name: Install pinned PostgreSQL 18.6 clients + if: matrix.backend == 'postgres' + run: *install_postgres_clients + + + - name: Set up Go 1.27.2 uses: actions/setup-go@v6 with: - go-version: '1.22' - cache-dependency-path: cmd/server/go.sum + go-version: '1.27.2' + cache-dependency-path: "**/go.sum" - name: Build Go server run: | cd cmd/server - go build -o ../../corescope-server . + CGO_ENABLED=1 go build -o ../../corescope-server . echo "Go server built successfully" + # Only test-user-management-e2e.js uses the e2etest build and its server, + # and it runs on shard 3. Moving that suite means moving these two too; + # it fails loudly (nothing on port 13582) rather than skipping. + - name: Build Go server (e2etest tag, user management) + if: matrix.shard == 3 + run: | + cd cmd/server + CGO_ENABLED=1 go build -tags e2etest -o ../../corescope-server-e2e . + - name: Build Go migrate tool run: | cd cmd/migrate - go build -o ../../corescope-migrate . + CGO_ENABLED=1 go build -o ../../corescope-migrate . echo "Go migrate tool built successfully" - name: Install npm dependencies @@ -366,8 +539,17 @@ jobs: - name: Instrument frontend JS for coverage run: sh scripts/instrument-frontend.sh + - name: Prepare disposable legacy fixture copy + run: | + mkdir -p "$RUNNER_TEMP/corescope-fixture" + cd cmd/migrate + go run ../../scripts/prepare-postgres-fixture.go \ + -source ../../test-fixtures/e2e-fixture.db \ + -destination "$RUNNER_TEMP/corescope-fixture/fixture.sqlite" + echo "E2E_SQLITE=$RUNNER_TEMP/corescope-fixture/fixture.sqlite" >> "$GITHUB_ENV" + - name: Freshen fixture timestamps - run: bash tools/freshen-fixture.sh test-fixtures/e2e-fixture.db + run: bash tools/freshen-fixture.sh "$E2E_SQLITE" - name: Seed grouped-packet row for #1486 collapse test # The committed fixture has 499 packets, each with exactly ONE @@ -376,39 +558,28 @@ jobs: # (toggle-select) row. Insert a NEW transmission with 3 # observations. # - # The server's async hash-migrate (cmd/server/hash_migrate.go) - # recomputes `transmissions.hash` from `raw_hex` via - # ComputeContentHash(), so the inserted hash MUST equal that - # function's output for the chosen raw_hex — otherwise the row - # gets relabelled and the E2E can't find it. + # Fixture hashes are normalized offline before PostgreSQL import. + # The read-only server verifies them and never rewrites telemetry. + # This seeded hash must equal the shared runtime ContentHash output. # # raw_hex 15000102030405060708090a0b0c0d0e0f # → header=0x15 (route_type=1, payload_type=5) # → ComputeContentHash(...) = fae0c9e6d357a814 # - # The first_seen / observation timestamps are pinned to a date - # within retentionHours but outside the default 15-min UI - # window so the row is hidden in the default view (keeping - # test-e2e-playwright's first-10-rows hex-pane test - # unaffected) and reachable via the explicit ?timeWindow=0 - # deep-link the #1486 test uses. - run: | - sqlite3 test-fixtures/e2e-fixture.db <<'SQL' + # Fixed old timestamps keep the row behind the freshly shifted + # fixture observations and outside the default 15-min UI window. + # The #1486 and #2104 tests reach it via an explicit hash + + # ?timeWindow=0 deep-link. + run: | + sqlite3 "$E2E_SQLITE" <<'SQL' -- Sort the seeded row LAST in BOTH default packets views: -- • flat view sorts by transmissions.id DESC → id=0 puts it last -- • grouped view (#default for the packets page) sorts by -- MAX(observations.timestamp) DESC → we must keep our obs -- timestamps OLDER than every other fixture observation. - -- Fixture (after freshen) has obs timestamps spanning - -- 2026-05-17 16:01:39Z .. 2026-05-28 00:00:00Z (max). - -- Note: freshen only shifts transmissions.first_seen forward - -- to ~now; observation.timestamp is left alone except for - -- the timestamp=0 case. - -- Use 2026-05-15 (~2 days older than the oldest fixture obs) - -- so our row sorts LAST in the grouped view too, keeping - -- test-e2e-playwright's first-10-rows hex-pane test - -- unaffected. The #1486 test still reaches the row via the - -- explicit hash + ?timeWindow=0 deep-link. + -- freshen-fixture.sh shifts fixture observations forward. + -- Keep this row at 2026-05-15 so it sorts LAST in the grouped + -- view too; the #1486 and #2104 tests use the explicit deep-link. INSERT INTO transmissions(id,raw_hex,hash,first_seen,route_type,payload_type,payload_version,decoded_json,channel_hash,from_pubkey) VALUES (0,'15000102030405060708090a0b0c0d0e0f','fae0c9e6d357a814','2026-05-15T00:00:00Z',1,5,0,'{"type":"CHAN","channel":"#test","text":"#1486 fixture"}',NULL,NULL); INSERT INTO observations(transmission_id,observer_idx,direction,snr,rssi,score,path_json,timestamp,resolved_path) VALUES @@ -438,20 +609,74 @@ jobs: (-1000000,1,'rx',7.0,-88,0,'[]',CAST(strftime('%s','now') AS INTEGER),'[]'); SQL - - name: Migrate fixture DB to current schema (#1287) - # Server now ASSERTs schema is migrated and refuses to start - # otherwise (cmd/server/main.go: dbschema.AssertReady). In prod - # the ingestor owns dbschema.Apply, but CI starts only the - # server against the committed e2e fixture — so we run the - # standalone migrate tool here to bring the fixture up to the - # required shape before the server boots. - run: ./corescope-migrate -db test-fixtures/e2e-fixture.db + - name: 'Seed distinct observation bytes for #2104' + # raw_hex exists only after migration. Keep the #1486 payload/hash + # while encoding each observation's existing one-hop path. + run: | + sqlite3 "$E2E_SQLITE" <<'SQL' + UPDATE observations SET raw_hex = CASE observer_idx + WHEN 1 THEN '1501aa0102030405060708090a0b0c0d0e0f' + WHEN 2 THEN '1501bb0102030405060708090a0b0c0d0e0f' + WHEN 3 THEN '1501cc0102030405060708090a0b0c0d0e0f' + END WHERE transmission_id = 0 AND observer_idx IN (1,2,3); + SQL + + - name: Seed deterministic Path Inspector routes (#2060) + run: sqlite3 "$E2E_SQLITE" < test-fixtures/path-inspector.sql + + - name: Normalize fixture hashes using the runtime parser + run: | + cd cmd/migrate + go run ../../scripts/migrate-fixture-hashes.go "$E2E_SQLITE" + + - name: Import fixture and grant restricted runtime roles + if: matrix.backend == 'postgres' + run: | + set -euo pipefail + for key in CORESCOPE_OWNER_PASSWORD CORESCOPE_READER_PASSWORD CORESCOPE_WRITER_PASSWORD CORESCOPE_ACCOUNTS_PASSWORD CORESCOPE_CHANNELS_PASSWORD; do + printf -v "$key" '%s' "$(openssl rand -hex 32)" + export "$key" + echo "::add-mask::$(printenv "$key")" + done + sh docker/postgres-init.sh + owner_url="postgresql://corescope_owner:$CORESCOPE_OWNER_PASSWORD@127.0.0.1:5432/corescope_telemetry?sslmode=disable" + owner_accounts="postgresql://corescope_owner:$CORESCOPE_OWNER_PASSWORD@127.0.0.1:5432/corescope_accounts?sslmode=disable" + CORESCOPE_DATABASE_URL="$owner_url" CORESCOPE_USERS_DATABASE_URL="$owner_accounts" \ + ./corescope-migrate -offline -from-sqlite "$E2E_SQLITE" -state-dir "$RUNNER_TEMP/corescope-import" + for database in corescope_telemetry corescope_accounts; do + PGUSER=corescope_owner PGPASSWORD="$CORESCOPE_OWNER_PASSWORD" PGDATABASE="$database" \ + psql -X -v ON_ERROR_STOP=1 -f docker/postgres-grants.sql + done + export CORESCOPE_DATABASE_URL="$owner_url" CORESCOPE_USERS_OWNER_DATABASE_URL="$owner_accounts" + export CORESCOPE_READER_DATABASE_URL="postgresql://corescope_reader:$CORESCOPE_READER_PASSWORD@127.0.0.1:5432/corescope_telemetry?sslmode=disable&search_path=public" + export CORESCOPE_WRITER_DATABASE_URL="postgresql://corescope_writer:$CORESCOPE_WRITER_PASSWORD@127.0.0.1:5432/corescope_telemetry?sslmode=disable&search_path=public" + export CORESCOPE_USERS_DATABASE_URL="postgresql://corescope_accounts:$CORESCOPE_ACCOUNTS_PASSWORD@127.0.0.1:5432/corescope_accounts?sslmode=disable&search_path=public" + export CORESCOPE_APPROVED_CHANNELS_DATABASE_URL="postgresql://corescope_channels:$CORESCOPE_CHANNELS_PASSWORD@127.0.0.1:5432/corescope_accounts?sslmode=disable&search_path=public" + for key in CORESCOPE_DATABASE_URL CORESCOPE_USERS_OWNER_DATABASE_URL CORESCOPE_READER_DATABASE_URL CORESCOPE_WRITER_DATABASE_URL CORESCOPE_USERS_DATABASE_URL CORESCOPE_APPROVED_CHANNELS_DATABASE_URL; do + printf '%s=%s\n' "$key" "${!key}" >> "$GITHUB_ENV" + done + mkdir -p "$RUNNER_TEMP/corescope-state" + ./corescope-migrate -storage-action=adopt -backend=postgres -offline \ + -selection-file "$RUNNER_TEMP/corescope-state/storage-selection.json" + # Retain the masked owner password for the later restore step only. + umask 077 + printf '%s' "$CORESCOPE_OWNER_PASSWORD" > "$RUNNER_TEMP/corescope-e2e-owner-password" + + - name: Adopt prepared SQLite fixture without a PostgreSQL service + if: matrix.backend == 'sqlite' + run: | + set -euo pipefail + mkdir -p "$RUNNER_TEMP/corescope-state" + ./corescope-migrate -storage-action=setup -backend=sqlite -offline \ + -selection-file "$RUNNER_TEMP/corescope-state/storage-selection.json" \ + -sqlite-path "$E2E_SQLITE" -users-sqlite-path "$RUNNER_TEMP/corescope-state/users.db" + printf 'DB_PATH=%s\n' "$E2E_SQLITE" >> "$GITHUB_ENV" - name: Start Go server with fixture DB run: | fuser -k 13581/tcp 2>/dev/null || true sleep 1 - ./corescope-server -port 13581 -db test-fixtures/e2e-fixture.db -public public-instrumented & + env -u PGPASSWORD -u PGUSER -u CORESCOPE_TEST_POSTGRES_URL -u CORESCOPE_DATABASE_URL -u CORESCOPE_USERS_OWNER_DATABASE_URL ./corescope-server -port 13581 -state-dir "$RUNNER_TEMP/corescope-state" -public public-instrumented & echo $! > .server.pid for i in $(seq 1 30); do if curl -sf http://localhost:13581/api/healthz > /dev/null 2>&1; then @@ -465,9 +690,65 @@ jobs: sleep 1 done + - name: Start user-management E2E server (fake mailer) + if: matrix.shard == 3 + run: | + fuser -k 13582/tcp 2>/dev/null || true + rm -rf /tmp/cs-um + mkdir -p /tmp/cs-um + # Shared read-only telemetry; the account writer has its own logical database. + cat > /tmp/cs-um/config.json <<'JSON' + { + "port": 13582, + "userManagement": { + "enabled": true, + "adminEmails": ["admin@e2e.test"], + "publicBaseUrl": "http://localhost:13582", + "mail": { "provider": "fake", "fromEmail": "noreply@e2e.test" }, + "channelProposals": { "enabled": true }, + "notifications": { "enabled": true } + } + } + JSON + if [ "$CORESCOPE_TEST_BACKEND" = sqlite ]; then + # Existing telemetry is adopted with no account target; initialize + # first accounts explicitly before starting the strict runtime. + ./corescope-migrate -storage-action=adopt -backend=sqlite -offline -config-dir /tmp/cs-um \ + -selection-file /tmp/cs-um/storage-selection.json -sqlite-path "$E2E_SQLITE" -users-sqlite-path /tmp/cs-um/users.db + ./corescope-migrate -storage-action=setup -backend=sqlite -offline -config-dir /tmp/cs-um \ + -selection-file /tmp/cs-um/storage-selection.json -sqlite-path "$E2E_SQLITE" -users-sqlite-path /tmp/cs-um/users.db + else + ./corescope-migrate -storage-action=adopt -backend=postgres -offline -config-dir /tmp/cs-um \ + -selection-file /tmp/cs-um/storage-selection.json + fi + env -u PGPASSWORD -u PGUSER -u CORESCOPE_TEST_POSTGRES_URL -u CORESCOPE_DATABASE_URL -u CORESCOPE_USERS_OWNER_DATABASE_URL ./corescope-server-e2e -config-dir /tmp/cs-um -port 13582 -state-dir /tmp/cs-um -public public-instrumented & + echo $! > .server-um.pid + for i in $(seq 1 30); do + if curl -sf http://localhost:13582/api/healthz > /dev/null 2>&1; then echo "UM server ready after ${i}s"; break; fi + if [ "$i" -eq 30 ]; then echo "UM server failed to start within 30s"; exit 1; fi + sleep 1 + done + - name: Run Playwright E2E tests (fail-fast) run: | - BASE_URL=http://localhost:13581 node test-e2e-playwright.js 2>&1 | tee e2e-output.txt + # suite [VAR=value ...]: runs tests/e2e/ with + # exactly those variables, but only on its own shard. The shard numbers + # balance the measured suite times; give a new suite the shard whose + # total is lowest. Anything but a number in 1..E2E_SHARDS (a missing + # number, 01, 4) fails the step, so a typo cannot silently drop a suite. + B=http://localhost:13581 + suite() { + local shard=$1 file=$2 + shift 2 + if ! [[ "$shard" =~ ^[1-9][0-9]*$ ]] || [ "$shard" -gt "$E2E_SHARDS" ]; then + echo "::error::$file is assigned to shard $shard, outside 1..$E2E_SHARDS" + exit 1 + fi + [ "$shard" -eq "$SHARD" ] || return 0 + echo "=== E2E SUITE: $file ===" | tee -a e2e-output.txt + env "$@" node "tests/e2e/$file" 2>&1 | tee -a e2e-output.txt + } + suite 2 test-e2e-playwright.js BASE_URL=$B # M5+M6 of #1668 — axe-core CI gate. # M5: color-contrast on desktop dark+light. # M6: expanded ruleset (image-alt, label, aria-required-attr, @@ -477,137 +758,262 @@ jobs: # color-contrast on mobile too). # Allowlist: tests/a11y-allowlist.yaml (0 entries — hard pass policy). # Per-viewport summary printed at the end; any net>0 fails the build. - BASE_URL=http://localhost:13581 AXE_SCREENSHOT_DIR=/tmp/axe-1668 \ - node test-a11y-axe-1668.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-filter-ux-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-channel-issue-1087-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-channel-issue-1111-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-map-modal-fluid-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-map-nodes-pagination-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-observer-iata-1188-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-1639-observers-sort-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1758-ng-filter-rerenders-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-nav-fluid-1055-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-nav-priority-1102-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-nav-priority-1311-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-nav-priority-1391-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1413-nav-overlap-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1400-nav-vertical-clip.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-nav-more-floor-1139-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-bottom-nav-1061-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-gestures-1062-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-gestures-1185-scroll-discriminator-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-gesture-hints-1065-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-touch-gestures-coverage-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-channel-fluid-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-table-fluid-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-charts-fluid-1058-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-slideover-1056-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-1692-packets-init-parallel-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-slideover-1168-munger-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-logo-pulse-1173-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-1122-packets-filter-ux-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-1128-packets-layout-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-1128-multi-viewport-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-1136-live-region-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-live-multibyte-only-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-1150-404-state-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-1146-path-link-contrast-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 node test-issue-1705-subpath-contrast-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-1147-section-order-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-1151-orphan-separators-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-1486-collapse-reopens-detail-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-logo-rebrand-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-logo-theme-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-logo-default-sage-teal-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1109-hamburger-dropdown-visible-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-live-layout-1178-1179-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1205-live-controls-anchor-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-live-mql-leak-1180-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1204-live-panel-structure-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1234-live-chrome-pass2-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1206-vcr-overlap-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1244-live-vcr-row-hints-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1510-live-nav-pin-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-live-fullscreen-1572-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1599-replay-freeze-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1648-m1-icons-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1648-m2-icons-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1648-m3-icons-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1648-m4-icons-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1657-analytics-channels-group-sprites-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-1224-channels-mobile-ux-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-1367-channels-chat-app-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-1236-map-mobile-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-1329-map-controls-accordion-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-1273-qr-overlay-height-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-1281-location-row-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-1279-legend-p2-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-1799-label-vocab-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-home-coverage-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-path-inspector-coverage-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1206-resize-observer-leak-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-nav-drawer-1064-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-audio-live-1297-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-audio-lab-1297-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-channel-decrypt-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-channel-qr-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-channel-color-picker-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-customize-theme-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-customize-branding-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-customize-display-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-customize-export-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-drag-manager-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1567-corner-clears-drag-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1306-collisions-terminology-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1374-route-map-a11y-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-channels-list-render-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-channels-selection-flow-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-channels-add-modal-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-channels-share-color-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-channels-ws-batch-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-channels-ws-race-1498-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1487-byop-modal-layout-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1630-reach-mobile-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-node-reach-coverage-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1640-compare-discovery-e2e.js 2>&1 | tee -a e2e-output.txt + suite 1 test-a11y-axe-1668.js BASE_URL=$B AXE_SCREENSHOT_DIR=/tmp/axe-1668 + suite 2 test-filter-ux-e2e.js BASE_URL=$B + suite 1 test-channel-issue-1087-e2e.js BASE_URL=$B + suite 1 test-channel-issue-1111-e2e.js BASE_URL=$B + suite 2 test-map-modal-fluid-e2e.js BASE_URL=$B + suite 2 test-map-nodes-pagination-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 3 test-observer-iata-1188-e2e.js BASE_URL=$B + suite 2 test-issue-1639-observers-sort-e2e.js BASE_URL=$B + suite 2 test-issue-1758-ng-filter-rerenders-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 2 test-nav-fluid-1055-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 3 test-nav-priority-1102-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 3 test-nav-priority-1311-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 2 test-nav-priority-1391-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 1 test-issue-1413-nav-overlap-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 2 test-issue-1400-nav-vertical-clip.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 2 test-nav-more-floor-1139-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 3 test-bottom-nav-1061-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 3 test-gestures-1062-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 1 test-gestures-1185-scroll-discriminator-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 3 test-gesture-hints-1065-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 3 test-touch-gestures-coverage-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 2 test-channel-fluid-e2e.js BASE_URL=$B + suite 2 test-table-fluid-e2e.js BASE_URL=$B + suite 3 test-charts-fluid-1058-e2e.js BASE_URL=$B + suite 3 test-slideover-1056-e2e.js BASE_URL=$B + suite 3 test-issue-1692-packets-init-parallel-e2e.js BASE_URL=$B + suite 2 test-slideover-1168-munger-e2e.js BASE_URL=$B + suite 3 test-logo-pulse-1173-e2e.js BASE_URL=$B + suite 2 test-issue-1122-packets-filter-ux-e2e.js BASE_URL=$B + suite 3 test-issue-1128-packets-layout-e2e.js BASE_URL=$B + suite 3 test-issue-1128-multi-viewport-e2e.js BASE_URL=$B + suite 2 test-issue-1136-live-region-e2e.js BASE_URL=$B + suite 1 test-live-multibyte-only-e2e.js BASE_URL=$B + suite 3 test-issue-1150-404-state-e2e.js BASE_URL=$B + suite 2 test-issue-1146-path-link-contrast-e2e.js BASE_URL=$B + suite 1 test-issue-1705-subpath-contrast-e2e.js CHROMIUM_REQUIRE=1 + suite 2 test-issue-1147-section-order-e2e.js BASE_URL=$B + suite 3 test-issue-1151-orphan-separators-e2e.js BASE_URL=$B + suite 3 test-issue-1486-collapse-reopens-detail-e2e.js BASE_URL=$B + suite 1 test-logo-rebrand-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 2 test-logo-theme-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 2 test-logo-default-sage-teal-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 3 test-issue-1109-hamburger-dropdown-visible-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 2 test-live-layout-1178-1179-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 2 test-issue-1205-live-controls-anchor-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 3 test-live-mql-leak-1180-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 2 test-issue-1204-live-panel-structure-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 3 test-issue-1234-live-chrome-pass2-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 1 test-issue-1206-vcr-overlap-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 1 test-issue-1833-legend-toggle-clickable-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 3 test-issue-1244-live-vcr-row-hints-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 2 test-issue-1510-live-nav-pin-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 3 test-live-fullscreen-1572-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 3 test-issue-1599-replay-freeze-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 2 test-issue-1648-m1-icons-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 3 test-issue-1648-m2-icons-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 2 test-issue-1648-m3-icons-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 1 test-issue-1648-m4-icons-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 1 test-issue-1657-analytics-channels-group-sprites-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 2 test-issue-1224-channels-mobile-ux-e2e.js BASE_URL=$B + suite 3 test-issue-1367-channels-chat-app-e2e.js BASE_URL=$B + suite 3 test-issue-1236-map-mobile-e2e.js BASE_URL=$B + suite 3 test-issue-1329-map-controls-accordion-e2e.js BASE_URL=$B + suite 1 test-issue-1273-qr-overlay-height-e2e.js BASE_URL=$B + suite 3 test-issue-1281-location-row-e2e.js BASE_URL=$B + suite 1 test-issue-1279-legend-p2-e2e.js BASE_URL=$B + suite 2 test-issue-1799-label-vocab-e2e.js BASE_URL=$B + suite 2 test-home-coverage-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 2 test-path-inspector-coverage-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 3 test-issue-1206-resize-observer-leak-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 3 test-nav-drawer-1064-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 3 test-audio-live-1297-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 3 test-audio-lab-1297-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 1 test-channel-decrypt-e2e.js BASE_URL=$B + suite 2 test-channel-qr-e2e.js BASE_URL=$B + suite 3 test-channel-color-picker-e2e.js BASE_URL=$B + suite 1 test-customize-theme-e2e.js BASE_URL=$B + suite 3 test-customize-branding-e2e.js BASE_URL=$B + suite 2 test-customize-display-e2e.js BASE_URL=$B + suite 3 test-customize-export-e2e.js BASE_URL=$B + suite 2 test-drag-manager-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 1 test-issue-1567-corner-clears-drag-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 3 test-issue-1306-collisions-terminology-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 2 test-issue-1374-route-map-a11y-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 2 test-channels-list-render-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 1 test-channels-selection-flow-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 3 test-channels-add-modal-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 1 test-channels-share-color-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 2 test-channels-ws-batch-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 3 test-channels-ws-race-1498-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 3 test-issue-1487-byop-modal-layout-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 2 test-issue-1630-reach-mobile-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 1 test-node-reach-coverage-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 3 test-issue-1640-compare-discovery-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 2 test-neighbor-map-btn-clip-e2e.js CHROMIUM_REQUIRE=1 + suite 3 test-rx-coverage-viewport-e2e.js BASE_URL=$B + suite 1 test-rx-coverage-noise-e2e.js BASE_URL=$B + # Issue #2037: these eight sat in tests/e2e/ with no runner at all. + # Measured on upstream master in CI run 35340835178: all eight pass, + # 45 seconds together. test-rx-coverage-mobile-nav-e2e.js is the + # ninth that passed and is deliberately NOT here: it exits 0 with + # "SKIP (clientRxCoverage disabled on this deployment)", which is the + # default, so wiring it in would add a vacuous pass rather than a guard. + suite 2 test-1110-live-filter.js BASE_URL=$B + suite 3 test-analytics-fluid-charts.js BASE_URL=$B + suite 3 test-e2e-1267-mobile-vcr.js BASE_URL=$B + suite 2 test-issue-1274-legend-coverage-e2e.js BASE_URL=$B + suite 3 test-issue-1648-m5-icons-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 2 test-issue-1997-distance-building-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 1 test-channel-modal-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 1 test-issue-1522-trace-url-sync-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 2 test-marker-outline-weight.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 2 test-node-reach-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 3 test-packets-compact-columns-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 3 test-packets-scope-column.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 3 test-path-inspector-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 3 test-pr-1490-live-map-gpu-animations-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 3 test-table-sort.js + suite 1 test-touch-targets.js CHROMIUM_REQUIRE=1 BASE_URL=$B + suite 3 test-live-dedup.js BASE_URL=$B + suite 1 test-nodes-export-e2e.js BASE_URL=$B + suite 2 test-show-neighbors.js BASE_URL=$B + suite 3 test-user-management-e2e.js BASE_URL=http://localhost:13582 BASE_URL_OFF=$B + + - name: Restore native HTTP backups and verify browser sessions + if: matrix.shard == 3 + run: | + set -euo pipefail + export E2E_BACKUP_DIR="$RUNNER_TEMP/corescope-http-backup" + export BASE_URL=http://localhost:13582 + echo "=== E2E SUITE: test-postgres-backup-e2e.js capture ===" | tee -a e2e-output.txt + node tests/e2e/test-postgres-backup-e2e.js capture 2>&1 | tee -a e2e-output.txt + for pidfile in .server.pid .server-um.pid; do + pid=$(cat "$pidfile") + kill "$pid" + for attempt in $(seq 1 30); do + kill -0 "$pid" 2>/dev/null || break + sleep 1 + done + ! kill -0 "$pid" 2>/dev/null || { echo 'Fixture server did not stop' >&2; exit 1; } + done + if [ "$CORESCOPE_TEST_BACKEND" = postgres ]; then + # Fresh databases, same deployment roles, no overwrite or role widening. + psql -X -v ON_ERROR_STOP=1 -d postgres <<'SQL' + CREATE DATABASE corescope_restore_telemetry OWNER corescope_owner TEMPLATE template0 ENCODING 'UTF8' LOCALE 'C'; + CREATE DATABASE corescope_restore_accounts OWNER corescope_owner TEMPLATE template0 ENCODING 'UTF8' LOCALE 'C'; + REVOKE ALL ON DATABASE corescope_restore_telemetry FROM PUBLIC; + REVOKE ALL ON DATABASE corescope_restore_accounts FROM PUBLIC; + GRANT CONNECT ON DATABASE corescope_restore_telemetry TO corescope_reader, corescope_writer; + GRANT CONNECT ON DATABASE corescope_restore_accounts TO corescope_accounts, corescope_channels; + SQL + for kind in telemetry accounts; do + PGUSER=corescope_owner PGPASSWORD="$(cat "$RUNNER_TEMP/corescope-e2e-owner-password")" \ + pg_restore --no-owner --no-privileges --exit-on-error --single-transaction \ + --dbname="corescope_restore_$kind" "$E2E_BACKUP_DIR/$kind.dump" + PGUSER=corescope_owner PGPASSWORD="$(cat "$RUNNER_TEMP/corescope-e2e-owner-password")" \ + psql -X -v ON_ERROR_STOP=1 -d "corescope_restore_$kind" -f docker/postgres-grants.sql + PGUSER=corescope_owner PGPASSWORD="$(cat "$RUNNER_TEMP/corescope-e2e-owner-password")" \ + psql -X -v ON_ERROR_STOP=1 -d "corescope_restore_$kind" -c 'ANALYZE' + done + # Switch restored database URLs. + restore_urls=$(python3 - <<'PY' + import os + from urllib.parse import urlsplit, urlunsplit + for key, database in ( + ("CORESCOPE_READER_DATABASE_URL", "corescope_restore_telemetry"), + ("CORESCOPE_USERS_DATABASE_URL", "corescope_restore_accounts"), + ("CORESCOPE_WRITER_DATABASE_URL", "corescope_restore_telemetry"), + ("CORESCOPE_APPROVED_CHANNELS_DATABASE_URL", "corescope_restore_accounts"), + ("CORESCOPE_DATABASE_URL", "corescope_restore_telemetry"), + ("CORESCOPE_USERS_OWNER_DATABASE_URL", "corescope_restore_accounts"), + ): + raw = os.environ[key] + try: + parsed = urlsplit(raw) + except ValueError: + raise SystemExit("Invalid restore database URL") from None + if parsed.scheme not in ("postgres", "postgresql") or not parsed.netloc or "\n" in raw or "\r" in raw: + raise SystemExit("Invalid restore database URL") + print(key + "=" + urlunsplit(parsed._replace(path="/" + database))) + PY + ) + while IFS= read -r assignment; do export "$assignment"; done <<< "$restore_urls" + unset restore_urls assignment + # End restored database URLs. + else + mkdir -m 700 "$RUNNER_TEMP/corescope-restored-sqlite" + for kind in telemetry accounts; do + cp "$E2E_BACKUP_DIR/$kind.db" "$RUNNER_TEMP/corescope-restored-sqlite/$kind.db" + test "$(sqlite3 -readonly "$RUNNER_TEMP/corescope-restored-sqlite/$kind.db" 'PRAGMA quick_check')" = ok + done + export DB_PATH="$RUNNER_TEMP/corescope-restored-sqlite/telemetry.db" + fi + # A restore uses new verified selection records; changing only role URLs + # must never silently replace the recorded old targets. + restored_state="$RUNNER_TEMP/corescope-restored-state" + restored_accounts_state="$RUNNER_TEMP/corescope-restored-accounts-state" + mkdir -m 700 "$restored_state" "$restored_accounts_state" + for selection in "$restored_state/storage-selection.json" "$restored_accounts_state/storage-selection.json"; do + if [ "$CORESCOPE_TEST_BACKEND" = sqlite ]; then + ./corescope-migrate -storage-action=adopt -backend=sqlite -offline \ + -selection-file "$selection" -sqlite-path "$DB_PATH" \ + -users-sqlite-path "$RUNNER_TEMP/corescope-restored-sqlite/accounts.db" + else + ./corescope-migrate -storage-action=adopt -backend=postgres -offline -selection-file "$selection" + fi + done + # Restart restored servers. + env -u PGPASSWORD -u PGUSER -u CORESCOPE_TEST_POSTGRES_URL -u CORESCOPE_DATABASE_URL -u CORESCOPE_USERS_OWNER_DATABASE_URL ./corescope-server -port 13581 -state-dir "$restored_state" -public public-instrumented & + echo $! > .server.pid + env -u PGPASSWORD -u PGUSER -u CORESCOPE_TEST_POSTGRES_URL -u CORESCOPE_DATABASE_URL -u CORESCOPE_USERS_OWNER_DATABASE_URL ./corescope-server-e2e -config-dir /tmp/cs-um -port 13582 -state-dir "$restored_accounts_state" -public public-instrumented & + echo $! > .server-um.pid + for port in 13581 13582; do + for attempt in $(seq 1 30); do + curl -sf "http://localhost:$port/api/healthz" >/dev/null && break + [ "$attempt" -lt 30 ] || { echo 'Restored server did not become ready' >&2; exit 1; } + sleep 1 + done + done + echo "=== E2E SUITE: test-postgres-backup-e2e.js verify ===" | tee -a e2e-output.txt + node tests/e2e/test-postgres-backup-e2e.js verify 2>&1 | tee -a e2e-output.txt # #1616: slide-over focus-restore flake-gate. Runs the slide-over - # E2E 20 consecutive times against the SAME backend instance so - # the Chromium-headless focus race documented in #1172/#1616 has - # a 20× shot at firing. Any single non-zero exit aborts. This is - # the architectural-fix gate — if it ever turns red post-merge, - # the focused-but-hidden state has crept back in. + # E2E 3 more times against the SAME backend instance (it already + # ran once in the list above) so the Chromium-headless focus race + # documented in #1172/#1616 gets repeated shots at firing. Any + # single non-zero exit aborts. This is the architectural-fix gate: + # if it ever turns red post-merge, the focused-but-hidden state has + # crept back in. # - # PERMANENT step. Adds ~3-4 min to the e2e-test job in exchange - # for closing out a flake family that was blocking ~8 unrelated - # PRs at a time. If profiling pressures the budget later, drop - # repeat count first; do not delete. + # PERMANENT step, on shard 3 (the shard that runs the slide-over + # suite). Adds ~70s to that shard in exchange for closing out a + # flake family that was blocking ~8 unrelated PRs at a time. If + # profiling pressures the budget later, drop repeat count first; + # do not delete. - name: Slide-over E2E flake-gate (#1616, --repeat-each=3) + if: matrix.shard == 3 run: | set -e for i in $(seq 1 3); do - echo "--- slide-over E2E run $i/20 ---" - BASE_URL=http://localhost:13581 node test-slideover-1056-e2e.js 2>&1 | tee -a slideover-repeat-output.txt + echo "--- slide-over E2E run $i/3 ---" + echo "=== E2E SUITE: test-slideover-1056-e2e.js ===" + BASE_URL=http://localhost:13581 node tests/e2e/test-slideover-1056-e2e.js 2>&1 | tee -a slideover-repeat-output.txt done echo "3 passed" + # Shard 2 runs test-e2e-playwright.js, the one suite that writes + # .nyc_output, so the coverage collection and its badge live there too. - name: Collect frontend coverage (parallel) - if: success() && github.event_name == 'push' + if: success() && github.event_name == 'push' && matrix.shard == 2 run: | BASE_URL=http://localhost:13581 node scripts/collect-frontend-coverage.js 2>&1 | tee fe-coverage-output.txt || true - - name: Generate frontend coverage badges + - name: Generate frontend coverage badge if: success() run: | - # Aggregate per-suite PASS/FAIL across every test-*-e2e.js summary. - # The previous regex (grep -oP '[0-9]+(?=/)' | tail -1) caught a - # stray digits-before-slash like the '2' in '2/3 tests passed' from - # some sub-output and stamped the badge as '2 passed'. See #1296. - eval "$(bash scripts/aggregate-e2e-pass.sh e2e-output.txt)" - E2E_PASS=${PASS:-0} - E2E_FAIL=${FAIL:-0} - mkdir -p .badges if [ -f .nyc_output/frontend-coverage.json ] || [ -f .nyc_output/e2e-coverage.json ]; then npx nyc report --reporter=text-summary --reporter=text 2>&1 | tee fe-report.txt @@ -619,14 +1025,6 @@ jobs: echo "{\"schemaVersion\":1,\"label\":\"frontend coverage\",\"message\":\"${FE_COVERAGE}%\",\"color\":\"${FE_COLOR}\"}" > .badges/frontend-coverage.json echo "## Frontend: ${FE_COVERAGE}% coverage" >> $GITHUB_STEP_SUMMARY fi - if [ "${E2E_FAIL:-0}" -gt 0 ]; then - E2E_MSG="${E2E_PASS:-0} passed, ${E2E_FAIL} failed" - E2E_COLOR="red" - else - E2E_MSG="${E2E_PASS:-0} passed" - E2E_COLOR="brightgreen" - fi - echo "{\"schemaVersion\":1,\"label\":\"e2e tests\",\"message\":\"${E2E_MSG}\",\"color\":\"${E2E_COLOR}\"}" > .badges/e2e-tests.json - name: Stop test server if: always() @@ -635,9 +1033,77 @@ jobs: kill $(cat .server.pid) 2>/dev/null || true rm -f .server.pid fi + if [ -f .server-um.pid ]; then + kill $(cat .server-um.pid) 2>/dev/null || true + rm -f .server-um.pid + fi - - name: Upload E2E badges + - name: Upload shard results if: success() + uses: actions/upload-artifact@v6 + with: + name: e2e-shard-${{ matrix.backend }}-${{ matrix.shard }} + path: | + e2e-output.txt + .badges/ + retention-days: 1 + if-no-files-found: ignore + include-hidden-files: true + + # The gate the rest of the pipeline (and branch protection) sees under the + # old single-job name. It folds the shard outputs into the same e2e-badges + # artifact the publish job reads. + # + # !cancelled() is what makes it a gate. Without it a failed shard would SKIP + # this job, and branch protection counts a skipped required check as + # passing. With it, the job runs and fails on its first step instead. + e2e-test: + name: "🎭 Playwright E2E Tests" + needs: [e2e-shard, changes] + runs-on: ubuntu-24.04 + if: ${{ !cancelled() && needs.changes.outputs.code == 'true' && !(startsWith(github.ref, 'refs/tags/v') && inputs.images_published) }} + steps: + - name: Require every shard to have passed + run: | + result="${{ needs['e2e-shard'].result }}" + echo "e2e-shard: $result" + [ "$result" = success ] || { echo "::error::E2E shards did not all pass (result: $result)"; exit 1; } + + - name: Checkout code + uses: actions/checkout@v5 + + - name: Download shard results + uses: actions/download-artifact@v6 + with: + pattern: e2e-shard-* + path: shards/ + + - name: Generate E2E badges + run: | + set -e + ls shards + [ "$(ls -d shards/e2e-shard-* | wc -l)" -eq 6 ] || { echo "::error::expected results from 3 shards for each backend"; exit 1; } + cat shards/e2e-shard-*/e2e-output.txt > e2e-output.txt + mkdir -p .badges + cp shards/e2e-shard-*/.badges/*.json .badges/ 2>/dev/null || true + # Aggregate per-suite PASS/FAIL across every test-*-e2e.js summary. + # The previous regex (grep -oP '[0-9]+(?=/)' | tail -1) caught a + # stray digits-before-slash like the '2' in '2/3 tests passed' from + # some sub-output and stamped the badge as '2 passed'. See #1296. + eval "$(bash scripts/aggregate-e2e-pass.sh e2e-output.txt)" + E2E_PASS=${PASS:-0} + E2E_FAIL=${FAIL:-0} + if [ "${E2E_FAIL:-0}" -gt 0 ]; then + E2E_MSG="${E2E_PASS:-0} passed, ${E2E_FAIL} failed" + E2E_COLOR="red" + else + E2E_MSG="${E2E_PASS:-0} passed" + E2E_COLOR="brightgreen" + fi + echo "{\"schemaVersion\":1,\"label\":\"e2e tests\",\"message\":\"${E2E_MSG}\",\"color\":\"${E2E_COLOR}\"}" > .badges/e2e-tests.json + echo "E2E: ${E2E_MSG} ($(grep -c '^=== E2E SUITE' e2e-output.txt) suite runs)" | tee -a "$GITHUB_STEP_SUMMARY" + + - name: Upload E2E badges uses: actions/upload-artifact@v6 with: name: e2e-badges @@ -647,25 +1113,29 @@ jobs: include-hidden-files: true # ─────────────────────────────────────────────────────────────── - # 3. Build & Publish Docker Image + # 3a. Two-arch image build — beside the tests, never publishes # ─────────────────────────────────────────────────────────────── - # The five GHCR steps below publish on a push to master (the :edge image) and - # on a tag ref. The tag half is not decoration: release-fast-path.yml re-tags - # :edge to :vX.Y.Z when the :edge revision label matches the tagged commit, - # and dispatches THIS workflow when it does not. That fallback previously - # produced no image at all, because a workflow_dispatch is not a push and - # every publishing step was gated on github.event_name == 'push'. It built - # locally, reported success, and pushed nothing. + # Native SQLite support uses cgo, so the container build needs a zig + # cross-toolchain and static musl linking. Builds both architectures and + # actually runs the arm64 binaries under QEMU, on every event: for a PR it is + # the only image check (the GHCR push below is push/tag-only, so without it + # the first signal would arrive on master); for a push it fills the layer + # cache that the push then reuses. # - # That went unnoticed until v3.10.0, where the tagged commit was - # documentation-only. Documentation-only commits skip this workflow (see the - # paths-ignore above), so no :edge image was ever built for it, the label - # comparison failed, and the fallback ran for the first time. - build-and-publish: - name: "🏗️ Build & Publish Docker Image" - needs: [e2e-test, changes] - runs-on: ubuntu-latest - if: needs.changes.outputs.code == 'true' + # It needs only `changes`, so its minutes overlap the test jobs instead of + # following them. It cannot publish anything, so running it before the tests + # pass is safe. The build metadata is computed here once and handed to + # build-and-publish: BUILD_TIME is a build arg of the Go layers, so a second + # timestamp would miss the cache and rebuild everything. + image-check: + name: "🧱 Two-arch image build" + needs: [changes] + runs-on: ubuntu-24.04 + if: needs.changes.outputs.code == 'true' && !(startsWith(github.ref, 'refs/tags/v') && inputs.images_published) + outputs: + build_time: ${{ steps.meta.outputs.build_time }} + git_commit: ${{ steps.meta.outputs.git_commit }} + app_version: ${{ steps.meta.outputs.app_version }} steps: - name: Checkout code uses: actions/checkout@v5 @@ -685,13 +1155,138 @@ jobs: echo "app_version=$APP_VERSION" >> "$GITHUB_OUTPUT" echo "Build: version=$APP_VERSION commit=$GIT_COMMIT time=$BUILD_TIME" - - name: Build Go Docker image (local staging) + # The staging compose file used to be built here as well. That image was + # never used: the runner is discarded, and the deploy job pulls :edge from + # GHCR on its own runner. It rebuilt the same Dockerfile as the step + # below for two minutes. Validating the compose file is what is left. + - name: Validate staging compose file run: | - GIT_COMMIT="${{ steps.meta.outputs.git_commit }}" \ - APP_VERSION="${{ steps.meta.outputs.app_version }}" \ - BUILD_TIME="${{ steps.meta.outputs.build_time }}" \ - docker compose -f "$STAGING_COMPOSE_FILE" -p corescope-staging build "$STAGING_SERVICE" - echo "Built Go staging image ✅" + for key in POSTGRES_ADMIN_PASSWORD CORESCOPE_OWNER_PASSWORD CORESCOPE_READER_PASSWORD CORESCOPE_WRITER_PASSWORD CORESCOPE_ACCOUNTS_PASSWORD CORESCOPE_CHANNELS_PASSWORD; do + export "$key=$(openssl rand -hex 32)" + done + docker compose -f "$STAGING_COMPOSE_FILE" -p corescope-staging config --quiet + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Set up QEMU (arm64 runtime stage + arm64 smoke) + uses: docker/setup-qemu-action@v3 + + - name: Two-arch build (no publish) + uses: docker/build-push-action@v6 + with: + context: . + push: false + platforms: linux/amd64,linux/arm64 + tags: corescope:check-${{ github.run_id }} + build-args: | + APP_VERSION=${{ steps.meta.outputs.app_version }} + GIT_COMMIT=${{ steps.meta.outputs.git_commit }} + BUILD_TIME=${{ steps.meta.outputs.build_time }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Load amd64 image for packaged Compose smoke (no publish) + uses: docker/build-push-action@v6 + with: + context: . + push: false + load: true + platforms: linux/amd64 + tags: corescope-pg-smoke:${{ github.sha }} + build-args: | + APP_VERSION=${{ steps.meta.outputs.app_version }} + GIT_COMMIT=${{ steps.meta.outputs.git_commit }} + BUILD_TIME=${{ steps.meta.outputs.build_time }} + cache-from: type=gha + + - name: Smoke the arm64 binaries under QEMU + env: + APP_VERSION: ${{ steps.meta.outputs.app_version }} + GIT_COMMIT: ${{ steps.meta.outputs.git_commit }} + BUILD_TIME: ${{ steps.meta.outputs.build_time }} + run: | + set -e + # Export the arm64 binaries so the runner's own `file` can confirm + # they are static, then execute one under QEMU. Cheap: the build above + # already populated the layer cache. + docker buildx build . --platform linux/arm64 \ + --build-arg APP_VERSION="$APP_VERSION" \ + --build-arg GIT_COMMIT="$GIT_COMMIT" \ + --build-arg BUILD_TIME="$BUILD_TIME" --cache-from type=gha \ + --output type=local,dest=./arm64-image + for b in corescope-server corescope-ingestor corescope-decrypt corescope-migrate; do + file "./arm64-image/app/$b" | tee /dev/stderr | grep -q 'statically linked' \ + || { echo "::error::$b is not statically linked"; exit 1; } + file "./arm64-image/app/$b" | grep -q 'ARM aarch64' \ + || { echo "::error::$b is not an arm64 binary"; exit 1; } + done + docker run --rm --platform linux/arm64 --network none \ + -v "$PWD/arm64-image/app:/bin-under-test:ro" alpine:3.24 \ + /bin-under-test/corescope-decrypt --version + rm -rf ./arm64-image + echo "All four arm64 binaries are static; decrypt --version ran under QEMU." + + - name: Exercise packaged PostgreSQL Compose startup + timeout-minutes: 10 + run: | + timeout --kill-after=150s 10m bash scripts/test-postgres-compose.sh "corescope-pg-smoke:$GITHUB_SHA" sqlite + timeout --kill-after=150s 10m bash scripts/test-postgres-compose.sh "corescope-pg-smoke:$GITHUB_SHA" postgres + + # ─────────────────────────────────────────────────────────────── + # 3b. Publish Docker Image — only after every test job passed + # ─────────────────────────────────────────────────────────────── + # The GHCR steps below publish on a push to master (the :edge image) and + # on a tag ref. The tag half is not decoration: release-fast-path.yml re-tags + # :edge to :vX.Y.Z when the :edge revision label matches the tagged commit, + # and dispatches THIS workflow when it does not. That fallback previously + # produced no image at all, because a workflow_dispatch is not a push and + # every publishing step was gated on github.event_name == 'push'. It built + # locally, reported success, and pushed nothing. + # + # That went unnoticed until v3.10.0, where the tagged commit was + # documentation-only. At that time a trigger-level paths-ignore skipped this + # workflow for such commits, so no :edge image was ever built for it, the label + # comparison failed, and the fallback ran for the first time. That filter is + # gone: the `changes` job now forces code=true for anything that is not a pull + # request, so every master commit gets an image and a documentation commit is + # safe to tag. See the comment on the `on:` block. + # + # On a pull request only the first step runs; the job stays so that its + # check still means "tests and image build all passed". + # + # !cancelled() plus the first step make that true for failures too: without + # them a failed needed job would SKIP this job, and branch protection counts + # a skipped required check as passing. race-test is skipped when no ingestor + # Go file changed, so for it skipped is accepted; a master push always runs it. + build-and-publish: + permissions: + contents: read + packages: write + name: "🏗️ Build & Publish Docker Image" + needs: [go-test, race-test, e2e-test, image-check, changes] + runs-on: ubuntu-24.04 + if: ${{ !cancelled() && needs.changes.outputs.code == 'true' && !(startsWith(github.ref, 'refs/tags/v') && inputs.images_published) && !inputs.postgres_benchmark }} + steps: + - name: Require every test job and the image check to have passed + run: | + fail=0 + require() { + echo "$1: $2" + case " $3 " in + *" $2 "*) ;; + *) echo "::error::$1 result is $2"; fail=1 ;; + esac + } + require go-test "${{ needs['go-test'].result }}" "success" + require race-test "${{ needs['race-test'].result }}" "success skipped" + require e2e-test "${{ needs['e2e-test'].result }}" "success" + require image-check "${{ needs['image-check'].result }}" "success" + exit $fail + + - name: Checkout code + if: ${{ github.event_name == 'push' || startsWith(github.ref, 'refs/tags/v') }} + uses: actions/checkout@v5 - name: Set up Docker Buildx if: ${{ github.event_name == 'push' || startsWith(github.ref, 'refs/tags/v') }} @@ -722,6 +1317,7 @@ jobs: type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v') }} type=edge,branch=master + # Same build args as image-check, so every layer is a cache hit. - name: Build and push to GHCR if: ${{ github.event_name == 'push' || startsWith(github.ref, 'refs/tags/v') }} uses: docker/build-push-action@v6 @@ -732,11 +1328,11 @@ jobs: tags: ${{ steps.docker-meta.outputs.tags }} labels: ${{ steps.docker-meta.outputs.labels }} build-args: | - APP_VERSION=${{ steps.meta.outputs.app_version }} - GIT_COMMIT=${{ steps.meta.outputs.git_commit }} - BUILD_TIME=${{ steps.meta.outputs.build_time }} + APP_VERSION=${{ needs.image-check.outputs.app_version }} + GIT_COMMIT=${{ needs.image-check.outputs.git_commit }} + BUILD_TIME=${{ needs.image-check.outputs.build_time }} + # No cache-to: image-check exported these same layers minutes ago. cache-from: type=gha - cache-to: type=gha,mode=max # ─────────────────────────────────────────────────────────────── # 4. Release Artifacts (tags only) @@ -744,35 +1340,96 @@ jobs: release-artifacts: name: "📦 Release Artifacts" needs: [go-test, changes] - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 permissions: contents: write - if: startsWith(github.ref, 'refs/tags/v') && needs.changes.outputs.code == 'true' + if: (startsWith(github.ref, 'refs/tags/v') && needs.changes.outputs.code == 'true') && !inputs.postgres_benchmark steps: - name: Checkout code uses: actions/checkout@v5 - - name: Set up Go 1.22 + - name: Set up Go 1.27.2 uses: actions/setup-go@v6 with: - go-version: '1.22' + go-version: '1.27.2' + cache-dependency-path: | + cmd/decrypt/go.sum + cmd/migrate/go.sum + internal/**/go.sum + + # Both release tools support native SQLite and need a cross C compiler. + - name: Set up Zig + uses: mlugg/setup-zig@v2 + with: + version: 0.16.0 - - name: Build corescope-decrypt (static, linux/amd64) + - name: Build static release tools for both architectures run: | - cd cmd/decrypt - CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-s -w -X main.version=${{ github.ref_name }}" -o ../../corescope-decrypt-linux-amd64 . + set -euo pipefail + for arch in amd64 arm64; do + case "$arch" in amd64) target=x86_64-linux-musl ;; arm64) target=aarch64-linux-musl ;; esac + ( + cd cmd/decrypt + CGO_ENABLED=1 GOOS=linux GOARCH="$arch" CC="zig cc -target $target" \ + go build -trimpath -tags netgo,osusergo,sqlite_omit_load_extension \ + -ldflags="-s -w -extldflags '-static -Wl,-s' -X main.version=$GITHUB_REF_NAME" -o "../../corescope-decrypt-linux-$arch" . + ) + ( + cd cmd/migrate + CGO_ENABLED=1 GOOS=linux GOARCH="$arch" CC="zig cc -target $target" \ + go build -trimpath -tags netgo,osusergo,sqlite_omit_load_extension \ + -ldflags="-s -w -extldflags '-static -Wl,-s'" -o "../../corescope-migrate-linux-$arch" . + ) + done - - name: Build corescope-decrypt (static, linux/arm64) + - name: Verify release binaries are static and runnable run: | - cd cmd/decrypt - CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -ldflags="-s -w -X main.version=${{ github.ref_name }}" -o ../../corescope-decrypt-linux-arm64 . + set -e + for f in corescope-{decrypt,migrate}-linux-{amd64,arm64}; do + file "$f" | grep -q 'statically linked' || { echo "::error::$f is not statically linked"; exit 1; } + done + # amd64 runs natively on the runner; arm64 is only checked structurally + # here (the container image job exercises arm64 under QEMU). + ./corescope-decrypt-linux-amd64 --version + ./corescope-migrate-linux-amd64 -help + + - name: Locate the release notes for this tag + id: notes + run: | + set -euo pipefail + # docs/release-notes/.md is the convention: v3.10.1.md, v3.12.0.md. + F="docs/release-notes/${GITHUB_REF_NAME}.md" + if [ -f "$F" ]; then + echo "path=$F" >> "$GITHUB_OUTPUT" + echo "found $F ($(wc -c < "$F") bytes)" + else + # Not an error. A release without a notes file still gets the + # generated pull request list below, which is more than the empty + # body v3.10.1, v3.11.0 and v3.12.0 were published with. + echo "path=" >> "$GITHUB_OUTPUT" + echo "::warning::no $F, publishing with generated notes only" + fi - name: Upload release assets + # Standard releases upload both assets to a draft before publishing. + # Keep one writer: published releases are immutable in this repository. + # + # body_path and generate_release_notes together: our own notes first, + # then GitHub's generated pull request list under them. Without either, + # this step publishes an empty release body, which is what happened to + # every release from v3.10.1 to v3.12.0 (v3.12.0's body was filled in by + # hand afterwards). The hand-written file is the substance; the generated + # list is the "what changed" index that a reader expects to find. uses: softprops/action-gh-release@v2 with: + fail_on_unmatched_files: true + body_path: ${{ steps.notes.outputs.path }} + generate_release_notes: true files: | corescope-decrypt-linux-amd64 corescope-decrypt-linux-arm64 + corescope-migrate-linux-amd64 + corescope-migrate-linux-arm64 # ─────────────────────────────────────────────────────────────── # 4b. Deploy Staging (master only) @@ -797,7 +1454,8 @@ jobs: # variable is set while the runner is still missing the job fails in ten # minutes rather than blocking the branch again. if: | - vars.ENABLE_STAGING_DEPLOY == 'true' + !inputs.postgres_benchmark + && vars.ENABLE_STAGING_DEPLOY == 'true' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref == 'refs/heads/master' needs: [build-and-publish] @@ -888,14 +1546,14 @@ jobs: # ─────────────────────────────────────────────────────────────── publish: name: "📝 Publish Badges & Summary" - if: github.event_name == 'push' + if: github.event_name == 'push' && !inputs.postgres_benchmark # Was needs: [deploy]. The badges are built from the go-test and e2e-test # artifacts and never needed the deploy step; depending on it meant the # coverage badges stopped updating whenever the self-hosted runner was # unavailable. build-and-publish already transitively requires both test # jobs, so ordering is unchanged. needs: [build-and-publish] - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Checkout code uses: actions/checkout@v5 diff --git a/.github/workflows/postgres-benchmark.yml b/.github/workflows/postgres-benchmark.yml new file mode 100644 index 000000000..1ccb47eb5 --- /dev/null +++ b/.github/workflows/postgres-benchmark.yml @@ -0,0 +1,136 @@ +name: PostgreSQL paired benchmark + +on: + workflow_dispatch: + inputs: + candidate_sha: + description: Exact 40-character candidate commit + type: string + required: true + workflow_call: + inputs: + candidate_sha: + type: string + required: true + +permissions: + contents: read + +concurrency: + group: postgres-benchmark-${{ github.run_id }} + cancel-in-progress: false + +jobs: + benchmark: + name: Five supplemental pairs (50 events/s, 128K transmissions / 2.048M observations) + runs-on: ubuntu-24.04 + timeout-minutes: 240 + env: + CANDIDATE_SHA: ${{ inputs.candidate_sha }} + CGO_ENABLED: '1' + GOMAXPROCS: '3' + steps: + - name: Validate exact candidate and runner capacity + run: | + set -euo pipefail + [[ "$CANDIDATE_SHA" =~ ^[0-9a-f]{40}$ ]] || { echo 'candidate_sha must be a full lowercase commit SHA' >&2; exit 1; } + [[ $(nproc) -ge 4 ]] || { echo 'At least four CPUs are required' >&2; exit 1; } + awk '/MemTotal:/ {exit !($2 >= 8*1024*1024)}' /proc/meminfo || { echo 'At least 8 GiB RAM is required' >&2; exit 1; } + test -f /sys/fs/cgroup/cgroup.controllers + echo "BENCH_OUTPUT=$RUNNER_TEMP/corescope-bench-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" >> "$GITHUB_ENV" + echo "SMOKE_OUTPUT=$RUNNER_TEMP/corescope-bench-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT-smoke" >> "$GITHUB_ENV" + + - name: Checkout the exact candidate + uses: actions/checkout@v5 + with: + repository: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name || github.repository }} + ref: ${{ inputs.candidate_sha }} + fetch-depth: 0 + persist-credentials: false + + - name: Verify candidate and workflow snapshot + env: + EXPECTED_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + WORKFLOW_SHA: ${{ github.workflow_sha }} + run: | + set -euo pipefail + test "$(git rev-parse HEAD)" = "$CANDIDATE_SHA" + test "$EXPECTED_SHA" = "$CANDIDATE_SHA" || { echo 'Candidate must match the PR head or dispatched workflow commit' >&2; exit 1; } + [[ "$WORKFLOW_SHA" =~ ^[0-9a-f]{40}$ ]] || { echo 'A full workflow commit SHA is required' >&2; exit 1; } + # PR workflows run from a merge ref, while the experiment measures the + # exact head. Refuse a merge that changes the executed workflow/harness. + git fetch --no-tags --depth=1 "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY.git" "$WORKFLOW_SHA" + git diff --quiet "$WORKFLOW_SHA" "$CANDIDATE_SHA" -- \ + .github/workflows/deploy.yml .github/workflows/postgres-benchmark.yml \ + scripts/postgres-benchmark scripts/install-postgres-ci.sh \ + || { echo 'Workflow or benchmark harness differs from candidate; update the candidate before rerunning' >&2; exit 1; } + git cat-file -e 3e21b179ecccae005aa5f6f45bc095353c94b343^{commit} + + - name: Test benchmark controls + run: python3 -m unittest discover -s scripts/postgres-benchmark -p 'test_*.py' + + - name: Use the same compiler for both revisions + uses: actions/setup-go@v6 + with: + go-version: '1.27.2' + cache-dependency-path: '**/go.sum' + + - name: Install exact PostgreSQL 18.6 binaries + run: bash scripts/install-postgres-ci.sh server + + - name: Provision owned cgroup and run supplemental comparison + run: | + set -euo pipefail + delegated="/sys/fs/cgroup/corescope-benchmark-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" + test ! -e "$delegated" + sudo mkdir "$delegated" + for controller in cpu memory io; do + grep -qw "$controller" "$delegated/cgroup.controllers" || { echo "Required cgroup controller unavailable: $controller" >&2; exit 1; } + done + printf '+cpu +memory +io\n' | sudo tee "$delegated/cgroup.subtree_control" >/dev/null + sudo mkdir "$delegated/coordinator" + sudo chown "$(id -u):$(id -g)" "$delegated" "$delegated/cgroup.procs" "$delegated/cgroup.subtree_control" "$delegated/coordinator" "$delegated/coordinator/cgroup.procs" + # The coordinator/generator live in a sibling of the measured app+DB + # group. The common ancestor is delegated so unprivileged child moves + # work; the runner validates the 3 CPU / 6 GiB measured envelope. + printf '%s\n' "$$" | sudo tee "$delegated/coordinator/cgroup.procs" >/dev/null + # Prove the controller and Linux prerequisites before preparing corpus B. + python3 scripts/postgres-benchmark/run.py --repo . \ + --candidate-sha "$CANDIDATE_SHA" --corpus S --pairs 1 --profile smoke \ + --ingest-rate 50 --http-rate 20 \ + --warmup 2 --seconds 10 --postgres-version 18.6 --go-version 1.27.2 \ + --postgres-bin /usr/lib/postgresql/18/bin \ + --output "$SMOKE_OUTPUT" --cgroup "$delegated" + python3 scripts/postgres-benchmark/run.py --repo . \ + --candidate-sha "$CANDIDATE_SHA" --corpus B --pairs 5 --profile primary \ + --ingest-rate 50 --http-rate 20 \ + --postgres-version 18.6 --go-version 1.27.2 \ + --postgres-bin /usr/lib/postgresql/18/bin \ + --output "$BENCH_OUTPUT" --cgroup "$delegated" + + - name: Upload only sanitized public evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: postgres-benchmark-${{ inputs.candidate_sha }} + path: | + ${{ env.SMOKE_OUTPUT }}/public/ + ${{ env.BENCH_OUTPUT }}/public/ + if-no-files-found: error + retention-days: 30 + + - name: Remove only this run's cgroup + if: always() + run: | + set -euo pipefail + delegated="/sys/fs/cgroup/corescope-benchmark-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" + if [ -d "$delegated" ]; then + test "$(realpath "$delegated")" = "$delegated" + if [ -f "$delegated/cgroup.kill" ]; then echo 1 | sudo tee "$delegated/cgroup.kill" >/dev/null; fi + for attempt in $(seq 1 10); do + sudo find "$delegated" -depth -type d -exec rmdir {} \; 2>/dev/null || true + test ! -d "$delegated" && break + sleep 1 + done + test ! -d "$delegated" + fi diff --git a/.github/workflows/release-fast-path.yml b/.github/workflows/release-fast-path.yml index b7f1b47fa..029804112 100644 --- a/.github/workflows/release-fast-path.yml +++ b/.github/workflows/release-fast-path.yml @@ -1,30 +1,41 @@ name: Release Fast-Path # Issue #1677: re-tag :edge as :vX.Y.Z when the tag SHA matches :edge's -# org.opencontainers.image.revision label. Skips ~30 min of Go test + -# Playwright + Docker rebuild because the bytes are identical — only the -# manifest name changes. Falls back to deploy.yml when SHAs differ so -# tags on older commits still go through full validation. +# org.opencontainers.image.revision label. Skips Playwright + Docker rebuild +# because the application bytes are identical. Both paths dispatch deploy.yml +# for Go validation and release binaries; mismatched SHAs also run the full +# image pipeline so tags on older commits still go through full validation. # # This workflow is the SOLE consumer of push.tags. deploy.yml's tag # trigger has been removed to prevent double-fire. +# +# It can also be dispatched by hand with an existing tag, to republish the +# images for a release whose tags are wrong (v3.11.0 shipped an amd64-only +# manifest, see below). A manual run never dispatches deploy.yml: the release +# for that tag already exists and this repository has immutable releases. on: push: tags: ['v[0-9]+.[0-9]+.[0-9]+'] + workflow_dispatch: + inputs: + tag: + description: 'Existing vX.Y.Z tag to republish container tags for' + required: true + type: string permissions: contents: read packages: write - actions: write # issue #1702: required so the fallback `gh workflow run deploy.yml` dispatch is allowed + actions: write # issue #1702: required for the `gh workflow run deploy.yml` dispatch concurrency: - group: release-fast-path-${{ github.ref }} + group: release-fast-path-${{ inputs.tag || github.ref }} cancel-in-progress: false jobs: retag-or-fallback: - name: "🏷️ Re-tag :edge → :vX.Y.Z (fast) or dispatch deploy.yml (fallback)" + name: "🏷️ Re-tag :edge and dispatch release artifacts or full fallback" runs-on: ubuntu-latest steps: - name: Log in to GHCR @@ -39,22 +50,41 @@ jobs: - name: Parse semver from tag id: semver + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | set -euo pipefail - TAG="${GITHUB_REF#refs/tags/}" - # Expect vMAJOR.MINOR.PATCH (workflow trigger already enforces this). + if [[ -n "${{ inputs.tag || '' }}" ]]; then + TAG="${{ inputs.tag || '' }}" + else + TAG="${GITHUB_REF#refs/tags/}" + fi + # Expect vMAJOR.MINOR.PATCH (the push trigger already enforces this; + # a dispatched tag has to be checked here). if [[ ! "$TAG" =~ ^v([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then echo "Tag $TAG does not match vMAJOR.MINOR.PATCH" >&2 exit 1 fi MAJOR="${BASH_REMATCH[1]}" MINOR="${BASH_REMATCH[2]}" + # On a dispatched run github.sha is the ref the workflow file came + # from, not the tagged commit, so resolve the tag itself. + if [[ -n "${{ inputs.tag || '' }}" ]]; then + TARGET_SHA="$(gh api "repos/${{ github.repository }}/git/refs/tags/$TAG" --jq '.object.sha')" + OBJ_TYPE="$(gh api "repos/${{ github.repository }}/git/refs/tags/$TAG" --jq '.object.type')" + if [[ "$OBJ_TYPE" == "tag" ]]; then + TARGET_SHA="$(gh api "repos/${{ github.repository }}/git/tags/$TARGET_SHA" --jq '.object.sha')" + fi + else + TARGET_SHA="${{ github.sha }}" + fi { echo "tag=$TAG" echo "vMajor=v$MAJOR" echo "vMajorMinor=v$MAJOR.$MINOR" + echo "targetSha=$TARGET_SHA" } >> "$GITHUB_OUTPUT" - echo "Parsed: $TAG → v$MAJOR / v$MAJOR.$MINOR / $TAG" + echo "Parsed: $TAG → v$MAJOR / v$MAJOR.$MINOR / $TAG at $TARGET_SHA" - name: Inspect :edge revision label id: edge @@ -76,17 +106,22 @@ jobs: echo "edge_revision=$REV" >> "$GITHUB_OUTPUT" echo "no_edge=false" >> "$GITHUB_OUTPUT" echo ":edge org.opencontainers.image.revision = $REV" - echo "tag SHA (github.sha) = ${{ github.sha }}" + echo "tagged commit = ${{ steps.semver.outputs.targetSha }}" # ─────────── FAST PATH: SHAs match, metadata-only retag ─────────── # Issue #1807: a plain `crane tag` keeps the ldflags-baked # Version="edge" inside the binary, so /api/stats reports "edge" on # tagged releases. Instead of a full rebuild we append a single-file # layer (/app/.image-version containing the tag) and set the OCI - # version label, then point the remaining tags at that mutated image. - # The server reads .image-version at startup (see resolveVersion()). + # version label. The server reads .image-version at startup (see + # resolveVersion()). + # + # `crane mutate` works on ONE image, not on an index: pointed at a + # multi-arch tag it silently returns the default platform. v3.11.0 + # shipped amd64-only that way. So mutate each platform child of the + # :edge index and reassemble an index from the results. - name: Re-tag :edge → :vX.Y.Z + :vX.Y + :vX + :latest (fast path) - if: steps.edge.outputs.no_edge == 'false' && steps.edge.outputs.edge_revision == github.sha + if: steps.edge.outputs.no_edge == 'false' && steps.edge.outputs.edge_revision == steps.semver.outputs.targetSha run: | set -euo pipefail IMAGE="ghcr.io/kpa-clawbot/corescope" @@ -100,11 +135,47 @@ jobs: printf '%s' "$TAG" > "${LAYER_DIR}/app/.image-version" tar --owner=0 --group=0 --mtime='UTC 2020-01-01' \ -C "$LAYER_DIR" -cf /tmp/image-version-layer.tar app - echo " crane mutate $SRC (+.image-version layer, +version label) → ${IMAGE}:${TAG}" - crane mutate "$SRC" \ - --append /tmp/image-version-layer.tar \ - --label "org.opencontainers.image.version=${TAG}" \ - --tag "${IMAGE}:${TAG}" + + crane manifest "$SRC" > /tmp/edge-manifest.json + MEDIA="$(jq -r '.mediaType // ""' /tmp/edge-manifest.json)" + + if [[ "$MEDIA" == *".index."* || "$MEDIA" == *".manifest.list."* ]]; then + # Attestation manifests carry platform architecture "unknown" and + # are rebuilt by buildx, not by us; carrying them over would point + # them at the pre-mutation digests, so the release index holds the + # runnable platforms only. + mapfile -t PLATFORMS < <(jq -r ' + .manifests[] + | select((.platform.architecture // "unknown") != "unknown") + | "\(.platform.os)/\(.platform.architecture)\(if .platform.variant then "/" + .platform.variant else "" end)\t\(.digest)" + ' /tmp/edge-manifest.json) + if [[ "${#PLATFORMS[@]}" -eq 0 ]]; then + echo "No runnable platforms in $SRC" >&2 + exit 1 + fi + MANIFEST_ARGS=() + for entry in "${PLATFORMS[@]}"; do + PLATFORM="${entry%%$'\t'*}" + DIGEST="${entry##*$'\t'}" + SAFE="${PLATFORM//\//-}" + TMP_TAG="${IMAGE}:tmp-${TAG}-${SAFE}" + echo " crane mutate ${SRC}@${DIGEST} ($PLATFORM) → $TMP_TAG" + crane mutate "${IMAGE}@${DIGEST}" \ + --append /tmp/image-version-layer.tar \ + --label "org.opencontainers.image.version=${TAG}" \ + --tag "$TMP_TAG" + MANIFEST_ARGS+=(-m "$TMP_TAG") + done + echo " crane index append ${MANIFEST_ARGS[*]} → ${IMAGE}:${TAG}" + crane index append "${MANIFEST_ARGS[@]}" -t "${IMAGE}:${TAG}" + else + # Single-platform :edge (nothing to reassemble). + echo " crane mutate $SRC (single platform) → ${IMAGE}:${TAG}" + crane mutate "$SRC" \ + --append /tmp/image-version-layer.tar \ + --label "org.opencontainers.image.version=${TAG}" \ + --tag "${IMAGE}:${TAG}" + fi for NEW_TAG in \ "${{ steps.semver.outputs.vMajorMinor }}" \ @@ -115,17 +186,77 @@ jobs: done echo "Fast-path complete — release tags point at :edge plus a one-file version layer." - # ─────────── FALLBACK: SHAs differ, run the full pipeline ─────────── - - name: Dispatch full deploy.yml pipeline (fallback) - if: steps.edge.outputs.no_edge == 'true' || steps.edge.outputs.edge_revision != github.sha + # A release tag that resolves to one platform is worse than a slow + # release: `docker pull` on the other architecture fails outright. + - name: Verify the published tags carry every :edge platform + if: steps.edge.outputs.no_edge == 'false' && steps.edge.outputs.edge_revision == steps.semver.outputs.targetSha + run: | + set -euo pipefail + IMAGE="ghcr.io/kpa-clawbot/corescope" + TAG="${{ steps.semver.outputs.tag }}" + platforms_of() { + crane manifest "$1" | jq -r ' + if (.mediaType // "") | test("index|manifest.list") then + [ .manifests[] + | select((.platform.architecture // "unknown") != "unknown") + | "\(.platform.os)/\(.platform.architecture)\(if .platform.variant then "/" + .platform.variant else "" end)" ] + | sort | join(",") + else + "single" + end' + } + WANT="$(platforms_of "${IMAGE}:edge")" + echo " :edge platforms = $WANT" + for CHECK in \ + "$TAG" \ + "${{ steps.semver.outputs.vMajorMinor }}" \ + "${{ steps.semver.outputs.vMajor }}" \ + "latest"; do + GOT="$(platforms_of "${IMAGE}:${CHECK}")" + echo " :${CHECK} platforms = $GOT" + if [[ "$GOT" != "$WANT" ]]; then + echo "::error::${IMAGE}:${CHECK} has platforms [$GOT], expected [$WANT]" + exit 1 + fi + done + echo "All release tags carry the same platforms as :edge." + + - name: Remove the per-platform scratch tags + if: steps.edge.outputs.no_edge == 'false' && steps.edge.outputs.edge_revision == steps.semver.outputs.targetSha + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -uo pipefail + IMAGE="ghcr.io/kpa-clawbot/corescope" + TAG="${{ steps.semver.outputs.tag }}" + # Best effort: the index already references the manifests by digest, + # so deleting the scratch tags does not affect it. Leaving them is + # only untidy, hence no failure here. This step is skipped when an + # earlier one failed, which keeps the scratch tags around to inspect. + for T in $(crane ls "$IMAGE" | grep "^tmp-${TAG}-" || true); do + echo " crane delete ${IMAGE}:${T}" + crane delete "${IMAGE}:${T}" || echo " (could not delete ${T}, leaving it)" + done + + # Both image routes need the single release-artifacts writer in + # deploy.yml. A manual republish skips it: the release exists already and + # releases here are immutable, so action-gh-release would fail. + - name: Dispatch release artifacts or full deploy.yml pipeline + if: ${{ !inputs.tag }} env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | set -euo pipefail - echo "SHA mismatch (or no :edge) — falling back to full pipeline" - echo " :edge revision = '${{ steps.edge.outputs.edge_revision }}'" - echo " tag SHA = '${{ github.sha }}'" + DISPATCH_ARGS=() + if [[ "${{ steps.edge.outputs.no_edge == 'false' && steps.edge.outputs.edge_revision == steps.semver.outputs.targetSha }}" == 'true' ]]; then + echo "Images published — dispatching Go validation and release artifacts" + DISPATCH_ARGS+=(--field images_published=true) + else + # No new inputs: an older tag may have an older deploy.yml schema. + echo "SHA mismatch (or no :edge) — falling back to full pipeline" + fi gh workflow run deploy.yml \ --repo "${{ github.repository }}" \ - --ref "${{ github.ref }}" + --ref "${{ github.ref }}" \ + "${DISPATCH_ARGS[@]}" echo "Dispatched deploy.yml against ${{ github.ref }}" diff --git a/.gitignore b/.gitignore index 548c25d66..6bc2d49ed 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ node_modules/ data/ +postgres-data/ +*.dump *.bak *.db *.db-journal @@ -10,6 +12,10 @@ config-lincomatic.json theme.json firmware/ coverage/ +dist/ +# internal/prunequeue writes these next to the database; the server tests leave +# them in cmd/server/. Runtime queue state, never repo content. +prune-requests/ public-instrumented/ .nyc_output/ .setup-state diff --git a/.squad/agents/bishop/charter.md b/.squad/agents/bishop/charter.md index 09b0a5d82..5ce7d6fe5 100644 --- a/.squad/agents/bishop/charter.md +++ b/.squad/agents/bishop/charter.md @@ -10,16 +10,15 @@ Unit tests, Playwright E2E, coverage gates, and quality assurance for CoreScope. ## Responsibilities -- Unit tests: test-packet-filter.js, test-aging.js, test-decoder.js, test-decoder-spec.js, test-server-helpers.js, test-server-routes.js, test-packet-store.js, test-db.js, test-frontend-helpers.js, test-regional-filter.js, test-regional-integration.js, test-live-dedup.js -- Playwright E2E: test-e2e-playwright.js (8 browser tests, default localhost:3000) -- E2E tools: tools/e2e-test.js, tools/frontend-test.js +- Unit tests: `tests/unit/`, all listed in `test-all.sh` (e.g. test-packet-filter.js, test-aging.js, test-frontend-helpers.js); Go tests under `cmd/` +- Playwright E2E: `tests/e2e/`, classified in `scripts/non-unit-tests.json` (e.g. test-e2e-playwright.js, default localhost:3000) - Coverage: Backend 85%+ (c8), Frontend 42%+ (Istanbul + nyc). Both only go up. - Review authority: May approve or reject work from Hicks and Newt based on test results ## Boundaries - Test the REAL code — import actual modules, don't copy-paste functions into test files -- Use vm.createContext for frontend helpers (see test-frontend-helpers.js pattern) +- Use vm.createContext for frontend helpers (see tests/unit/test-frontend-helpers.js pattern) - Playwright tests default to localhost:3000 — NEVER run against prod - Every bug fix gets a regression test - Every new feature must add tests — test count only goes up @@ -37,10 +36,9 @@ Unit tests, Playwright E2E, coverage gates, and quality assurance for CoreScope. npm test # all backend tests + coverage summary npm run test:unit # fast: unit tests only npm run test:coverage # all tests + HTML coverage report -node test-packet-filter.js # filter engine -node test-decoder.js # packet decoder -node test-server-routes.js # API routes via supertest -node test-e2e-playwright.js # 8 Playwright browser tests +sh test-all.sh # every suite in tests/unit +node tests/unit/test-packet-filter.js # filter engine +node tests/e2e/test-e2e-playwright.js # Playwright browser tests ``` ## Model diff --git a/.squad/agents/newt/charter.md b/.squad/agents/newt/charter.md index 223f424b2..fc87a6f5a 100644 --- a/.squad/agents/newt/charter.md +++ b/.squad/agents/newt/charter.md @@ -24,7 +24,7 @@ Vanilla JS UI, Leaflet maps, live visualization, theming, and all public/ module - Use shared helpers from roles.js (ROLE_COLORS, TYPE_COLORS, getNodeStatus, getHealthThresholds) - Prefer `n.last_heard || n.last_seen` for display and status - No per-packet API calls from frontend — fetch bulk, filter client-side -- Run `node test-packet-filter.js` and `node test-frontend-helpers.js` after filter/helper changes +- Run `node tests/unit/test-packet-filter.js` and `node tests/unit/test-frontend-helpers.js` after filter/helper changes - Always bump cache busters in the SAME commit as code changes ## Key Files diff --git a/.squad/decisions/decisions.md b/.squad/decisions/decisions.md index d5aef0851..37c8bb32f 100644 --- a/.squad/decisions/decisions.md +++ b/.squad/decisions/decisions.md @@ -211,7 +211,9 @@ Reconciled `manage.sh` and `docker-compose.yml` Docker volume names: Standalone Go MQTT ingestor service. Separate process from Node.js web server that handles MQTT packet ingestion + writes to shared SQLite DB. **Architecture:** -- Single binary, no CGO (uses `modernc.org/sqlite` pure Go) +- Single statically-linked binary. Was CGO-free via `modernc.org/sqlite`; now cgo, + using `github.com/mattn/go-sqlite3` for read performance, cross-compiled with + `zig cc` against musl so the artifact stays a single self-contained file - Reads same `config.json` (mqttSources array) - Shares SQLite DB with Node.js (WAL mode for concurrent access) - Format 1 (raw packet) MQTT only — companion bridge stays in Node.js diff --git a/AGENTS.md b/AGENTS.md index 6fc1db4bc..340f18b1b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -13,10 +13,12 @@ Go backend + static frontend. No build step. No framework. No bundler. cmd/server/ — Go API server (REST + WebSocket broadcast + static file serving) main.go — Entry point, flags, SPA handler routes.go — All /api/* endpoints - store.go — In-memory packet store + analytics + SQLite queries + store.go — In-memory packet store + analytics + PostgreSQL queries config.go — Configuration loading decoder.go — MeshCore packet decoder -cmd/ingestor/ — Go MQTT ingestor (separate binary, writes to shared SQLite DB) +cmd/ingestor/ — Go MQTT ingestor (separate binary, writes telemetry to PostgreSQL) +cmd/migrate/ — Offline schema bootstrap and verified SQLite upgrade importer +internal/pgutil/ — PostgreSQL connections, role checks, native backup helpers public/ — Frontend (vanilla JS, one file per page) — ACTIVE, NOT DEPRECATED app.js — SPA router, shared globals, theme loading roles.js — ROLE_COLORS, TYPE_COLORS, health thresholds, shared helpers @@ -34,21 +36,34 @@ public/ — Frontend (vanilla JS, one file per page) — ACTIVE, NOT live.css — Live page styles home.css — Home page styles index.html — SPA shell, script/style tags with __BUST__ placeholder (auto-replaced at server startup) -test-fixtures/ — Real data SQLite fixture from staging (used for E2E tests) +test-fixtures/ — Legacy SQLite fixture; prepare and import into PostgreSQL for E2E scripts/ — Tooling (coverage collector, fixture capture, frontend instrumentation) ``` ### Data Flow -1. MQTT brokers → Go ingestor (`cmd/ingestor/`) ingests packets → decodes → writes to SQLite -2. Go server (`cmd/server/`) polls SQLite for new packets, broadcasts via WebSocket +1. MQTT brokers → Go ingestor (`cmd/ingestor/`) ingests packets → decodes → writes to PostgreSQL +2. Go server (`cmd/server/`) polls PostgreSQL for new packets, broadcasts via WebSocket 3. Frontend fetches via REST API (`/api/*`), filters/sorts client-side ### Read/Write Separation Invariant (#1283) -- **All DB writes live in `cmd/ingestor/`.** INSERT / UPDATE / DELETE / VACUUM / - schema migrations / retention all run in the ingestor process. -- **`cmd/server/` is read-only.** It opens SQLite with `mode=ro` and must not - acquire a write lock. Adding a write-side helper (e.g. a `cachedRW`-style - RW connection) regresses this invariant and races the ingestor → SQLITE_BUSY. +- **Runtime telemetry writes live in `cmd/ingestor/`.** INSERT / UPDATE / + DELETE and retention use a restricted writer role. PostgreSQL handles + autovacuum; offline `cmd/migrate/` owns schema installation and upgrades. +- **`cmd/server/` never writes measurement data.** Its telemetry connection + uses a SELECT-only role and read-only transactions. Do not add a write-side + connection or grant the reader schema/owner privileges. +- **Single exception: the separate accounts database (optional user + management).** When `userManagement.enabled`, the server writes accounts + through `internal/users` only. `users.Open` rejects the actual telemetry + database, including URL aliases, and `TestUsersOpenIsTheOnlyServerWritePath` + pins the one call site. Account and measurement databases stay separate. +- **The ingestor reads approved channel names, never account rows.** With + `userManagement.channelProposals.enabled`, its dedicated role may SELECT + only `approved_channels` and the readiness marker. It uses a read-only + snapshot and retains the last good keys if readiness or refresh fails. +- **Runtime roles never run DDL or mark an import ready.** Bootstrap/import + uses separate owner credentials. Tests must cover denied writes and + cross-database access with the real runtime grants. - Enforcement: `cmd/server/readonly_invariant_test.go` reflect-asserts that `PruneOldPackets`, `PruneOldMetrics`, and `RemoveStaleObservers` are NOT methods on the server's `*DB`. If you need a new write, add it to @@ -90,7 +105,21 @@ Every change must consider performance impact BEFORE implementation. This codeba No proof = no merge. ### 1. No commit without tests -Every change that touches logic MUST have tests. For Go backend: `cd cmd/server && go test ./...` and `cd cmd/ingestor && go test ./...`. For frontend: `node test-packet-filter.js && node test-aging.js && node test-frontend-helpers.js`. If you add new logic, add tests. No exceptions. +Every change that touches logic MUST have tests. For Go backend: `cd cmd/server && go test ./...` and `cd cmd/ingestor && go test ./...`, or `make test` for all Go modules. For frontend: `node tests/unit/test-packet-filter.js && node tests/unit/test-aging.js && node tests/unit/test-frontend-helpers.js`. If you add new logic, add tests. No exceptions. + +The PostgreSQL runtime uses native Go (`pgx`). SQLite is retained only for +the offline upgrade importer and legacy test fixtures. The importer still +requires cgo (`github.com/mattn/go-sqlite3`); `CGO_ENABLED=0` links a stub +that fails on the first SQLite query. Use `make build` / `make crossbuild` +for the complete release, including the importer. Cross-building that +importer requires [`zig`](https://ziglang.org/download/) or another suitable +C cross compiler. Validate a real import, not just a successful build. + +Database integration tests require a disposable PostgreSQL 18 instance and +`CORESCOPE_TEST_POSTGRES_URL` with test schema/database/role creation rights. +They fail when the database is unavailable rather than silently skipping. +Never point tests at a live instance. Runtime credentials stay in deployment +environment variables and must not appear in logs or committed fixtures. ### 2. No commit without browser validation After pushing, verify the change works in an actual browser. Use `browser profile=openclaw` against the running instance. Take a screenshot if the change is visual. If you can't validate it, say so — don't claim it works. @@ -200,7 +229,7 @@ All colors MUST use CSS variables. Never hardcode `#hex` values outside of `:roo ### Packet Filter (packet-filter.js) Standalone module. No dependencies on app globals (copies what it needs). Testable in Node.js: ```bash -node test-packet-filter.js +node tests/unit/test-packet-filter.js ``` Uses firmware-standard type names (GRP_TXT, TXT_MSG, REQ) with aliases for convenience. @@ -217,21 +246,13 @@ npm run test:full-coverage # backend + instrumented frontend coverage via Playw ### Test Files ```bash # Backend (deterministic, run before every push) -node test-packet-filter.js # filter engine -node test-aging.js # node aging system -node test-regional-filter.js # regional observer filtering -node test-decoder.js # packet decoder -node test-decoder-spec.js # spec-driven + golden fixture tests -node test-server-helpers.js # extracted server functions -node test-server-routes.js # API route tests via supertest -node test-packet-store.js # in-memory packet store -node test-db.js # SQLite operations -node test-frontend-helpers.js # frontend logic (via vm.createContext) -node tools/e2e-test.js # E2E: temp server + synthetic packets -node tools/frontend-test.js # frontend smoke: HTML, JS refs, API shapes +sh test-all.sh # every suite in tests/unit +node tests/unit/test-packet-filter.js # filter engine +node tests/unit/test-aging.js # node aging system +node tests/unit/test-frontend-helpers.js # frontend logic (via vm.createContext) # Frontend E2E (requires running server or Playwright) -node test-e2e-playwright.js # 8 Playwright browser tests (default: localhost:3000) +node tests/e2e/test-e2e-playwright.js # Playwright browser tests (default: localhost:3000) ``` ### Rules @@ -246,14 +267,14 @@ node test-e2e-playwright.js # 8 Playwright browser tests (default: localho 2. Write unit tests for the logic 3. Write/update Playwright tests if it's a UI change 4. Run `npm test` — all tests must pass -5. Run `node test-e2e-playwright.js` against a local server — E2E must pass +5. Run `node tests/e2e/test-e2e-playwright.js` against a local server — E2E must pass 6. THEN push to master ### Testing infrastructure - **Backend coverage**: c8 tracks server-side code in-process - **Frontend coverage**: Istanbul instruments `public/*.js` → Playwright exercises them → `window.__coverage__` extracted → nyc reports. Instrumented files are generated fresh each CI run, never checked in. - **CI pipeline**: backend tests + coverage → instrument frontend → start local server → Playwright E2E + coverage collection → badges update → deploy (only if all pass) -- **Playwright tests default to localhost:3000** — NEVER run against prod. CI sets `BASE_URL=http://localhost:13581`. Running locally: start your server, then `node test-e2e-playwright.js` +- **Playwright tests default to localhost:3000** — NEVER run against prod. CI sets `BASE_URL=http://localhost:13581`. Running locally: start your server, then `node tests/e2e/test-e2e-playwright.js` - **ARM machines**: Basic Playwright tests work with system chromium (`CHROMIUM_PATH=/usr/bin/chromium-browser`). Heavy coverage collection scripts may crash — use CI for those. Tests that need live mesh data can use `https://analyzer.00id.net` — all API endpoints are public, no auth required. @@ -288,7 +309,7 @@ If the same logic exists in two places, it MUST be extracted into a shared funct ### Testability - **Write functions that are easy to test.** Pure functions (input → output, no side effects) are ideal. If a function reads from the DOM, the DB, and localStorage, it's untestable without mocking everything. - **Dependency injection enables testing.** Pass the node list, the map reference, the API function as parameters. Tests can substitute fakes. -- **Test the real code, not copies.** Don't paste a function into a test file and test the copy. Import/require the actual module. If the module isn't importable (IIFE, browser-only), refactor it so it is — or use `vm.createContext` like `test-frontend-helpers.js` does. +- **Test the real code, not copies.** Don't paste a function into a test file and test the copy. Import/require the actual module. If the module isn't importable (IIFE, browser-only), refactor it so it is — or use `vm.createContext` like `tests/unit/test-frontend-helpers.js` does. - **Every bug fix gets a regression test.** If it broke once, it'll break again. The test proves it stays fixed. ### Type Safety (without TypeScript) @@ -370,7 +391,7 @@ One logical change per commit. Each commit is deployable. Each commit has its te | Subagent timed out mid-work | 4 | Give clear scope, don't try to run slow pipelines locally | ## File Naming -- Tests: `test-{feature}.js` in repo root +- Tests: `test-{feature}.js` in `tests/unit/` (standalone, listed in `test-all.sh`) or `tests/e2e/` (browser, listed in `scripts/non-unit-tests.json`) — never in repo root - No build step, no transpilation — write ES2020 for server, ES5/6 for frontend (broad browser support) ### Deep Linking diff --git a/CHANGELOG.md b/CHANGELOG.md index 1bc1459b2..e22132db6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,65 @@ ## [Unreleased] +## [3.14.0] - 2026-10-08 + +See [docs/release-notes/v3.14.0.md](docs/release-notes/v3.14.0.md) for the full notes. 23 commits since v3.13.1: 13 fix, 8 feat, 1 ci, 1 docs. + +### Highlights +- **Optional user accounts, off by default** (#2129, #2130, #2138 to #2141) - accounts and roles, settings sync across devices, an admin area with an audit log, hashtag channel proposals with admin approval, mail notifications for watched nodes, data export and daily `users.db` backups. Setup: `docs/user-guide/accounts.md`. +- **New limits on unauthenticated endpoints** (#2119, #2120, #2122, #2123, #2127) - requests above them now get 400 or 429 instead of an answer. **Operator awareness required** for API clients that send long `nodes=` lists. +- **`traffic_share_score` drops after the upgrade and no longer drifts with uptime** (#2117). +- **Regional `/api/nodes` queries no longer exhaust the database pool** (#2114, #2115) - 154.8 s to 37 ms for an uncached region set. + +No manual migration step. With user management off, no new file or table is created. + +## [3.13.1] - 2026-10-04 + +See [docs/release-notes/v3.13.1.md](docs/release-notes/v3.13.1.md) for the full notes. 1 commit since v3.13.0: 1 fix. + +### Highlights +- **Node-discover replies count as coverage** (#2111) - a CoreDrive RX companion's discover replies were dropped from `client_receptions`, so the coverage page showed nothing for them. On a deployment that has had the fix since August they are 44% of coverage rows over 7 days. Replies received before the upgrade are not recovered. + +No manual migration step, no configuration change. + +## [3.13.0] - 2026-10-04 + +See [docs/release-notes/v3.13.0.md](docs/release-notes/v3.13.0.md) for the full notes. 16 commits since v3.12.0: 10 fix, 4 test, 2 feat. + +### Highlights +- **The first start backfills advert route evidence in the background, and flood counts read low until it finishes** (#2088) - measured at 24m 1s on 16.8M observations and 36m 23s on 16.5M on a busier host. Startup and ingest do not wait for it. Runs once per database and resumes after a restart. **Operator awareness required.** +- **Adverts are split by the route they arrived on** (#2088, #2085) - flood, direct with an empty path, or mixed, in Analytics and on node detail. +- **Path hops are resolved with the observer that heard them** (#2099) - with one per-path warning in the packets list when hops stay ambiguous. +- **A compact packets table with a Full Names toggle** (#2090). +- **Channel unread counts and key-only channels survive a list refresh** (#2096). + +No manual migration step: `advert_route_evidence` is created at boot. + +## [3.12.0] - 2026-09-26 + +See [docs/release-notes/v3.12.0.md](docs/release-notes/v3.12.0.md) for the full notes. 29 commits since v3.11.0: 15 fix, 5 test, 3 perf, 3 feat, 2 ci, 1 chore. + +### Highlights +- **The first start after this release runs `ANALYZE`, and ingest stalls while it does** (#2058) - measured at 3m43.9s on a 9.4 GB database, once per database, buffered with nothing dropped. It buys 25% fewer pages read on the region-filtered channel query (143,442 to 107,429). Set `db.analysisLimit` negative to skip it. **Operator awareness required**, and worth picking the moment on a busy instance. +- **`maxMemoryMB` eviction actually triggers** (#2035) - path, decode-cache and dedup-key bytes were unaccounted, so the measured footprint stayed under the configured limit and eviction never fired. Instances relying on that limit will evict where they previously grew. +- **Reception is attributed more honestly** (#2057, #2063, #2064) - zero-hop adverts count as direct reception, Heard By lists only observers that heard the node on air, and its empty state no longer claims the node is out of range. +- **Each observation carries its own wire bytes** (#1999) - the packet detail API handed every observation the same canonical frame. +- **CoreDrive RX region answers are stored** (#2047) - `node_declared_regions` was read but created by nothing, so the Scope Audit described a source that never arrived. Opt-in through `clientRegions`, kept with position, repeater clock and per-collector history. +- **Eight E2E suites had no runner at all** (#2045, #2053) - the aggregate reported a pass while skipping them. + +No manual migration step: `node_declared_regions` (#2047) and `sqlite_stat1` (#2058) are both created at boot. + +## [3.11.0] - 2026-09-16 + +Released and deployed, but it never received a changelog entry at the time; this one is written after the fact and there is no `docs/release-notes/v3.11.0.md`. 52 commits since v3.10.1: 27 fix, 16 feat, 4 test, 2 perf. + +### Highlights +- **Network-wide Scope Audit page** (#1976), with declared-region verification against a repeater's own traffic (#1990), unnameable-traffic accounting (#1987), and opt-in region keys derived from what nodes declare (#1989). +- **SQLite driver swapped** from `modernc.org/sqlite` to `mattn/go-sqlite3`, cross-built with zig (#1992). This makes the build cgo-dependent. +- **Map colours and filters repeaters by scope-configuration state** (#2006), and filters repeaters by region name (#1862). +- Analytics additions: retransmission pressure over time (#1699), scope adverts by node role (#1979), per-node hop-count statistics (#1812). +- The scope-match tally survives ingestor restarts (#2002). + ## [3.10.1] - 2026-09-04 v3.10.0 was tagged and withdrawn before any container image or release asset was published; nothing was ever available under that number. Same release, next number. diff --git a/DEPLOY.md b/DEPLOY.md index af011657a..4a71895eb 100644 --- a/DEPLOY.md +++ b/DEPLOY.md @@ -4,33 +4,38 @@ Pre-built images are published to GHCR for `linux/amd64` and `linux/arm64` (Rasp ## Quick Start -### Docker run +### Complete Compose checkout + +SQLite is the default and runs inside the application container. PostgreSQL18.6 is optional. Existing installations keep their recorded backend; changing it is a verified [offline storage switch](docs/storage.md), not an image update. + +Set `CORESCOPE_REF` to the reviewed application revision, then retain the complete checkout: the Compose variants require `docker/postgres.compose.yml` and its initialization scripts. ```bash -docker run -d --name corescope \ - -p 80:80 \ - -v corescope-data:/app/data \ - -e DISABLE_CADDY=true \ - ghcr.io/kpa-clawbot/corescope:latest +git clone https://github.com/Kpa-clawbot/CoreScope.git +cd CoreScope +git checkout --detach "$CORESCOPE_REF" +test -f .env || cp .env.example .env +chmod 600 .env +# SQLite needs no database credentials. Preserve an existing .env and DATA_DIR. +# Set CORESCOPE_IMAGE in .env to the matching reviewed image tag or digest. +docker compose -f docker-compose.example.yml up -d ``` -Open `http://localhost` — done. +Packaged setup validates an existing SQLite installation or initializes a provably fresh one before starting the application. Open `http://localhost` and verify `/api/healthz` plus real packet ingestion. The default HTTP port is 80; adjust `HTTP_PORT` when using the example variant. -### Docker Compose +For a **new PostgreSQL installation**, merge `.env.postgres.example` into the private `.env`, generate distinct passwords only for the first empty PostgreSQL directory, and run: -```bash -curl -sL https://raw.githubusercontent.com/Kpa-clawbot/CoreScope/master/docker-compose.example.yml \ - -o docker-compose.yml -docker compose up -d +```sh +docker compose -f docker-compose.example.yml -f docker-compose.example.postgres.yml up -d ``` +The optional overlay adds PostgreSQL and owner bootstrap. Runtime services receive only restricted role credentials. Adding/removing an overlay does not convert existing data; use [storage switching](docs/storage.md). Keep the same overlay on PostgreSQL updates. + ## Image Tags | Tag | Description | |-----|-------------| -| `v3.4.1` | Pinned release (recommended for production) | -| `v3.4` | Latest patch in v3.4.x | -| `v3` | Latest minor+patch in v3.x | +| Reviewed release tag or image digest | Pin the application and its matching repository files together | | `latest` | Latest release tag | | `edge` | Built from master — unstable, for testing | @@ -40,7 +45,7 @@ Settings can be overridden via environment variables: | Variable | Default | Description | |----------|---------|-------------| -| `DISABLE_CADDY` | `false` | Skip internal Caddy (set `true` behind a reverse proxy) | +| `DISABLE_CADDY` | `false` in the example | Caddy forwards published port 80 to the Go server on 3000. If disabled, also change your proxy/port mapping to target 3000. | | `DISABLE_MOSQUITTO` | `true` in `docker-compose.staging.yml`; `false` elsewhere | Skip internal MQTT broker. Default flipped to `true` for the staging deploy in v3.7+ because a standalone `mqtt-broker` container owns MQTT on that host — see "Standalone MQTT broker (staging)" below. | | `HTTP_PORT` | `80` | Host port mapping | | `DATA_DIR` | `./data` | Host path for persistent data | @@ -50,22 +55,19 @@ For advanced configuration, mount a `config.json` into `/app/data/config.json`. ## Updating ```bash -docker compose pull -docker compose up -d +docker compose -f docker-compose.example.yml pull +docker compose -f docker-compose.example.yml up -d ``` ## Data -All persistent data lives in `/app/data`: -- `meshcore.db` — SQLite database (packets, nodes) -- `config.json` — custom config (optional) -- `theme.json` — custom theme (optional) +SQLite stores telemetry and accounts as separate files under the data mount. When selected, PostgreSQL persists them in its own host directory (`POSTGRES_DATA_DIR`, or the production/staging equivalent). `/app/data` holds config, theme, queues, statistics and account backup files. Keep these locations separate and preserve the private `.env` with your recovery material. -**Backup:** `cp data/meshcore.db ~/backup/` +Use native SQLite snapshots or PostgreSQL `pg_dump` archives, selected automatically by the recorded backend, or the authenticated backup endpoints. `./manage.sh backup ` and `restore` operate the production Compose variant managed by that script; they do not select `docker-compose.example.yml`. The [backup and restore instructions](docs/postgresql-upgrade.md#native-backups-and-restores) include the example variant. Do not copy live PostgreSQL storage or a SQLite main file while ignoring its WAL. Preserve the private state archive and configuration with the native backup pair. ## TLS -Option A — **External reverse proxy** (recommended): Run with `DISABLE_CADDY=true`, put nginx/traefik/Cloudflare in front. +Option A — **External reverse proxy**: Keep the default internal Caddy and proxy to the published HTTP port. Alternatively, set `DISABLE_CADDY=true` and have the proxy reach container port 3000; change the Compose port mapping from `:80` to `:3000` if the proxy runs on the host. Setting the flag alone leaves the existing port-80 mapping without a listener. Option B — **Built-in Caddy**: Mount a custom Caddyfile at `/etc/caddy/Caddyfile` and expose ports 80+443. @@ -134,13 +136,13 @@ your fork or ensure the network exists. If you're currently deploying with `manage.sh` (git clone + local build), you have two options going forward: -### Option A: Keep using manage.sh (no changes needed) +### Option A: Keep using manage.sh -`manage.sh update` continues to work exactly as before — it fetches the latest tag, builds locally, and restarts. Nothing breaks. +`manage.sh setup` offers SQLite (default) or PostgreSQL. Start/update use the recorded selection, and storage changes require `./manage.sh storage switch sqlite|postgres` with writers stopped. Backups/restores use the selected engine's native tools; existing staging data is retained, and only telemetry is cloned into a fresh staging installation. See [storage operations](docs/storage.md). ```bash ./manage.sh update # latest release -./manage.sh update v3.5.0 # specific version +./manage.sh update "$CORESCOPE_REF" # reviewed version ``` ### Option B: Switch to pre-built images (recommended) @@ -154,16 +156,18 @@ Pre-built images skip the build step entirely — faster updates, no Go toolchai ./manage.sh stop ``` -2. Your data is in `~/meshcore-data/` (or whatever `PROD_DATA_DIR` is set to). It's untouched — the database, config, and theme files persist. +2. Preserve `PROD_DATA_DIR`, the separate PostgreSQL directory, credentials and the native backup pair. Keep the selected backend and its recovery files; use an explicit offline switch only when changing engines. -3. Copy `docker-compose.example.yml` to where you want to run from: +3. Keep the complete pinned checkout and choose the matching image in `.env`: ```bash - cp docker-compose.example.yml ~/docker-compose.yml + # CORESCOPE_IMAGE= + # DATA_DIR= + # POSTGRES_DATA_DIR= ``` 4. Start with the pre-built image: ```bash - cd ~ && docker compose up -d + docker compose -f docker-compose.example.yml up -d ``` 5. Verify it picked up your existing data: @@ -173,19 +177,20 @@ Pre-built images skip the build step entirely — faster updates, no Go toolchai **Updates after migration:** ```bash -docker compose pull && docker compose up -d +docker compose -f docker-compose.example.yml pull +docker compose -f docker-compose.example.yml up -d ``` ### What about manage.sh features? | manage.sh command | Pre-built equivalent | |---|---| -| `./manage.sh update` | `docker compose pull && docker compose up -d` | -| `./manage.sh stop` | `docker compose down` | -| `./manage.sh start` | `docker compose up -d` | -| `./manage.sh logs` | `docker compose logs -f` | -| `./manage.sh status` | `docker compose ps` | -| `./manage.sh setup` | Copy `docker-compose.example.yml`, edit env vars | +| `./manage.sh update` | `docker compose -f docker-compose.example.yml pull` then `up -d` with the same file | +| `./manage.sh stop` | `docker compose -f docker-compose.example.yml stop corescope` | +| `./manage.sh start` | `docker compose -f docker-compose.example.yml up -d` | +| `./manage.sh logs` | `docker compose -f docker-compose.example.yml logs -f` | +| `./manage.sh status` | `docker compose -f docker-compose.example.yml ps` | +| `./manage.sh setup` | Retain the full checkout, configure private credentials and select the Compose variant | `manage.sh` remains available for advanced use cases (building from source, custom patches, development). Pre-built images are recommended for most production deployments. diff --git a/Dockerfile b/Dockerfile index bfb8fd009..58661a72c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,15 @@ -# Build stage always runs natively on the builder's arch ($BUILDPLATFORM) -# and cross-compiles to $TARGETOS/$TARGETARCH via Go toolchain. No QEMU. +# syntax=docker/dockerfile:1 +# Build stage always runs natively on the builder's arch ($BUILDPLATFORM) and +# cross-compiles to $TARGETOS/$TARGETARCH. No QEMU for compilation. +# +# All four binaries support native SQLite and PostgreSQL. SQLite requires cgo; +# the upstream Zig cross-compiler targets musl so both release architectures +# remain fully static and independent of the runtime image libc. +# # BUILDPLATFORM is auto-set by buildx; default to linux/amd64 so plain # `docker build` (without buildx) doesn't fail on an empty platform string. ARG BUILDPLATFORM=linux/amd64 -FROM --platform=$BUILDPLATFORM golang:1.22-alpine AS builder +FROM --platform=$BUILDPLATFORM golang:1.27-alpine AS builder ARG APP_VERSION=unknown ARG GIT_COMMIT=unknown @@ -12,7 +18,38 @@ ARG BUILD_TIME=unknown ARG TARGETOS ARG TARGETARCH -# Build server (pure-Go sqlite — no CGO needed, cross-compiles cleanly) +# Keep these in step with the Makefile. netgo/osusergo retain Go DNS/user +# lookup; sqlite_omit_load_extension permits static linking. +ENV GO_BUILD_TAGS=netgo,osusergo,sqlite_omit_load_extension \ + ZIG_GLOBAL_CACHE_DIR=/tmp/zig-cache + +# Pinned zig, checksum-verified. Arch comes from `uname -m` rather than +# TARGETARCH because this stage is pinned to BUILDPLATFORM — it has to work when +# someone builds on an arm64 machine too. +ARG ZIG_VERSION=0.16.0 +RUN apk add --no-cache curl xz && \ + case "$(uname -m)" in \ + x86_64) ZA=x86_64; ZSHA=70e49664a74374b48b51e6f3fdfbf437f6395d42509050588bd49abe52ba3d00 ;; \ + aarch64) ZA=aarch64; ZSHA=ea4b09bfb22ec6f6c6ceac57ab63efb6b46e17ab08d21f69f3a48b38e1534f17 ;; \ + *) echo "unsupported builder arch $(uname -m)" >&2; exit 1 ;; \ + esac && \ + curl -sSLo /tmp/zig.tar.xz "https://ziglang.org/download/${ZIG_VERSION}/zig-${ZA}-linux-${ZIG_VERSION}.tar.xz" && \ + echo "${ZSHA} /tmp/zig.tar.xz" | sha256sum -c - && \ + mkdir -p /opt/zig && tar -xJf /tmp/zig.tar.xz -C /opt/zig --strip-components=1 && \ + ln -s /opt/zig/zig /usr/local/bin/zig && rm /tmp/zig.tar.xz && \ + zig version + +# zigcc selects the native SQLite C target from the requested image architecture. +RUN printf '%s\n' '#!/bin/sh' \ + 'case "$TARGETARCH" in' \ + ' amd64) t=x86_64-linux-musl ;;' \ + ' arm64) t=aarch64-linux-musl ;;' \ + ' *) echo "unsupported TARGETARCH=$TARGETARCH" >&2; exit 1 ;;' \ + 'esac' \ + 'exec zig cc -target "$t" "$@"' > /usr/local/bin/zigcc && chmod +x /usr/local/bin/zigcc +ENV CC=zigcc CGO_ENABLED=1 + +# Build server WORKDIR /build/server COPY cmd/server/go.mod cmd/server/go.sum ./ COPY internal/geofilter/ ../../internal/geofilter/ @@ -24,14 +61,28 @@ COPY internal/prunequeue/ ../../internal/prunequeue/ COPY internal/perfio/ ../../internal/perfio/ COPY internal/mbcapqueue/ ../../internal/mbcapqueue/ COPY internal/lora/ ../../internal/lora/ -RUN go mod download +COPY internal/users/ ../../internal/users/ +COPY internal/channel/ ../../internal/channel/ +COPY internal/mailer/ ../../internal/mailer/ +COPY internal/pgutil/ ../../internal/pgutil/ +COPY internal/sqliteutil/ ../../internal/sqliteutil/ +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + --mount=type=cache,target=/tmp/zig-cache \ + go mod download COPY cmd/server/ ./ -RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ - go build -ldflags "-X main.Version=${APP_VERSION} -X main.Commit=${GIT_COMMIT} -X main.BuildTime=${BUILD_TIME}" -o /corescope-server . +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + --mount=type=cache,target=/tmp/zig-cache \ + GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ + go build -trimpath -tags ${GO_BUILD_TAGS} \ + -ldflags "-s -w -extldflags '-static -Wl,-s' -X main.Version=${APP_VERSION} -X main.Commit=${GIT_COMMIT} -X main.BuildTime=${BUILD_TIME}" \ + -o /corescope-server . # Build ingestor WORKDIR /build/ingestor COPY cmd/ingestor/go.mod cmd/ingestor/go.sum ./ +COPY internal/channel/ ../../internal/channel/ COPY internal/geofilter/ ../../internal/geofilter/ COPY internal/sigvalidate/ ../../internal/sigvalidate/ COPY internal/packetpath/ ../../internal/packetpath/ @@ -40,29 +91,70 @@ COPY internal/dbschema/ ../../internal/dbschema/ COPY internal/prunequeue/ ../../internal/prunequeue/ COPY internal/perfio/ ../../internal/perfio/ COPY internal/mbcapqueue/ ../../internal/mbcapqueue/ -RUN go mod download +COPY internal/pgutil/ ../../internal/pgutil/ +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + --mount=type=cache,target=/tmp/zig-cache \ + go mod download COPY cmd/ingestor/ ./ -RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ - go build -o /corescope-ingestor . +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + --mount=type=cache,target=/tmp/zig-cache \ + GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ + go build -trimpath -tags ${GO_BUILD_TAGS} \ + -ldflags "-s -w -extldflags '-static -Wl,-s'" \ + -o /corescope-ingestor . # Build decrypt CLI WORKDIR /build/decrypt COPY cmd/decrypt/go.mod cmd/decrypt/go.sum ./ COPY internal/channel/ ../../internal/channel/ -RUN go mod download +COPY internal/dbschema/ ../../internal/dbschema/ +COPY internal/pgutil/ ../../internal/pgutil/ +COPY internal/dbconfig/ ../../internal/dbconfig/ +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + --mount=type=cache,target=/tmp/zig-cache \ + go mod download COPY cmd/decrypt/ ./ -RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ - go build -ldflags="-s -w" -o /corescope-decrypt . - -# Runtime image -FROM alpine:3.20 - -RUN apk add --no-cache mosquitto mosquitto-clients supervisor caddy wget +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + --mount=type=cache,target=/tmp/zig-cache \ + GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ + go build -trimpath -tags ${GO_BUILD_TAGS} \ + -ldflags "-s -w -extldflags '-static -Wl,-s' -X main.version=${APP_VERSION}" \ + -o /corescope-decrypt . + +# Offline bootstrap and verified conversion CLI for both storage engines. +WORKDIR /build/migrate +COPY cmd/migrate/go.mod cmd/migrate/go.sum ./ +COPY internal/packetpath/ ../../internal/packetpath/ +COPY internal/dbschema/ ../../internal/dbschema/ +COPY internal/users/ ../../internal/users/ +COPY internal/pgutil/ ../../internal/pgutil/ +COPY internal/sqliteutil/ ../../internal/sqliteutil/ +COPY internal/dbconfig/ ../../internal/dbconfig/ +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + go mod download +COPY cmd/migrate/ ./ +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + --mount=type=cache,target=/tmp/zig-cache \ + GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ + go build -trimpath -tags ${GO_BUILD_TAGS} \ + -ldflags "-s -w -extldflags '-static -Wl,-s'" \ + -o /corescope-migrate . + +# The client major matches PostgreSQL 18 for native backup/restore support. +FROM alpine:3.24 + +RUN apk add --no-cache mosquitto mosquitto-clients supervisor caddy wget postgresql18-client sqlite jq WORKDIR /app -# Go binaries -COPY --from=builder /corescope-server /corescope-ingestor /corescope-decrypt /app/ +# Go binaries (statically linked; they do not use this image's libc) +COPY --from=builder /corescope-server /corescope-ingestor /corescope-decrypt /corescope-migrate /app/ # Frontend assets + config COPY public/ ./public/ @@ -86,7 +178,8 @@ RUN mkdir -p /app/data /var/lib/mosquitto /data/caddy && \ # Entrypoint COPY docker/entrypoint-go.sh /entrypoint.sh -RUN chmod +x /entrypoint.sh +COPY docker/storage.sh docker/postgres-bootstrap.sh docker/postgres-grants.sql /app/ +RUN chmod +x /entrypoint.sh /app/storage.sh /app/postgres-bootstrap.sh EXPOSE 80 443 1883 diff --git a/Dockerfile.go b/Dockerfile.go deleted file mode 100644 index 77e0329a8..000000000 --- a/Dockerfile.go +++ /dev/null @@ -1,61 +0,0 @@ -FROM golang:1.22-alpine AS builder - -RUN apk add --no-cache build-base - -ARG APP_VERSION=unknown -ARG GIT_COMMIT=unknown -ARG BUILD_TIME=unknown - -# Build server -WORKDIR /build/server -COPY cmd/server/go.mod cmd/server/go.sum ./ -RUN go mod download -COPY cmd/server/ ./ -RUN go build -ldflags "-X main.Version=${APP_VERSION} -X main.Commit=${GIT_COMMIT} -X main.BuildTime=${BUILD_TIME}" -o /corescope-server . - -# Build ingestor -WORKDIR /build/ingestor -COPY cmd/ingestor/go.mod cmd/ingestor/go.sum ./ -RUN go mod download -COPY cmd/ingestor/ ./ -RUN go build -o /corescope-ingestor . - -# Runtime image -FROM alpine:3.20 - -RUN apk add --no-cache mosquitto mosquitto-clients supervisor caddy wget - -WORKDIR /app - -# Go binaries -COPY --from=builder /corescope-server /corescope-ingestor /app/ - -# Frontend assets + config -COPY public/ ./public/ -COPY config.example.json channel-rainbow.json ./ - -# Bake git commit SHA (CI writes .git-commit before build; fallback for non-ldflags usage) -COPY .git-commi[t] ./ -RUN if [ ! -f .git-commit ]; then echo "unknown" > .git-commit; fi - -# Supervisor + Mosquitto + Caddy config -COPY docker/supervisord-go.conf /etc/supervisor/conf.d/supervisord.conf -COPY docker/supervisord-go-no-mosquitto.conf /etc/supervisor/conf.d/supervisord-no-mosquitto.conf -COPY docker/supervisord-go-no-caddy.conf /etc/supervisor/conf.d/supervisord-no-caddy.conf -COPY docker/supervisord-go-no-mosquitto-no-caddy.conf /etc/supervisor/conf.d/supervisord-no-mosquitto-no-caddy.conf -COPY docker/mosquitto.conf /etc/mosquitto/mosquitto.conf -COPY docker/Caddyfile /etc/caddy/Caddyfile - -# Data directory -RUN mkdir -p /app/data /var/lib/mosquitto /data/caddy && \ - chown -R mosquitto:mosquitto /var/lib/mosquitto - -# Entrypoint -COPY docker/entrypoint-go.sh /entrypoint.sh -RUN chmod +x /entrypoint.sh - -EXPOSE 80 443 1883 - -VOLUME ["/app/data", "/data/caddy"] - -ENTRYPOINT ["/entrypoint.sh"] diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..51037f30f --- /dev/null +++ b/Makefile @@ -0,0 +1,130 @@ +# corescope build entry point. +# +# Each Go module resolves internal dependencies through local replace directives. +# SQLite and PostgreSQL are supported by the same binaries. Native SQLite +# needs cgo; Linux cross-builds use zig cc with musl for fully static binaries. +# +# Quick reference: +# make build # all four binaries for the host +# make build-server # just one +# make crossbuild # linux/amd64 + linux/arm64, static, into dist/ +# make test / vet / fmt-check # across all modules +# make docker-build # multi-arch image via buildx + +GO ?= $(shell command -v go) +GIT_VERSION ?= $(shell git describe --tags --match "v*" 2>/dev/null || echo unknown) +GIT_COMMIT ?= $(shell git rev-parse --short HEAD 2>/dev/null || echo unknown) +BUILD_TIME ?= $(shell date -u +%Y-%m-%dT%H:%M:%SZ) + +GOENV_GOOS := $(shell "$(GO)" env GOOS) +GOENV_GOARCH := $(shell "$(GO)" env GOARCH) +GOOS ?= $(GOENV_GOOS) +GOARCH ?= $(GOENV_GOARCH) + +CGO_server := 1 +CGO_ingestor := 1 +CGO_decrypt := 1 +CGO_migrate := 1 +GO_BUILD_TAGS ?= netgo,osusergo,sqlite_omit_load_extension +GO_BUILD_FLAGS ?= -trimpath +GO_LDFLAGS_OPTIMS ?= -s -w + +CMDS := server ingestor decrypt migrate +DIST := dist + +# Per-binary version stamping. Only these two read build metadata; ingestor and +# migrate take none. Kept as make variables so crossbuild and the host build +# cannot drift apart. +LDFLAGS_server := -X main.Version=$(GIT_VERSION) -X main.Commit=$(GIT_COMMIT) -X main.BuildTime=$(BUILD_TIME) +LDFLAGS_ingestor := +LDFLAGS_decrypt := -X main.version=$(GIT_VERSION) +LDFLAGS_migrate := + +# zig target triples for the platforms we ship. musl, so the result is static. +ZIG_TARGET_linux_amd64 := x86_64-linux-musl +ZIG_TARGET_linux_arm64 := aarch64-linux-musl +comma := , +CROSS_PLATFORMS := linux/amd64 linux/arm64 + +DOCKER_IMAGE ?= ghcr.io/kpa-clawbot/corescope +DOCKER_TAG ?= $(GIT_VERSION) +DOCKER_PLATFORMS ?= linux/amd64,linux/arm64 + +.PHONY: all build crossbuild test vet fmt-check tidy clean docker-build docker-push help + +all: build + +help: + @echo "targets: build crossbuild test vet fmt-check tidy clean docker-build docker-push" + @echo " build-{$(shell echo $(CMDS) | tr ' ' ',')}" + +# -- host builds --------------------------------------------------------------- + +build: $(addprefix build-,$(CMDS)) + +build-%: + @mkdir -p $(DIST) + cd cmd/$* && CGO_ENABLED=$(CGO_$*) GOOS=$(GOOS) GOARCH=$(GOARCH) "$(GO)" build \ + -tags $(GO_BUILD_TAGS) $(GO_BUILD_FLAGS) \ + -ldflags "$(GO_LDFLAGS_OPTIMS) $(LDFLAGS_$*)" \ + -o ../../$(DIST)/corescope-$* . + +# -- cross builds -------------------------------------------------------------- +# +# The host build above deliberately leaves CC alone: native builds, `go vet` and +# `-race` must use the host compiler. Only these recipes hand the build to zig. + +crossbuild: $(foreach p,$(CROSS_PLATFORMS),$(foreach c,$(CMDS),crossbuild-$(c)-$(subst /,-,$(p)))) + +define CROSSBUILD_RULE +.PHONY: crossbuild-$(2)-$(3)-$(4) +crossbuild-$(2)-$(3)-$(4): + @mkdir -p $$(DIST) + @command -v zig >/dev/null || { echo "zig not found: needed to cross-compile native SQLite. See https://ziglang.org/download/"; exit 1; } + cd cmd/$(2) && CGO_ENABLED=$$(CGO_$(2)) GOOS=$(3) GOARCH=$(4) CC="zig cc -target $(1)" \ + "$$(GO)" build -tags $$(GO_BUILD_TAGS) $$(GO_BUILD_FLAGS) \ + -ldflags '$$(GO_LDFLAGS_OPTIMS) -extldflags "-static -Wl$(comma)-s" $$(LDFLAGS_$(2))' \ + -o ../../$$(DIST)/corescope-$(2)-$(3)-$(4) . +endef + +$(foreach c,$(CMDS),\ + $(eval $(call CROSSBUILD_RULE,$(ZIG_TARGET_linux_amd64),$(c),linux,amd64))\ + $(eval $(call CROSSBUILD_RULE,$(ZIG_TARGET_linux_arm64),$(c),linux,arm64))) + +# -- checks -------------------------------------------------------------------- + +test: + CGO_ENABLED=1 bash scripts/allmod.sh test ./... + +vet: + bash scripts/allmod.sh vet ./... + +fmt-check: + @unformatted=$$(gofmt -l $$(git ls-files '*.go')); \ + if [ -n "$$unformatted" ]; then echo "gofmt required on:"; echo "$$unformatted"; exit 1; fi; \ + echo "gofmt: clean" + +tidy: + bash scripts/allmod.sh mod tidy + +clean: + rm -rf $(DIST) + +# -- docker -------------------------------------------------------------------- + +docker-build: + docker buildx build . \ + --platform=$(DOCKER_PLATFORMS) \ + --build-arg=APP_VERSION=$(GIT_VERSION) \ + --build-arg=GIT_COMMIT=$(GIT_COMMIT) \ + --build-arg=BUILD_TIME=$(BUILD_TIME) \ + -t $(DOCKER_IMAGE):$(DOCKER_TAG) + +docker-push: DOCKER_PUSH_FLAG := --push +docker-push: + docker buildx build . \ + --platform=$(DOCKER_PLATFORMS) \ + --build-arg=APP_VERSION=$(GIT_VERSION) \ + --build-arg=GIT_COMMIT=$(GIT_COMMIT) \ + --build-arg=BUILD_TIME=$(BUILD_TIME) \ + -t $(DOCKER_IMAGE):$(DOCKER_TAG) $(DOCKER_PUSH_FLAG) diff --git a/README.md b/README.md index 5ba0b3c46..f222e9810 100644 --- a/README.md +++ b/README.md @@ -6,24 +6,15 @@ [![Frontend Coverage](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/Kpa-clawbot/CoreScope/master/.badges/frontend-coverage.json)](https://github.com/Kpa-clawbot/CoreScope/actions/workflows/deploy.yml) [![Deploy](https://github.com/Kpa-clawbot/CoreScope/actions/workflows/deploy.yml/badge.svg)](https://github.com/Kpa-clawbot/CoreScope/actions/workflows/deploy.yml) -> High-performance mesh network analyzer powered by Go. Sub-millisecond packet queries, ~300 MB memory for 56K+ packets, real-time WebSocket broadcast, full channel decryption. +> MeshCore packet analysis with a Go backend, SQLite persistence by default, with optional PostgreSQL, real-time WebSocket updates and channel decryption. Self-hosted, open-source MeshCore packet analyzer. Collects MeshCore packets via MQTT, decodes them in real time, and presents a full web UI with live packet feed, interactive maps, channel chat, packet tracing, and per-node analytics. ## ⚡ Performance -The Go backend serves all 40+ API endpoints from an in-memory packet store with 5 indexes (hash, txID, obsID, observer, node). SQLite is for persistence only — reads never touch disk. +The Go backend combines a packet cache with SQLite or PostgreSQL storage and SQL-backed analytics. SQLite is the default. The ingestor owns telemetry writes; the server reads telemetry and uses a separate optional account store. Ordinary updates keep the recorded backend. Use the [offline storage workflow](docs/storage.md) to change engines without discarding existing data. -| Metric | Value | -|--------|-------| -| Packet queries | **< 1 ms** (in-memory) | -| All API endpoints | **< 100 ms** | -| Memory (56K packets) | **~300 MB** (vs 1.3 GB on Node.js) | -| WebSocket broadcast | **Real-time** to all connected browsers | -| Channel decryption | **AES-128-ECB** with rainbow table | -| GOMEMLIMIT (memory-constrained hosts) | **set to ≥1.5× working set** (e.g. 1536 MiB on a 2 GB Pi for a ~1 GB store). Lower values trigger a GC death-spiral. Configure via the `GOMEMLIMIT` env var or `runtime.maxMemoryMB` in `config.json`; env wins. Applies to both server and ingestor. See [#1010](https://github.com/Kpa-clawbot/CoreScope/issues/1010). | - -See [PERFORMANCE.md](PERFORMANCE.md) for full benchmarks. +Budget for the application and PostgreSQL together, including retained data, indexes, WAL and the packet cache. Historical SQLite measurements in [PERFORMANCE.md](PERFORMANCE.md) are baseline context, not PostgreSQL results. Use the [paired benchmark workflow](.github/workflows/postgres-benchmark.yml) for an exact candidate comparison. ## ✨ Features @@ -75,50 +66,32 @@ Full experience on your phone — proper touch controls, iOS safe area support, ## Quick Start -### Pre-built Image (Recommended) - -No build step required — just run: - -```bash -docker run -d --name corescope \ - --restart=unless-stopped \ - -p 80:80 -p 1883:1883 \ - -v /your/data:/app/data \ - ghcr.io/kpa-clawbot/corescope:latest -``` - -Open `http://localhost` — done. No config file needed; CoreScope starts with sensible defaults. - -For HTTPS with a custom domain, add `-p 443:443` and mount your Caddyfile: -```bash -docker run -d --name corescope \ - --restart=unless-stopped \ - -p 80:80 -p 443:443 -p 1883:1883 \ - -v /your/data:/app/data \ - -v /your/Caddyfile:/etc/caddy/Caddyfile:ro \ - -v /your/caddy-data:/data/caddy \ - ghcr.io/kpa-clawbot/corescope:latest -``` - -Disable built-in services with `-e DISABLE_MOSQUITTO=true` or `-e DISABLE_CADDY=true`, or drop a `.env` file in your data volume. See [docs/deployment.md](docs/deployment.md) for the full reference. +### Pre-built image — SQLite by default -### Build from Source +Retain the complete repository at the same reviewed revision as the image. The optional PostgreSQL overrides use the matching shared initialization scripts. ```bash git clone https://github.com/Kpa-clawbot/CoreScope.git cd CoreScope -./manage.sh setup +git checkout --detach "$CORESCOPE_REF" +test -f .env || cp .env.example .env +chmod 600 .env +# No PostgreSQL service or credentials are needed for the default SQLite install. +# Set CORESCOPE_IMAGE to the matching reviewed image tag or digest. +docker compose -f docker-compose.example.yml up -d ``` -The setup wizard walks you through config, domain, HTTPS, build, and run. +Open `http://localhost` and verify `/api/healthz` plus actual ingestion. Packaged setup validates existing SQLite data or initializes a provably fresh installation before the application starts. Keep the existing `DATA_DIR` on updates. For a new PostgreSQL install, merge `.env.postgres.example` into the private `.env` and add `-f docker-compose.example.postgres.yml`; existing installations use the verified [storage switch](docs/storage.md). See [DEPLOY.md](DEPLOY.md) for ports, MQTT, HTTPS and backups. + +### Build from source + +Use the same complete checkout and private `.env`, then run `./manage.sh setup`. The wizard offers SQLite (default) or PostgreSQL, configures, builds and starts the deployment. It keeps an existing recorded backend; PostgreSQL credentials are generated only for a new empty database directory, preserving existing private settings. ```bash -./manage.sh status # Health check + packet/node counts -./manage.sh logs # Follow logs -./manage.sh backup # Backup database -./manage.sh update # Pull latest + rebuild + restart -./manage.sh mqtt-test # Check if observer data is flowing -./manage.sh help # All commands +./manage.sh status +./manage.sh logs +./manage.sh backup ./backups/instance +./manage.sh update ``` ### Configure @@ -156,32 +129,36 @@ Copy `config.example.json` to `config.json` and edit: | `mqttSources` | External MQTT broker connections (optional) | | `channelKeys` | Channel decryption keys (hex). Hashtag channels auto-derived via SHA256 | | `defaultRegion` | Default IATA region code for the UI | -| `dbPath` | SQLite database path (default: `data/meshcore.db`) | +| `db.backend` | Initial choice: `sqlite` (default) or `postgres`; the installation record wins after setup | +| `dbPath` | SQLite telemetry path; `DB_PATH` is its environment override | +| `databaseURL` | Optional PostgreSQL telemetry URL; prefer the per-process role credentials below | +| `stateDir` | Filesystem directory for queues, statistics and account backups (default: `data`) | ### Environment Variables | Variable | Description | |----------|-------------| | `PORT` | Override config port | -| `DB_PATH` | Override SQLite database path | +| `CORESCOPE_DB_BACKEND` | Bootstrap choice only; use an offline switch to change an installed backend | +| `CORESCOPE_READER_DATABASE_URL` | Server telemetry reader URL | +| `CORESCOPE_WRITER_DATABASE_URL` | Ingestor telemetry writer URL | +| `CORESCOPE_USERS_DATABASE_URL` | Optional account writer URL in a separate database | +| `CORESCOPE_APPROVED_CHANNELS_DATABASE_URL` | Ingestor's restricted approved-channel reader URL | +| `CORESCOPE_STATE_DIR` | Shared state-directory path; never a database URL | ## Architecture -``` - ┌─────────────────────────────────────────────┐ - │ Docker Container │ - │ │ -Observer → USB → │ Mosquitto ──→ Go Ingestor ──→ SQLite DB │ - meshcoretomqtt → MQTT ──→│ │ │ - │ Go HTTP Server ──→ WebSocket │ - │ │ │ │ - │ Caddy (HTTPS) ←───────┘ │ - └────────────────────┼────────────────────────┘ - │ - Browser +```mermaid +flowchart LR + MQTT[MQTT observers] --> Ingestor[Go ingestor] + Ingestor -->|telemetry writer| Telemetry[(SQLite or PostgreSQL telemetry)] + Telemetry -->|read-only role| Server[Go API server] + Server <-->|account writer| Accounts[(Separate account database)] + Accounts -->|approved-channel view| Ingestor + Server <-->|REST and WebSocket| Browser ``` -**Two-process model:** The Go ingestor handles MQTT ingestion and packet decoding. The Go HTTP server loads all packets into an in-memory store on startup (5 indexes for fast lookups) and serves the REST API + WebSocket broadcast. Both are managed by supervisord inside a single container with Caddy for HTTPS and Mosquitto for local MQTT. +**Two-process model:** The ingestor handles MQTT, decoding, telemetry writes and retention. The HTTP server loads its packet cache and serves REST/WebSocket requests using a read-only telemetry role. Accounts use a separate SQLite file or PostgreSQL database. Supervisord runs the application processes with Caddy and optional Mosquitto; PostgreSQL runs as a separate service only when selected. Bootstrap/conversion credentials are not runtime credentials. ## MQTT Setup @@ -201,15 +178,15 @@ corescope/ │ │ ├── main.go # Entry point │ │ ├── routes.go # 40+ API endpoint handlers │ │ ├── store.go # In-memory packet store (5 indexes) -│ │ ├── db.go # SQLite persistence layer +│ │ ├── db.go # Read-only native storage access │ │ ├── decoder.go # MeshCore packet decoder │ │ ├── websocket.go # WebSocket broadcast -│ │ └── *_test.go # 327 test functions +│ │ └── *_test.go # Server tests │ └── ingestor/ # Go MQTT ingestor │ ├── main.go # MQTT subscription + packet processing │ ├── decoder.go # Packet decoder (shared logic) -│ ├── db.go # SQLite write path -│ └── *_test.go # 53 test functions +│ ├── db.go # Native writer and retention path +│ └── *_test.go # Ingestor tests ├── proto/ # Protobuf API definitions ├── public/ # Vanilla JS frontend (no build step) │ ├── index.html # SPA shell @@ -228,26 +205,51 @@ corescope/ │ └── entrypoint-go.sh # Container entrypoint ├── Dockerfile # Multi-stage Go build + Alpine runtime ├── config.example.json # Example configuration -├── test-*.js # Node.js test suite (frontend + legacy) +├── tests/ # Node.js test suite: unit/ (test-all.sh) and e2e/ (Playwright) +├── test-all.sh # Runs every suite in tests/unit └── tools/ # Generators, E2E tests, utilities ``` ## For Developers +### Building + +```bash +make build # all four binaries for your machine, into dist/ +make build-server # just one +make crossbuild # static linux/amd64 + linux/arm64 binaries +``` + +Runtime database access uses pgx through `database/sql`. The offline importer retains +the cgo [`mattn/go-sqlite3`](https://github.com/mattn/go-sqlite3) driver for legacy +sources, so cross-compiling that binary needs a C compiler. `make crossbuild` uses +[`zig`](https://ziglang.org/download/) as that compiler (install it and it just +works) and links statically against musl, so the result is one self-contained file +that runs on Alpine or scratch. The container build does the same thing — see +`Dockerfile`. + ### Test Suite -**380 Go tests** covering the backend, plus **150+ Node.js tests** for the frontend and legacy logic, plus **49 Playwright E2E tests** for browser validation. +Default Go suites exercise native SQLite without a PostgreSQL service. The explicit PostgreSQL matrix covers role boundaries, concurrency, migration and native backup/restore. The standalone frontend list remains `test-all.sh`; Playwright covers the imported fixture and accounts on/off. CI pins Go 1.27.2 and PostgreSQL 18.6. ```bash +# Default SQLite tests need no service. For the separate PostgreSQL matrix: +export CORESCOPE_TEST_BACKEND=postgres +# Point this at an isolated PostgreSQL18 administrator database. +# Keep the URL in private environment configuration; pg_dump/pg_restore 18 must be on PATH. +# CORESCOPE_TEST_POSTGRES_URL is required; database tests do not silently skip. # Go backend tests cd cmd/server && go test ./... -v cd cmd/ingestor && go test ./... -v +# Or across all modules +make test + # Node.js frontend + integration tests npm test # Playwright E2E (requires running server on localhost:3000) -node test-e2e-playwright.js +node tests/e2e/test-e2e-playwright.js ``` ### Generate Test Data @@ -258,7 +260,7 @@ node tools/generate-packets.js --api --count 200 ### Migrating from Node.js -If you're running an existing Node.js deployment, see [docs/go-migration.md](docs/go-migration.md) for a step-by-step guide. The Go engine reads the same SQLite database and `config.json` — no data migration needed. +Historical Node.js/SQLite deployments must first reach a supported SQLite layout using the prior version's explicit upgrade procedure. They can keep SQLite. Use the [offline storage switch](docs/storage.md) only when changing engines. Preserve original data, configuration and keys; a container/image update does not perform a backend conversion. ## Contributing diff --git a/channel-rainbow.json b/channel-rainbow.json index 73ad3a237..23e1e7829 100644 --- a/channel-rainbow.json +++ b/channel-rainbow.json @@ -295,5 +295,28 @@ "#bogota": "6d0864985b64350ce4cbfebf4979e970", "#peru": "7e6fc347bf29a4c128ac3156865bd521", "#lima": "5f167ce354eca08ab742463df10ef255", - "Public": "8b3387e9c5cdea6ac9e5edbaa115cd72" + "Public": "8b3387e9c5cdea6ac9e5edbaa115cd72", + "#Public": "7534fe292203ccdce93db79377d4b566", + "#newbie": "ff2ecc59f59dbf051a10d880f0b4308d", + "#power": "c4d23a51329aeaee767234644d52cff8", + "#garden": "56ff6fe3500872c2765dbc178350005c", + "#jokes": "abe610d61dee508c0ac18793c36c41c3", + "#space": "5caa27568ba7a216bcc09fcb69c77ea1", + "#path": "29eed6c759d96346b30c869e23fd86f8", + "#hamradio": "83c8b01997654265938da8765cbc7db9", + "#amrron": "68ec397a1dd4f3416911d00211ed0ef6", + "#family": "7d1b805d4bf059c2ef06ed221794663b", + "#testing": "cde5e82cf515647dcb547a79a4f065d1", + "#wicked": "05e990d48943ee03c943c41dea0ddfe1", + "#gay": "3a612fa3bfc9f3b439cc230b01cbbc00", + "#t": "7b5336e85005be805b496bc27c722ff3", + "#doom": "7034910637d6b9b6c2cc3bcd8dfe9bb5", + "#empty": "3351cfa0dad99e1777cce210c983b561", + "#puppy": "c20cf85aab9561e6e65ac4a19a852198", + "#naked": "f423f23ddcf9e472c2ace38dce7591bc", + "#cruising": "550857053360e3492755251f9142e74c", + "#nightowls": "1411c5c3732d3df1b855644f78c47417", + "#worldcup": "d5786cc7bcee5a4828bbb20c3825c5a9", + "#nofilter": "cab31f589959debcd1721b7d252259b1", + "#politics": "64cc9fdaad67f03f23b4cec8c9a4468f" } diff --git a/cmd/decrypt/README.md b/cmd/decrypt/README.md index 3aa70a482..68d8b47c3 100644 --- a/cmd/decrypt/README.md +++ b/cmd/decrypt/README.md @@ -1,6 +1,6 @@ # corescope-decrypt -Standalone CLI tool to decrypt and export MeshCore hashtag channel messages from a CoreScope SQLite database. +Standalone CLI tool to decrypt and export MeshCore hashtag channel messages from CoreScope storage. SQLite is the default; PostgreSQL is optional. Both are opened read-only. ## Why @@ -24,7 +24,7 @@ docker exec corescope-prod /app/corescope-decrypt --channel "#wardriving" --db / ### From GitHub release -Download the static binary from the [Releases](https://github.com/Kpa-clawbot/CoreScope/releases) page: +Download the binary for your platform from the [Releases](https://github.com/Kpa-clawbot/CoreScope/releases) page: ```bash # Linux amd64 @@ -37,19 +37,31 @@ chmod +x corescope-decrypt-linux-amd64 ```bash cd cmd/decrypt -CGO_ENABLED=0 go build -ldflags="-s -w" -o corescope-decrypt . +go build -ldflags="-s -w" -o corescope-decrypt . # cgo: the SQLite driver needs it ``` -The binary is statically linked — no dependencies, runs on any Linux. +Building SQLite support requires a C compiler and CGO enabled. Use the packaged image or a release binary matching your platform. ## Usage ``` -corescope-decrypt --channel NAME --db PATH [--format FORMAT] [--output FILE] +corescope-decrypt --channel NAME [--db PATH] [--format FORMAT] [--output FILE] ``` Run `corescope-decrypt --help` for full flag documentation. +### Storage selection + +The tool reads storage fields from `config.json`, then `data/config.json` if the first file is absent. `--config FILE` selects an explicit file. An unreadable or malformed selected config is an error. Relative paths use the working directory. + +For an installed instance, `storage-selection.json` is authoritative: stale `--db` paths or bootstrap backend settings do not override a completed backend switch. The tool finds that record using `--state-dir`, `CORESCOPE_STATE_DIR`, or configured `stateDir`; otherwise it uses the legacy SQLite database directory (default `data`). Keep the installation's stable state directory when exporting after a switch. A shared lease blocks a switch until the export exits. + +Without a recorded installation, `--db PATH` reads an existing SQLite file; the default is `data/meshcore.db`. It never creates, adopts, or migrates a database. Incomplete conversions, missing files, and incompatible schemas fail. Offline legacy files remain readable when they contain the columns needed for export. + +For PostgreSQL, supply a restricted reader URL through `CORESCOPE_READER_DATABASE_URL`, or the generic `CORESCOPE_DATABASE_URL`. `--database-url` is available but environment settings keep credentials out of command history. PostgreSQL role and schema-readiness checks remain enforced; a failed connection never falls back to SQLite. + +Before a selection has been recorded, `db.backend`, `CORESCOPE_DB_BACKEND`, or `--backend sqlite|postgres` makes the choice explicit; conflicting config/environment selectors fail. Otherwise unambiguous native settings determine the engine. CLI paths/URLs override their environment/config values; environment values override config. `DB_PATH` is the legacy SQLite path variable. Mixed unrecorded file and URL settings require an explicit backend choice. + ### JSON output (default) Machine-readable, includes all metadata (observers, path hops, raw hex): @@ -123,6 +135,9 @@ See the firmware source at `firmware/src/helpers/BaseChatMesh.cpp` for the canon cd cmd/decrypt go test ./... +# Explicit PostgreSQL matrix: requires a disposable administrator URL. +CORESCOPE_TEST_BACKEND=postgres go test ./... + # Manual test with the real fixture: go run . --channel "#wardriving" --db ../../test-fixtures/e2e-fixture.db --format irc ``` diff --git a/cmd/decrypt/cli_test.go b/cmd/decrypt/cli_test.go new file mode 100644 index 000000000..0d13ef233 --- /dev/null +++ b/cmd/decrypt/cli_test.go @@ -0,0 +1,80 @@ +package main + +import ( + "bytes" + "encoding/json" + "flag" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// Run the real flag/storage/query/render path without building another binary. +func TestDecryptCLIProcess(t *testing.T) { + if os.Getenv("CORESCOPE_DECRYPT_TEST_CHILD") != "1" { + return + } + for i, arg := range os.Args { + if arg == "--" { + os.Args = append([]string{"corescope-decrypt"}, os.Args[i+1:]...) + break + } + } + flag.CommandLine = flag.NewFlagSet("corescope-decrypt", flag.ExitOnError) + main() + os.Exit(0) +} + +func runDecryptCLI(t *testing.T, base string, args ...string) ([]byte, string, error) { + t.Helper() + executable, err := os.Executable() + if err != nil { + t.Fatal(err) + } + cmd := exec.Command(executable, append([]string{"-test.run=^TestDecryptCLIProcess$", "--"}, args...)...) + cmd.Dir = base + for _, value := range os.Environ() { + key, _, _ := strings.Cut(value, "=") + switch key { + case "DB_PATH", "CORESCOPE_DB_BACKEND", "CORESCOPE_DATABASE_URL", "CORESCOPE_READER_DATABASE_URL", "CORESCOPE_STATE_DIR", "CORESCOPE_DECRYPT_TEST_CHILD": + continue + } + cmd.Env = append(cmd.Env, value) + } + cmd.Env = append(cmd.Env, "CORESCOPE_DECRYPT_TEST_CHILD=1") + var out, diagnostic bytes.Buffer + cmd.Stdout, cmd.Stderr = &out, &diagnostic + err = cmd.Run() + return out.Bytes(), diagnostic.String(), err +} + +func TestDecryptNativeCLI(t *testing.T) { + base := t.TempDir() + data := filepath.Join(base, "data") + if err := os.Mkdir(data, 0700); err != nil { + t.Fatal(err) + } + path := filepath.Join(data, "meshcore.db") + db := sqliteExportFixture(t, path, true) + for _, args := range [][]string{ + {"--channel", "#example"}, + {"--channel", "#example", "--db", path}, + } { + out, diagnostic, err := runDecryptCLI(t, base, args...) + if err != nil || !json.Valid(out) || !strings.Contains(diagnostic, "Scanned 0 GRP_TXT packets") { + t.Fatalf("native CLI export failed: %v; %s", err, diagnostic) + } + } + db.Close() + if err := os.Remove(path); err != nil { + t.Fatal(err) + } + if _, _, err := runDecryptCLI(t, base, "--channel", "#example"); err == nil { + t.Fatal("default CLI accepted a missing database") + } + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Fatal("default CLI created a missing database") + } +} diff --git a/cmd/decrypt/go.mod b/cmd/decrypt/go.mod index 185a76aee..fc8c6bb40 100644 --- a/cmd/decrypt/go.mod +++ b/cmd/decrypt/go.mod @@ -1,22 +1,29 @@ module github.com/corescope/decrypt -go 1.22 +go 1.25.0 require ( github.com/meshcore-analyzer/channel v0.0.0 - modernc.org/sqlite v1.34.5 + github.com/meshcore-analyzer/dbschema v0.0.0 + github.com/meshcore-analyzer/pgutil v0.0.0 ) require ( - github.com/dustin/go-humanize v1.0.1 // indirect - github.com/google/uuid v1.6.0 // indirect - github.com/mattn/go-isatty v0.0.20 // indirect - github.com/ncruces/go-strftime v0.1.9 // indirect - github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect - golang.org/x/sys v0.22.0 // indirect - modernc.org/libc v1.55.3 // indirect - modernc.org/mathutil v1.6.0 // indirect - modernc.org/memory v1.8.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/pgx/v5 v5.7.6 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/mattn/go-sqlite3 v1.14.52 + github.com/meshcore-analyzer/dbconfig v0.0.0 + golang.org/x/crypto v0.37.0 // indirect + golang.org/x/sync v0.13.0 // indirect + golang.org/x/text v0.24.0 // indirect ) replace github.com/meshcore-analyzer/channel => ../../internal/channel + +replace github.com/meshcore-analyzer/dbschema => ../../internal/dbschema + +replace github.com/meshcore-analyzer/pgutil => ../../internal/pgutil + +replace github.com/meshcore-analyzer/dbconfig => ../../internal/dbconfig diff --git a/cmd/decrypt/go.sum b/cmd/decrypt/go.sum index 5424fe41d..c83dde8cc 100644 --- a/cmd/decrypt/go.sum +++ b/cmd/decrypt/go.sum @@ -1,43 +1,30 @@ -github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo= -github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= -github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= -github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= -golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= -golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw= -golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc= -modernc.org/cc/v4 v4.21.4 h1:3Be/Rdo1fpr8GrQ7IVw9OHtplU4gWbb+wNgeoBMmGLQ= -modernc.org/cc/v4 v4.21.4/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ= -modernc.org/ccgo/v4 v4.19.2 h1:lwQZgvboKD0jBwdaeVCTouxhxAyN6iawF3STraAal8Y= -modernc.org/ccgo/v4 v4.19.2/go.mod h1:ysS3mxiMV38XGRTTcgo0DQTeTmAO4oCmJl1nX9VFI3s= -modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE= -modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ= -modernc.org/gc/v2 v2.4.1 h1:9cNzOqPyMJBvrUipmynX0ZohMhcxPtMccYgGOJdOiBw= -modernc.org/gc/v2 v2.4.1/go.mod h1:wzN5dK1AzVGoH6XOzc3YZ+ey/jPgYHLuVckd62P0GYU= -modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U= -modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w= -modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= -modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= -modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E= -modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU= -modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= -modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= -modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc= -modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss= -modernc.org/sqlite v1.34.5 h1:Bb6SR13/fjp15jt70CL4f18JIN7p7dnMExd+UFnF15g= -modernc.org/sqlite v1.34.5/go.mod h1:YLuNmX9NKs8wRNK2ko1LW1NGYcc9FkBO69JOt1AR9JE= -modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= -modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= -modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= -modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.7.6 h1:rWQc5FwZSPX58r1OQmkuaNicxdmExaEz5A2DO2hUuTk= +github.com/jackc/pgx/v5 v5.7.6/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/mattn/go-sqlite3 v1.14.52 h1:wVbm2Qnf4OXkqhBTSPuCRZDRnxfbVrrmiCEroVdog8U= +github.com/mattn/go-sqlite3 v1.14.52/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= +golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= +golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= +golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= +golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/cmd/decrypt/main.go b/cmd/decrypt/main.go index 9a19ca69d..48583de9d 100644 --- a/cmd/decrypt/main.go +++ b/cmd/decrypt/main.go @@ -1,8 +1,8 @@ -// corescope-decrypt decrypts and exports hashtag channel messages from a CoreScope SQLite database. +// corescope-decrypt decrypts and exports hashtag channel messages from CoreScope storage. // // Usage: // -// corescope-decrypt --channel "#wardriving" --db meshcore.db [--format json|html] [--output file] +// corescope-decrypt --channel "#wardriving" [--format json|html] [--output file] package main import ( @@ -19,7 +19,7 @@ import ( "time" "github.com/meshcore-analyzer/channel" - _ "modernc.org/sqlite" + "github.com/meshcore-analyzer/dbconfig" ) // Version info (set via ldflags). @@ -47,7 +47,11 @@ type Observer struct { func main() { channelName := flag.String("channel", "", "Channel name (e.g. \"#wardriving\")") - dbPath := flag.String("db", "", "Path to CoreScope SQLite database") + databaseURL := flag.String("database-url", "", "PostgreSQL reader URL (prefer CORESCOPE_READER_DATABASE_URL)") + dbPath := flag.String("db", "", "Existing SQLite database path (default: data/meshcore.db)") + backend := flag.String("backend", "", "Bootstrap backend: sqlite or postgres (installed selection wins)") + stateDir := flag.String("state-dir", "", "Installation state directory (CORESCOPE_STATE_DIR)") + configPath := flag.String("config", "", "Storage config file (default: config.json, then data/config.json)") format := flag.String("format", "json", "Output format: json, html, irc (or log)") output := flag.String("output", "", "Output file (default: stdout)") showVersion := flag.Bool("version", false, "Print version and exit") @@ -56,12 +60,17 @@ func main() { fmt.Fprintf(os.Stderr, `corescope-decrypt — Decrypt and export MeshCore hashtag channel messages USAGE - corescope-decrypt --channel NAME --db PATH [--format FORMAT] [--output FILE] + corescope-decrypt --channel NAME [--format FORMAT] [--output FILE] FLAGS --channel NAME Channel name to decrypt (e.g. "#wardriving", "wardriving") The "#" prefix is added automatically if missing. - --db PATH Path to a CoreScope SQLite database file (read-only access). + --db PATH Existing SQLite database (default: data/meshcore.db). + --backend NAME Bootstrap choice: sqlite or postgres. + --state-dir DIR Find the installed storage selection in this directory. + --config FILE Read storage settings from this config file. + --database-url PostgreSQL reader URL. Prefer CORESCOPE_READER_DATABASE_URL + or CORESCOPE_DATABASE_URL to keep credentials out of argv. --format FORMAT Output format (default: json): json — Machine-readable JSON array with full metadata html — Self-contained HTML viewer with search and sorting @@ -70,19 +79,22 @@ FLAGS --output FILE Write output to FILE instead of stdout. --version Print version and exit. + The installed selection takes precedence over bootstrap paths and backend flags. + Export holds a shared selection lease and never creates or migrates a database. + EXAMPLES # Export #wardriving messages as JSON - corescope-decrypt --channel "#wardriving" --db /app/data/meshcore.db + corescope-decrypt --channel "#wardriving" # Generate an interactive HTML viewer - corescope-decrypt --channel wardriving --db meshcore.db --format html --output wardriving.html + corescope-decrypt --channel wardriving --format html --output wardriving.html # Greppable IRC log - corescope-decrypt --channel "#MeshCore" --db meshcore.db --format irc --output meshcore.log + corescope-decrypt --channel "#MeshCore" --format irc --output meshcore.log grep "KE6QR" meshcore.log # From the Docker container - docker exec corescope-prod /app/corescope-decrypt --channel "#wardriving" --db /app/data/meshcore.db + docker exec corescope-prod /app/corescope-decrypt --channel "#wardriving" RETROACTIVE DECRYPTION MeshCore hashtag channels use symmetric encryption — the key is derived from the @@ -107,7 +119,7 @@ LIMITATIONS os.Exit(0) } - if *channelName == "" || *dbPath == "" { + if *channelName == "" { flag.Usage() os.Exit(1) } @@ -121,14 +133,21 @@ LIMITATIONS key := channel.DeriveKey(ch) chHash := channel.ChannelHash(key) - db, err := sql.Open("sqlite", *dbPath+"?mode=ro") + raw, err := exportStorageInputs(".", exportOptions{Backend: dbconfig.Backend(*backend), DBPath: *dbPath, DatabaseURL: *databaseURL, StateDir: *stateDir, ConfigPath: *configPath}, os.Getenv) + if err != nil { + log.Fatalf("Storage configuration: %v", err) + } + db, lease, err := openSelectedExport(raw) if err != nil { log.Fatalf("Failed to open database: %v", err) } + if lease != nil { + defer lease.Close() + } defer db.Close() // Query all GRP_TXT packets - rows, err := db.Query(`SELECT id, hash, raw_hex, first_seen FROM transmissions WHERE payload_type = 5`) + rows, err := db.Query(exportTransmissionsSQL) if err != nil { log.Fatalf("Query failed: %v", err) } @@ -278,7 +297,7 @@ func extractGRPPayload(rawHex string) ([]byte, error) { func getPathFromDB(db *sql.DB, txID int) []string { var decodedJSON sql.NullString - err := db.QueryRow(`SELECT decoded_json FROM transmissions WHERE id = ?`, txID).Scan(&decodedJSON) + err := db.QueryRow(exportPathSQL, txID).Scan(&decodedJSON) if err != nil || !decodedJSON.Valid { return nil } @@ -295,13 +314,7 @@ func getPathFromDB(db *sql.DB, txID int) []string { } func getObservers(db *sql.DB, txID int) []Observer { - rows, err := db.Query(` - SELECT o.name, obs.snr, obs.rssi, obs.timestamp - FROM observations obs - LEFT JOIN observers o ON o.id = CAST(obs.observer_idx AS TEXT) - WHERE obs.transmission_id = ? - ORDER BY obs.timestamp - `, txID) + rows, err := db.Query(exportObserversSQL, txID) if err != nil { return nil } diff --git a/cmd/decrypt/main_test.go b/cmd/decrypt/main_test.go index 76588a458..5d173a80f 100644 --- a/cmd/decrypt/main_test.go +++ b/cmd/decrypt/main_test.go @@ -4,6 +4,7 @@ import ( "encoding/hex" "encoding/json" "os" + "path/filepath" "strings" "testing" @@ -113,17 +114,27 @@ func TestJSONOutputParseable(t *testing.T) { } } -// Integration test against fixture DB (skipped if DB not found) +// Exercise the actual CLI against the checked-in, captured packet fixture. func TestFixtureDecrypt(t *testing.T) { - dbPath := "../../test-fixtures/e2e-fixture.db" - if _, err := os.Stat(dbPath); os.IsNotExist(err) { - t.Skip("fixture DB not found") + dbPath, err := filepath.Abs("../../test-fixtures/e2e-fixture.db") + if err != nil { + t.Fatal(err) } - - // We know the fixture has #wardriving messages with channelHash 0x81 - key := channel.DeriveKey("#wardriving") - ch := channel.ChannelHash(key) - if ch != 0x81 { - t.Fatalf("unexpected channel hash: %02X", ch) + if _, err := os.Stat(dbPath); err != nil { + t.Fatal("checked-in fixture database is required") + } + base := t.TempDir() + out, diagnostic, err := runDecryptCLI(t, base, "--channel", "#wardriving", "--db", dbPath, "--state-dir", base) + if err != nil { + t.Fatalf("fixture export failed: %v; %s", err, diagnostic) + } + var messages []ChannelMessage + if err := json.Unmarshal(out, &messages); err != nil || len(messages) == 0 { + t.Fatalf("fixture export produced no valid messages: %v", err) + } + for _, message := range messages { + if message.Channel != "#wardriving" || message.RawHex == "" || message.Timestamp == "" { + t.Fatal("fixture export lost message fields") + } } } diff --git a/cmd/decrypt/postgres_test.go b/cmd/decrypt/postgres_test.go new file mode 100644 index 000000000..9441d1e70 --- /dev/null +++ b/cmd/decrypt/postgres_test.go @@ -0,0 +1,64 @@ +package main + +import ( + "os" + "path/filepath" + "testing" + + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/pgutil" + "github.com/meshcore-analyzer/pgutil/pgtest" +) + +func TestPostgresExportReadsPathAndObservers(t *testing.T) { + if backend := os.Getenv("CORESCOPE_TEST_BACKEND"); backend != "postgres" { + if backend != "" && backend != "sqlite" { + t.Fatal("CORESCOPE_TEST_BACKEND must be sqlite or postgres") + } + t.Skip("PostgreSQL matrix is not selected") + } + dsn := pgtest.NewSchema(t) + writer, err := pgutil.Open(dsn, false) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { writer.Close() }) + if err := dbschema.Apply(writer, t.Logf); err != nil { + t.Fatal(err) + } + for _, query := range []string{ + `INSERT INTO transmissions(id,raw_hex,hash,first_seen,decoded_json) VALUES(1,'00','export-fixture','2026-01-01','{"path":{"hops":["ab","cd"]}}')`, + `INSERT INTO observers(rowid,id,name) VALUES(10,'observer','Example observer')`, + `INSERT INTO observations(transmission_id,observer_idx,snr,rssi,timestamp,path_json) VALUES(1,10,1.5,-90,1767225600,'["ab"]')`, + } { + if _, err := writer.Exec(query); err != nil { + t.Fatal(err) + } + } + reader, err := openExportDB(pgtest.ReadOnly(t, dsn)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { reader.Close() }) + assertExportMetadata(t, reader) + if _, err := reader.Exec(`DELETE FROM observations`); err == nil { + t.Fatal("PostgreSQL export reader permitted a write") + } + base := t.TempDir() + readerURL := pgtest.ReadOnly(t, dsn) + recordExportStorage(t, dbconfig.Storage{Backend: dbconfig.Postgres, StateDir: base, ReaderDatabaseURL: readerURL}) + selected, lease, err := openSelectedExport(dbconfig.StorageInputs{BaseDir: base, DBPath: filepath.Join(base, "stale.db"), Backend: dbconfig.SQLite, ReaderDatabaseURL: readerURL}) + if err != nil { + t.Fatal(err) + } + assertExportMetadata(t, selected) + selected.Close() + lease.Close() + if _, err := writer.Exec(`UPDATE corescope_schema SET ready=false`); err != nil { + t.Fatal(err) + } + if db, lease, err := openSelectedExport(dbconfig.StorageInputs{BaseDir: base, StateDir: base, ReaderDatabaseURL: readerURL}); err == nil || db != nil || lease != nil { + t.Fatal("export accepted an incomplete PostgreSQL import") + } +} diff --git a/cmd/decrypt/storage.go b/cmd/decrypt/storage.go new file mode 100644 index 000000000..51f4762ad --- /dev/null +++ b/cmd/decrypt/storage.go @@ -0,0 +1,173 @@ +package main + +import ( + "database/sql" + "errors" + "io" + "net/url" + "os" + "path/filepath" + "strings" + + _ "github.com/mattn/go-sqlite3" + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/pgutil" +) + +const exportTransmissionsSQL = `SELECT id, hash, raw_hex, first_seen FROM transmissions WHERE payload_type = 5` +const exportPathSQL = `SELECT decoded_json FROM transmissions WHERE id = $1` +const exportObserversSQL = ` + SELECT o.name, obs.snr, obs.rssi, obs.timestamp + FROM observations obs + LEFT JOIN observers o ON o.rowid = obs.observer_idx + WHERE obs.transmission_id = $1 + ORDER BY obs.timestamp, obs.id` + +type exportOptions struct { + Backend dbconfig.Backend + DBPath, DatabaseURL, StateDir, ConfigPath string +} + +// Use the installation's storage fields without depending on the server's full +// configuration. Only absent search candidates fall through to defaults. +func exportStorageInputs(base string, flags exportOptions, getenv func(string) string) (dbconfig.StorageInputs, error) { + base, err := filepath.Abs(base) + if err != nil { + return dbconfig.StorageInputs{}, err + } + paths := []string{filepath.Join(base, "config.json"), filepath.Join(base, "data", "config.json")} + if flags.ConfigPath != "" { + paths = []string{flags.ConfigPath} + } + var raw dbconfig.StorageInputs + for _, path := range paths { + raw, err = dbconfig.ReadStorageConfig(path) + if err == nil { + break + } + if flags.ConfigPath != "" || !errors.Is(err, os.ErrNotExist) { + return raw, err + } + } + raw.BaseDir = base + raw.EnvBackend = dbconfig.Backend(getenv("CORESCOPE_DB_BACKEND")) + for key, target := range map[string]*string{ + "DB_PATH": &raw.DBPath, "CORESCOPE_STATE_DIR": &raw.StateDir, + "CORESCOPE_DATABASE_URL": &raw.DatabaseURL, "CORESCOPE_READER_DATABASE_URL": &raw.ReaderDatabaseURL, + } { + if value := strings.TrimSpace(getenv(key)); value != "" { + *target = value + } + } + if flags.Backend != "" { + raw.Backend = flags.Backend + } + if flags.DBPath != "" { + raw.DBPath = flags.DBPath + } + if flags.DatabaseURL != "" { + raw.DatabaseURL, raw.ReaderDatabaseURL = flags.DatabaseURL, flags.DatabaseURL + } + if flags.StateDir != "" { + raw.StateDir = flags.StateDir + } + return raw, nil +} + +func openSelectedExport(raw dbconfig.StorageInputs) (*sql.DB, io.Closer, error) { + // The legacy SQLite directory remains the discovery anchor after a switch. + state := raw.StateDir + if state == "" { + path := raw.DBPath + if path == "" { + path = filepath.Join("data", "meshcore.db") + } + if strings.Contains(path, "://") || strings.HasPrefix(path, "postgres:") || strings.HasPrefix(path, "postgresql:") { + return nil, nil, errors.New("--db requires a SQLite filesystem path") + } + state = filepath.Dir(path) + } + if strings.Contains(state, "://") || strings.ContainsRune(state, 0) || (filepath.VolumeName(state) != "" && !filepath.IsAbs(state)) { + return nil, nil, errors.New("invalid storage state directory") + } + if !filepath.IsAbs(state) { + state = filepath.Join(raw.BaseDir, state) + } + selected, lease, err := dbconfig.OpenSelection(dbconfig.SelectionPath(filepath.Clean(state))) + if err == nil { + raw, err = selected.ApplyTo(raw) + } else if errors.Is(err, dbconfig.ErrSelectionMissing) { + // An offline/legacy export can read an existing source without adopting + // it. This default is never permission to create or migrate a database. + err = nil + if raw.Backend == "" && raw.EnvBackend == "" && raw.DBPath == "" && raw.UsersDBPath == "" && raw.DatabaseURL == "" && raw.ReaderDatabaseURL == "" && raw.WriterDatabaseURL == "" && raw.UsersDatabaseURL == "" && raw.ApprovedChannelsDatabaseURL == "" { + raw.DBPath = filepath.Join("data", "meshcore.db") + } + } + var db *sql.DB + if err == nil { + var storage dbconfig.Storage + storage, err = dbconfig.ResolveStorage(raw) + if err == nil { + target := storage.DBPath + if storage.Backend == dbconfig.Postgres { + target = storage.ReaderDatabaseURL + } + db, err = openExportDB(target) + } + } + if err != nil && lease != nil { + lease.Close() + lease = nil + } + return db, lease, err +} + +func openExportDB(target string) (*sql.DB, error) { + postgres := strings.HasPrefix(target, "postgres://") || strings.HasPrefix(target, "postgresql://") + var db *sql.DB + var err error + if postgres { + db, err = pgutil.Open(target, true) + } else { + if target == "" || strings.Contains(target, "://") || strings.HasPrefix(target, "postgres:") || strings.HasPrefix(target, "postgresql:") { + return nil, errors.New("export requires a SQLite filesystem path or PostgreSQL reader URL") + } + var dsn string + dsn, err = dbconfig.SQLiteURI(target, url.Values{"mode": {"ro"}, "_query_only": {"on"}, "_busy_timeout": {"5000"}}) + if err == nil { + db, err = sql.Open("sqlite3", dsn) + } + } + if err != nil { + return nil, err + } + db.SetMaxOpenConns(4) + db.SetMaxIdleConns(4) + if postgres { + err = pgutil.AssertReadOnly(db) + if err == nil { + err = dbschema.AssertPostgresReady(db) + } + } else { + err = dbconfig.AssertSQLiteImportComplete(db) + // Export needs only its existing read columns, including on older + // offline snapshots. Do not run migrations or require writer indexes. + for _, query := range []string{exportTransmissionsSQL, exportPathSQL, exportObserversSQL} { + if err != nil { + break + } + var stmt *sql.Stmt + stmt, err = db.Prepare(query) + if err == nil { + err = stmt.Close() + } + } + } + if err != nil { + db.Close() + return nil, err + } + return db, nil +} diff --git a/cmd/decrypt/storage_test.go b/cmd/decrypt/storage_test.go new file mode 100644 index 000000000..2b342eda3 --- /dev/null +++ b/cmd/decrypt/storage_test.go @@ -0,0 +1,306 @@ +package main + +import ( + "bytes" + "database/sql" + "errors" + "net" + "os" + "path/filepath" + "strings" + "testing" + + _ "github.com/mattn/go-sqlite3" + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/dbschema/legacy" +) + +func sqliteExportFixture(t *testing.T, path string, current bool) *sql.DB { + t.Helper() + dsn, err := legacy.WriterDSN(path) + if err != nil { + t.Fatal(err) + } + db, err := sql.Open("sqlite3", dsn) + if err != nil { + t.Fatal(err) + } + db.SetMaxOpenConns(1) + t.Cleanup(func() { db.Close() }) + if current { + err = dbschema.ApplySQLite(db, t.Logf) + } else if err = legacy.ApplyBase(db); err == nil { + err = legacy.Apply(db, t.Logf) + } + if err != nil { + t.Fatal(err) + } + for _, query := range []string{ + `INSERT INTO transmissions(id,raw_hex,hash,first_seen,decoded_json) VALUES(1,'00','export-fixture','2026-01-01','{"path":{"hops":["ab","cd"]}}')`, + `INSERT INTO observers(rowid,id,name) VALUES(10,'observer','Example observer')`, + `INSERT INTO observations(transmission_id,observer_idx,snr,rssi,timestamp,path_json) VALUES(1,10,1.5,-90,1767225600,'["ab"]')`, + } { + if _, err := db.Exec(query); err != nil { + t.Fatal(err) + } + } + return db +} + +func assertExportMetadata(t *testing.T, db *sql.DB) { + t.Helper() + path := getPathFromDB(db, 1) + if len(path) != 2 || path[0] != "ab" || path[1] != "cd" { + t.Fatalf("path query lost data: %v", path) + } + observers := getObservers(db, 1) + if len(observers) != 1 || observers[0].Name != "Example observer" || observers[0].SNR != 1.5 || observers[0].RSSI != -90 || observers[0].Timestamp != "2026-01-01T00:00:00Z" { + t.Fatalf("observer query lost data: %+v", observers) + } +} + +func TestSQLiteExportReadsPathAndObservers(t *testing.T) { + for _, current := range []bool{false, true} { + name := "legacy" + if current { + name = "current" + } + t.Run(name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "telemetry #percent%.db") + sqliteExportFixture(t, path, current) + db, err := openExportDB(path) + if err != nil { + t.Fatalf("native SQLite export should open its existing source: %v", err) + } + defer db.Close() + assertExportMetadata(t, db) + if _, err := db.Exec(`DELETE FROM observations`); err == nil { + t.Fatal("export reader permitted a write") + } + var queryOnly int + if err := db.QueryRow(`PRAGMA query_only`).Scan(&queryOnly); err != nil || queryOnly != 1 { + t.Fatalf("query_only=%d err=%v", queryOnly, err) + } + }) + } +} + +func TestSQLiteExportRejectsMissingEmptyWrongAndPendingSources(t *testing.T) { + for _, kind := range []string{"missing", "missing-directory", "empty", "header-only", "wrong-schema", "pending"} { + t.Run(kind, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "telemetry.db") + switch kind { + case "missing-directory": + path = filepath.Join(filepath.Dir(path), "absent", "telemetry.db") + case "empty": + if err := os.WriteFile(path, nil, 0600); err != nil { + t.Fatal(err) + } + case "header-only", "wrong-schema": + dsn, err := dbconfig.SQLiteURI(path, nil) + if err != nil { + t.Fatal(err) + } + db, err := sql.Open("sqlite3", dsn) + if err != nil { + t.Fatal(err) + } + query := `PRAGMA user_version=123` + if kind == "wrong-schema" { + query = `CREATE TABLE keep(value TEXT); INSERT INTO keep VALUES('preserve')` + } + if _, err := db.Exec(query); err != nil { + t.Fatal(err) + } + db.Close() + case "pending": + db := sqliteExportFixture(t, path, true) + if _, err := db.Exec(`CREATE TABLE corescope_reverse_progress(value TEXT)`); err != nil { + t.Fatal(err) + } + db.Close() + } + before, beforeErr := os.ReadFile(path) + db, err := openExportDB(path) + if err == nil { + db.Close() + t.Fatal("invalid source was accepted") + } + if kind == "pending" && !errors.Is(err, dbconfig.ErrSQLiteImportIncomplete) { + t.Fatalf("pending import guard lost: %v", err) + } + after, afterErr := os.ReadFile(path) + if errors.Is(beforeErr, os.ErrNotExist) { + if !errors.Is(afterErr, os.ErrNotExist) { + t.Fatal("export created a replacement database") + } + } else if beforeErr != nil || afterErr != nil || !bytes.Equal(before, after) { + t.Fatal("export modified its rejected source") + } + }) + } +} + +func recordExportStorage(t *testing.T, storage dbconfig.Storage) dbconfig.Selection { + t.Helper() + selected, err := dbconfig.NewSelection(storage) + if err != nil { + t.Fatal(err) + } + selected, err = dbconfig.AdoptSelection(dbconfig.SelectionPath(storage.StateDir), selected) + if err != nil { + t.Fatal(err) + } + return selected +} + +func TestExportHonorsRecordedSQLiteAndHoldsLease(t *testing.T) { + base := t.TempDir() + path := filepath.Join(base, "installed.db") + sqliteExportFixture(t, path, true) + selected := recordExportStorage(t, dbconfig.Storage{Backend: dbconfig.SQLite, StateDir: base, DBPath: path}) + db, lease, err := openSelectedExport(dbconfig.StorageInputs{BaseDir: base, StateDir: base, Backend: dbconfig.Postgres, DBPath: "stale.db"}) + if err != nil { + t.Fatal(err) + } + defer db.Close() + defer lease.Close() + assertExportMetadata(t, db) + update, err := dbconfig.BeginSelectionUpdate(dbconfig.SelectionPath(base), selected.Generation) + if err == nil { + update.Abort() + update.Close() + t.Fatal("export did not hold the selection lease") + } + if !errors.Is(err, dbconfig.ErrSelectionBusy) { + t.Fatalf("unexpected lease error: %v", err) + } + db.Close() + lease.Close() + update, err = dbconfig.BeginSelectionUpdate(dbconfig.SelectionPath(base), selected.Generation) + if err != nil { + t.Fatalf("export did not release the lease: %v", err) + } + defer update.Close() + if err := update.Abort(); err != nil { + t.Fatal(err) + } +} + +func TestRecordedSQLiteExportDoesNotFallBackOrCreate(t *testing.T) { + base := t.TempDir() + path := filepath.Join(base, "lost.db") + recordExportStorage(t, dbconfig.Storage{Backend: dbconfig.SQLite, StateDir: base, DBPath: path}) + stale := filepath.Join(base, "stale.db") + sqliteExportFixture(t, stale, true) + db, lease, err := openSelectedExport(dbconfig.StorageInputs{BaseDir: base, StateDir: base, DBPath: stale}) + if err == nil || db != nil || lease != nil { + t.Fatalf("lost recorded source did not fail closed: db=%t lease=%t err=%v", db != nil, lease != nil, err) + } + if _, err := os.Stat(path); !errors.Is(err, os.ErrNotExist) { + t.Fatal("export created the missing selected source") + } +} + +func TestExportSelectionErrorsCannotFallBack(t *testing.T) { + for _, kind := range []string{"corrupt", "switching"} { + t.Run(kind, func(t *testing.T) { + base := t.TempDir() + path := filepath.Join(base, "legacy.db") + sqliteExportFixture(t, path, true) + metadata := dbconfig.SelectionPath(base) + if kind == "switching" { + metadata += ".switch.json" + } + if err := os.WriteFile(metadata, []byte(`{}`), 0600); err != nil { + t.Fatal(err) + } + if db, lease, err := openSelectedExport(dbconfig.StorageInputs{BaseDir: base, DBPath: path}); err == nil || db != nil || lease != nil { + t.Fatal("invalid selection fell back to legacy SQLite") + } + }) + } +} + +func TestRecordedPostgresExportNeverUsesStaleSQLite(t *testing.T) { + base := t.TempDir() + stale := filepath.Join(base, "meshcore.db") + w := sqliteExportFixture(t, stale, true) + w.Close() + before, err := os.ReadFile(stale) + if err != nil { + t.Fatal(err) + } + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + endpoint := listener.Addr().String() + listener.Close() + dsn := "postgres://reader:never-print-this@" + endpoint + "/unavailable?sslmode=disable&connect_timeout=1" + recordExportStorage(t, dbconfig.Storage{Backend: dbconfig.Postgres, StateDir: base, ReaderDatabaseURL: dsn}) + for _, reader := range []string{"", dsn} { + db, lease, err := openSelectedExport(dbconfig.StorageInputs{BaseDir: base, DBPath: stale, Backend: dbconfig.SQLite, ReaderDatabaseURL: reader}) + if err == nil || db != nil || lease != nil { + t.Fatal("missing PostgreSQL selection fell back to SQLite") + } + if strings.Contains(err.Error(), "never-print-this") { + t.Fatal("connection failure exposed its password") + } + } + after, err := os.ReadFile(stale) + if err != nil || !bytes.Equal(before, after) { + t.Fatal("stale SQLite source changed") + } +} + +func TestExportConfigResolution(t *testing.T) { + base := t.TempDir() + data := filepath.Join(base, "data") + if err := os.Mkdir(data, 0700); err != nil { + t.Fatal(err) + } + path := filepath.Join(data, "meshcore.db") + sqliteExportFixture(t, path, true) + noEnv := func(string) string { return "" } + raw, err := exportStorageInputs(base, exportOptions{}, noEnv) + if err != nil { + t.Fatal(err) + } + db, lease, err := openSelectedExport(raw) + if err != nil { + t.Fatalf("default SQLite export: %v", err) + } + if lease != nil { + lease.Close() + t.Fatal("unrecorded export must not adopt a selection") + } + assertExportMetadata(t, db) + db.Close() + if _, err := os.Stat(dbconfig.SelectionPath(data)); !errors.Is(err, os.ErrNotExist) { + t.Fatal("export persisted a selection") + } + if err := os.WriteFile(filepath.Join(data, "config.json"), []byte(`{"dbPath":"data/meshcore.db"}`), 0600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(base, "config.json"), []byte(`{"db":`), 0600); err != nil { + t.Fatal(err) + } + if _, err := exportStorageInputs(base, exportOptions{}, noEnv); err == nil { + t.Fatal("malformed primary config fell back to a valid secondary config") + } + if _, err := exportStorageInputs(base, exportOptions{ConfigPath: filepath.Join(base, "missing.json")}, noEnv); err == nil { + t.Fatal("missing explicit config was ignored") + } + getenv := func(key string) string { + return map[string]string{"DB_PATH": "env.db", "CORESCOPE_DATABASE_URL": "postgres://reader@localhost/example", "CORESCOPE_READER_DATABASE_URL": "postgres://reader@localhost/reader", "CORESCOPE_STATE_DIR": "env-state"}[key] + } + raw, err = exportStorageInputs(base, exportOptions{ConfigPath: filepath.Join(data, "config.json"), DBPath: "cli.db", DatabaseURL: "postgres://reader@localhost/cli", StateDir: "cli-state"}, getenv) + if err != nil || raw.DBPath != "cli.db" || raw.DatabaseURL != "postgres://reader@localhost/cli" || raw.ReaderDatabaseURL != raw.DatabaseURL || raw.StateDir != "cli-state" { + t.Fatal("explicit flag precedence was lost") + } + if _, err := dbconfig.ResolveStorage(raw); err == nil { + t.Fatal("ambiguous unrecorded file and URL inputs were accepted") + } +} diff --git a/cmd/ingestor/README.md b/cmd/ingestor/README.md index 5e7bce73a..8655ed76e 100644 --- a/cmd/ingestor/README.md +++ b/cmd/ingestor/README.md @@ -1,25 +1,25 @@ # MeshCore MQTT Ingestor (Go) -Standalone MQTT ingestion service for CoreScope. Connects to MQTT brokers, decodes raw MeshCore packets, and writes to the same SQLite database used by the Node.js web server. +Standalone MQTT ingestion service for CoreScope. Connects to MQTT brokers, decodes raw MeshCore packets, and writes to PostgreSQL. The separate Go API server uses a restricted reader credential. -This is the first step of a larger Go rewrite — separating MQTT ingestion from the web server. +The migration command owns schema changes. The ingestor only connects to an initialized, ready telemetry schema. ## Architecture ``` -MQTT Broker(s) → Go Ingestor → SQLite DB ← Node.js Web Server +MQTT Broker(s) → Go Ingestor → PostgreSQL ← Go API Server (this binary) (shared) ``` -- **Single static binary** — no runtime dependencies, no CGO -- **SQLite** via `modernc.org/sqlite` (pure Go) +- **Single binary** — PostgreSQL runtime driver is pure Go +- **PostgreSQL 18** via pgx through `database/sql` - **MQTT** via `github.com/eclipse/paho.mqtt.golang` -- Runs **alongside** the Node.js server — they share the DB file -- Does NOT serve HTTP/WebSocket — that stays in Node.js +- Runs alongside the Go API server with separate writer and reader roles +- Does not serve HTTP/WebSocket — that stays in the Go API server ## Build -Requires Go 1.22+. +Requires Go 1.25+. ```bash cd cmd/ingestor @@ -38,16 +38,19 @@ GOOS=linux GOARCH=amd64 go build -o corescope-ingestor . ./corescope-ingestor -config /path/to/config.json ``` -The config file uses the same format as the Node.js `config.json`. The ingestor reads the `mqttSources` array (or legacy `mqtt` object) and `dbPath` fields. +The ingestor reads `mqttSources` (or the legacy `mqtt` object), `databaseURL` and `stateDir`. `-database-url` and `-state-dir` override configuration. `dbPath` and `-db` are never interpreted as database URLs: import existing SQLite snapshots offline first. Do not place connection URLs in command logs. ### Environment Variables | Variable | Description | Default | |----------|-------------|---------| -| `DB_PATH` | SQLite database path | `data/meshcore.db` | +| `CORESCOPE_WRITER_DATABASE_URL` | Ingestor-specific PostgreSQL writer URL; overrides the generic URL | required when no generic/config URL exists | +| `CORESCOPE_DATABASE_URL` | Generic PostgreSQL URL for standalone runs | config `databaseURL` | +| `CORESCOPE_STATE_DIR` | Local queue/statistics directory shared with the API server | `data` | +| `CORESCOPE_APPROVED_CHANNELS_DATABASE_URL` | Separate restricted account reader for `approved_channels` view | config `userManagement.approvedChannelsDatabaseURL` | | `MQTT_BROKER` | Single MQTT broker URL (overrides config) | — | | `MQTT_TOPIC` | MQTT topic (used with `MQTT_BROKER`) | `meshcore/#` | -| `CORESCOPE_INGESTOR_STATS` | Path to the per-second stats JSON file consumed by the server's `/api/perf/io` and `/api/perf/write-sources` endpoints (#1120) | `/tmp/corescope-ingestor-stats.json` | +| `CORESCOPE_INGESTOR_STATS` | Path to the per-second stats JSON file consumed by the server's `/api/perf/io` and `/api/perf/write-sources` endpoints (#1120) | `/ingestor-stats.json` | ### Stats file (`CORESCOPE_INGESTOR_STATS`) @@ -60,17 +63,15 @@ the Perf page so operators can self-diagnose write-volume anomalies. The writer uses `O_NOFOLLOW | O_CREAT | O_TRUNC` mode `0o600`, so a pre-planted symlink at the path cannot be used to clobber an arbitrary file. -**Security note:** the default lives in `/tmp`, which is world-writable on -most hosts (sticky bit only protects deletion, not creation). On -shared/multi-tenant hosts, override `CORESCOPE_INGESTOR_STATS` to point at a -private directory (e.g. `/var/lib/corescope/ingestor-stats.json`) that only -the corescope user can write to. +The state directory is created privately and stores small queue/statistics files. +Database URLs never become filesystem paths. ### Minimal Config ```json { - "dbPath": "data/meshcore.db", + "databaseURL": "postgres://telemetry_writer@localhost/telemetry", + "stateDir": "data", "mqttSources": [ { "name": "local", @@ -81,7 +82,7 @@ the corescope user can write to. } ``` -### Full Config (same as Node.js) +### Full Config The ingestor reads these fields from the existing `config.json`: @@ -91,14 +92,16 @@ The ingestor reads these fields from the existing `config.json`: - `username` / `password` — auth credentials - `topics` — array of topic patterns to subscribe - `iataFilter` — optional regional filter + - `clientId`: optional MQTT ClientID. Default `corescope--`, new suffix per ingestor start. Two running ingestors must not share a value on one broker - `mqtt` — legacy single-broker config (auto-converted to `mqttSources`) -- `dbPath` — SQLite DB path (default: `data/meshcore.db`) +- `databaseURL` — restricted telemetry writer URL; schema-owner credentials are refused +- `stateDir` — local queue/statistics directory (default: `data`) ## Test ```bash cd cmd/ingestor -go test -v ./... +CORESCOPE_TEST_POSTGRES_URL=postgres://test_admin@localhost/postgres go test -v ./... ``` ## What It Does @@ -108,7 +111,7 @@ go test -v ./... 3. Receives raw hex packets via JSON messages (`{ "raw": "...", "SNR": ..., "RSSI": ... }`) 4. Decodes MeshCore packet headers, paths, and payloads (ported from `decoder.js`) 5. Computes content hashes (path-independent, SHA-256-based) -6. Writes to SQLite: `transmissions` + `observations` tables +6. Writes atomically to PostgreSQL: `transmissions` + `observations` tables 7. Upserts `nodes` from decoded ADVERT packets (with validation) 8. Upserts `observers` from MQTT topic metadata @@ -146,3 +149,10 @@ cmd/ingestor/ util.go — shared utilities go.mod / go.sum — Go module definition ``` + +PostgreSQL autovacuum, automatic analyze and checkpoints are managed by the +PostgreSQL service, not by runtime owner privileges. The previous SQLite +`db.vacuumOnStartup`, `db.incrementalVacuumPages` and `db.analysisLimit` +settings have no PostgreSQL effect. The migration/restore procedure must run +`ANALYZE` after bulk imports; runtime credentials cannot alter planner targets. +Packet retention remains bounded to 250 transmissions per transaction. diff --git a/cmd/ingestor/advert_evidence_migration_race_test.go b/cmd/ingestor/advert_evidence_migration_race_test.go new file mode 100644 index 000000000..2eb6f37b1 --- /dev/null +++ b/cmd/ingestor/advert_evidence_migration_race_test.go @@ -0,0 +1,237 @@ +package main + +import ( + "context" + "fmt" + "path/filepath" + "strings" + "testing" +) + +func seedUnbackfilledAdvert(t *testing.T, canonical, observed string) (*Store, string) { + t.Helper() + path := filepath.Join(t.TempDir(), "legacy-advert.db") + s, err := openPostgresTestStore(t, path) + if err != nil { + t.Fatal(err) + } + if err := s.UpsertObserver("fixture-observer", "Fixture observer", "", nil); err != nil { + t.Fatal(err) + } + if _, err := s.db.Exec(`INSERT INTO transmissions(id,hash,raw_hex,first_seen,payload_type,route_type) VALUES(1,'legacy-advert',$1,'2026-01-01T00:00:00Z',4,$2)`, canonical, int(canonical[1]-'0')&3); err != nil { + s.Close() + t.Fatal(err) + } + if _, err := s.db.Exec(`INSERT INTO observations(id,transmission_id,observer_idx,raw_hex,path_json,timestamp) VALUES(1,1,(SELECT rowid FROM observers WHERE id='fixture-observer'),$1,'[]',1)`, observed); err != nil { + t.Fatal(err) + } + if _, err := testAdmin(t, s).Exec(testNativeSQL(`UPDATE sqlite_sequence SET seq=1 WHERE name='observations'`, `SELECT setval('observations_id_seq',1,true)`)); err != nil { + t.Fatal(err) + } + + return s, path +} + +func TestAdvertRouteEvidenceLegacyProtectionErrorsDoNotDropIncomingRaw(t *testing.T) { + for _, failure := range []string{"lookup", "write"} { + t.Run(failure, func(t *testing.T) { + s, _ := seedUnbackfilledAdvert(t, "1100aa", "1200aa") + defer s.Close() + stmt := `DROP TABLE advert_evidence_backfill` + if failure == "write" { + stmt = testNativeSQL(`CREATE TRIGGER fail_old_evidence BEFORE INSERT ON advert_route_evidence WHEN NEW.bit=2 BEGIN SELECT RAISE(ABORT,'fixture old evidence failure'); END`, `CREATE OR REPLACE FUNCTION fail_old_evidence_fn() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.bit=2 THEN RAISE EXCEPTION 'fixture old evidence failure'; END IF; RETURN NEW; END $$; CREATE TRIGGER fail_old_evidence BEFORE INSERT ON advert_route_evidence FOR EACH ROW EXECUTE FUNCTION fail_old_evidence_fn()`) + } + if _, err := testAdmin(t, s).Exec(stmt); err != nil { + t.Fatal(err) + } + _, err := s.InsertTransmission(&PacketData{Hash: "legacy-advert", ObserverID: "fixture-observer", PayloadType: 4, RouteType: 1, RawHex: "1100aa", PathJSON: "[]"}) + if err != nil { + t.Errorf("analytics preservation failure dropped incoming frame: %v", err) + } + var old string + if err := s.db.QueryRow(`SELECT raw_hex FROM observations WHERE id=1`).Scan(&old); err != nil { + t.Fatal(err) + } + if old != "1100aa" { + t.Fatalf("failed analytics preservation prevented incoming raw: %q", old) + } + }) + } +} + +func TestAdvertRouteEvidenceLegacyCoalescedPathAndIndex(t *testing.T) { + s, _ := seedUnbackfilledAdvert(t, "1100aa", "1200aa") + defer s.Close() + if _, err := s.db.Exec(`UPDATE observations SET path_json=NULL WHERE id=1`); err != nil { + t.Fatal(err) + } + if _, err := s.InsertTransmission(&PacketData{Hash: "legacy-advert", ObserverID: "fixture-observer", PayloadType: 4, RouteType: 1, RawHex: "1100aa", PathJSON: ""}); err != nil { + t.Fatal(err) + } + var count, mask int + if err := s.db.QueryRow(`SELECT COUNT(*) FROM observations`).Scan(&count); err != nil { + t.Fatal(err) + } + if count != 1 { + t.Fatalf("NULL and empty path must conflict, got %d observations", count) + } + if err := s.db.QueryRow(`SELECT SUM(bit) FROM advert_route_evidence`).Scan(&mask); err != nil { + t.Fatal(err) + } + if mask != 3 { + t.Fatalf("coalesced path lookup lost old evidence: mask=%d", mask) + } + if testBackendValue() == "postgres" { + if _, err := s.db.Exec(`SET enable_seqscan=off`); err != nil { + t.Fatal(err) + } + } + rows, err := s.db.Query(testNativeSQL(`EXPLAIN QUERY PLAN `, `EXPLAIN (COSTS OFF) `)+legacyAdvertObservationSQL, 1, 1, "") + if err != nil { + t.Fatal(err) + } + defer rows.Close() + indexed := false + for rows.Next() { + var detail string + if err := scanTestPlan(rows, &detail); err != nil { + t.Fatal(err) + } + if strings.Contains(detail, "idx_observations_dedup") { + indexed = true + } + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + if !indexed { + t.Fatal("legacy preservation must use the unique observation conflict index") + } +} + +func TestAdvertRouteEvidenceCompletionSkipsLookupAfterRestart(t *testing.T) { + s, path := seedUnbackfilledAdvert(t, "1100aa", "1200aa") + defer func() { s.Close() }() + if err := s.RunAsyncMigration(context.Background(), "advert_route_evidence_v1", s.backfillAdvertEvidence); err != nil { + t.Fatal(err) + } + s.WaitForAsyncMigrations() + if !s.advertEvidenceComplete.Load() { + t.Fatal("backfill did not close preservation window") + } + s.Close() + var err error + s, err = openPostgresTestStore(t, path) + if err != nil { + t.Fatal(err) + } + if !s.advertEvidenceComplete.Load() { + t.Fatal("restart did not restore persisted migration completion") + } + // Any accidental legacy lookup now fails. Completed stores must incur + // no extra SQL on this steady-state overwrite path. + if _, err := testAdmin(t, s).Exec(`DROP TABLE advert_evidence_backfill`); err != nil { + t.Fatal(err) + } + if _, err := s.InsertTransmission(&PacketData{Hash: "legacy-advert", ObserverID: "fixture-observer", PayloadType: 4, RouteType: 1, RawHex: "1100aa", PathJSON: "[]"}); err != nil { + t.Fatal(err) + } +} + +func BenchmarkAdvertEvidenceLegacyPreservation(b *testing.B) { + s, err := openPostgresTestStore(b, filepath.Join(b.TempDir(), "legacy-preservation.db")) + if err != nil { + b.Fatal(err) + } + defer s.Close() + s.WaitForAsyncMigrations() + if err := s.UpsertObserver("fixture-observer", "Fixture observer", "", nil); err != nil { + b.Fatal(err) + } + if _, err := s.InsertTransmission(&PacketData{Hash: "bench-legacy", ObserverID: "fixture-observer", PayloadType: 4, RouteType: 1, RawHex: "1100aa", PathJSON: "[]"}); err != nil { + b.Fatal(err) + } + var observerIdx int64 + if err := s.db.QueryRow(`SELECT rowid FROM observers WHERE id='fixture-observer'`).Scan(&observerIdx); err != nil { + b.Fatal(err) + } + for _, state := range []string{"pending", "checkpointed", "complete"} { + b.Run(state, func(b *testing.B) { + cursor := 0 + if state != "pending" { + cursor = 1 + } + if _, err := s.db.Exec(`INSERT INTO advert_evidence_backfill(id,obs_cursor) VALUES(1,$1) ON CONFLICT(id) DO UPDATE SET obs_cursor=excluded.obs_cursor`, cursor); err != nil { + b.Fatal(err) + } + s.advertEvidenceComplete.Store(state == "complete") + writerMu.Lock() + defer writerMu.Unlock() + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if err := func() error { + tx, e := beginWrite(s.db, s.Backend()) + if e != nil { + return e + } + defer tx.Rollback() + if e = s.preserveLegacyAdvertObservation(tx, 1, observerIdx, "[]"); e != nil { + return e + } + return tx.Commit() + }(); err != nil { + b.Fatal(err) + } + } + }) + } +} + +func TestAdvertRouteEvidencePreservesLegacyConflictBeforeBackfill(t *testing.T) { + for _, tc := range []struct{ name, canonical, oldRaw, incoming string }{ + {"flood_zero_then_flood", "1100aa", "1200aa", "1100aa"}, + {"zero_flood_then_zero", "1200aa", "1100aa", "1200aa"}, + {"flood_zero_then_malformed", "1100aa", "1200aa", "11zzaa"}, + {"zero_flood_then_malformed", "1200aa", "1100aa", "12zzaa"}, + } { + for _, restart := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/restart=%v", tc.name, restart), func(t *testing.T) { + s, path := seedUnbackfilledAdvert(t, tc.canonical, tc.oldRaw) + defer func() { s.Close() }() + // Do not replay oldRaw: it only exists in the legacy row and + // this one incoming frame will replace it before backfill. + data := &PacketData{Hash: "legacy-advert", ObserverID: "fixture-observer", PayloadType: 4, RouteType: int(tc.canonical[1]-'0') & 3, RawHex: tc.incoming, PathJSON: "[]"} + if _, err := s.InsertTransmission(data); err != nil { + t.Fatal(err) + } + var count, id int + var surviving string + if err := s.db.QueryRow(`SELECT COUNT(*),MIN(id),MIN(raw_hex) FROM observations`).Scan(&count, &id, &surviving); err != nil { + t.Fatal(err) + } + if count != 1 || id != 1 || surviving != tc.incoming { + t.Fatalf("fixture did not replace same observation: count=%d id=%d raw=%s", count, id, surviving) + } + if restart { + s.Close() + var err error + s, err = openPostgresTestStore(t, path) + if err != nil { + t.Fatal(err) + } + } + if err := s.backfillAdvertEvidence(context.Background(), s.db); err != nil { + t.Fatal(err) + } + var mask int + if err := s.db.QueryRow(`SELECT COUNT(*),COALESCE(SUM(bit),0) FROM advert_route_evidence WHERE tx_id=1`).Scan(&count, &mask); err != nil { + t.Fatal(err) + } + if count != 2 || mask != 3 { + t.Fatalf("upgrade overwrote available legacy route evidence: rows=%d mask=%d, want 2/3", count, mask) + } + }) + } + } +} diff --git a/cmd/ingestor/advert_evidence_scale_test.go b/cmd/ingestor/advert_evidence_scale_test.go new file mode 100644 index 000000000..b6aabc767 --- /dev/null +++ b/cmd/ingestor/advert_evidence_scale_test.go @@ -0,0 +1,120 @@ +//go:build advert_evidence_scale + +package main + +import ( + "context" + "os" + "path/filepath" + "sort" + "strings" + "testing" + "time" +) + +// Opt-in disk benchmark: a new database with 1M transmissions and 11M +// observations (10% adverts, 11 observations per transmission, 120-byte frames). +// ADVERT_SCALE_DB retains the fixture for the server catch-up benchmark. +func TestAdvertBackfillScale(t *testing.T) { + path := os.Getenv("ADVERT_SCALE_DB") + if path == "" { + path = filepath.Join(t.TempDir(), "advert-scale.db") + } + if _, err := os.Stat(path); err == nil { + t.Fatal("ADVERT_SCALE_DB must be a new temporary database") + } else if !os.IsNotExist(err) { + t.Fatal(err) + } + s, err := openPostgresTestStore(t, path) + if err != nil { + t.Fatal(err) + } + defer s.Close() + s.WaitForAsyncMigrations() + var existing int + if err := s.db.QueryRow(`SELECT COUNT(*) FROM observations`).Scan(&existing); err != nil { + t.Fatal(err) + } + if existing != 0 { + t.Fatal("scale benchmark requires a new empty database") + } + started := time.Now() + { + raw := "1100" + strings.Repeat("ab", 118) + for start := 1; start <= 1000000; start += 10000 { + _, err = s.db.Exec(`WITH RECURSIVE ids(n) AS (VALUES($1) UNION ALL SELECT n+1 FROM ids WHERE n<$2) + INSERT INTO transmissions(id,hash,raw_hex,first_seen,payload_type,route_type,last_seen) + SELECT n,'scale-'||n,$3,'2026-10-01T00:00:00Z',CASE WHEN n%10=0 THEN 4 ELSE 3 END,1,1790812800 FROM ids`, start, start+9999, raw) + if err != nil { + t.Fatal(err) + } + } + for start := 1; start <= 11000000; start += 10000 { + _, err = s.db.Exec(`WITH RECURSIVE ids(n) AS (VALUES($1) UNION ALL SELECT n+1 FROM ids WHERE n<$2) + INSERT INTO observations(id,transmission_id,raw_hex,path_json,timestamp) + SELECT n,((n-1)%1000000)+1,CASE WHEN ((n-1)/1000000)%2=0 THEN $3 ELSE $4 END,'[]',1790812800 FROM ids`, start, start+9999, raw, "12"+raw[2:]) + if err != nil { + t.Fatal(err) + } + if start%1000000 == 1 { + t.Logf("seeded observations=%d elapsed=%s", start+9999, time.Since(started)) + } + } + t.Logf("seed complete elapsed=%s", time.Since(started)) + } + if _, err := s.db.Exec(`DELETE FROM advert_route_evidence;DELETE FROM advert_evidence_backfill`); err != nil { + t.Fatal(err) + } + s.advertEvidenceComplete.Store(false) + stop := make(chan struct{}) + done := make(chan struct{}) + var latencies []time.Duration + var failures int + go func() { + defer close(done) + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + for { + select { + case <-stop: + return + case <-ticker.C: + at := time.Now() + _, err := s.InsertTransmission(&PacketData{Hash: "scale-live", RawHex: "1100aa", PayloadType: 4, RouteType: 1, PathJSON: "[]", Timestamp: time.Now().UTC().Format(time.RFC3339)}) + latencies = append(latencies, time.Since(at)) + if err != nil { + failures++ + } + } + } + }() + started = time.Now() + err = s.backfillAdvertEvidence(context.Background(), s.db) + elapsed := time.Since(started) + close(stop) + <-done + if err != nil { + t.Fatal(err) + } + if len(latencies) == 0 { + t.Fatal("live writer produced no samples") + } + sort.Slice(latencies, func(i, j int) bool { return latencies[i] < latencies[j] }) + var rows, txCursor, obsCursor, liveObservations int + if err := s.db.QueryRow(`SELECT COUNT(*) FROM advert_route_evidence`).Scan(&rows); err != nil { + t.Fatal(err) + } + if err := s.db.QueryRow(`SELECT tx_cursor,obs_cursor FROM advert_evidence_backfill`).Scan(&txCursor, &obsCursor); err != nil { + t.Fatal(err) + } + if err := s.db.QueryRow(`SELECT COUNT(*) FROM observations WHERE transmission_id=(SELECT id FROM transmissions WHERE hash='scale-live')`).Scan(&liveObservations); err != nil { + t.Fatal(err) + } + if liveObservations != len(latencies) || s.Stats.WriteErrors.Load() != 0 { + t.Fatalf("live write loss: observations=%d samples=%d write_errors=%d", liveObservations, len(latencies), s.Stats.WriteErrors.Load()) + } + t.Logf("BACKFILL transmissions=1000000 observations=11000000 advert_fraction=10%% frame_bytes=120 elapsed=%s evidence_rows=%d cursors=%d/%d live_samples=%d failures=%d p50=%s p95=%s p99=%s max=%s", elapsed, rows, txCursor, obsCursor, len(latencies), failures, latencies[len(latencies)/2], latencies[len(latencies)*95/100], latencies[len(latencies)*99/100], latencies[len(latencies)-1]) + if rows != 200001 || failures != 0 { + t.Fatalf("incomplete result: evidence_rows=%d failures=%d", rows, failures) + } +} diff --git a/cmd/ingestor/advert_route_evidence.go b/cmd/ingestor/advert_route_evidence.go new file mode 100644 index 000000000..16eb6438c --- /dev/null +++ b/cmd/ingestor/advert_route_evidence.go @@ -0,0 +1,136 @@ +package main + +import ( + "context" + "database/sql" + "fmt" + "time" + + "github.com/meshcore-analyzer/packetpath" +) + +// The indexed existence probe avoids consuming an identity on duplicates. +const insertAdvertEvidenceSQL = `INSERT INTO advert_route_evidence(tx_id,bit) + SELECT $1,$2 WHERE NOT EXISTS (SELECT 1 FROM advert_route_evidence WHERE tx_id=$3 AND bit=$4)` + +// Match the UPSERT's exact expression-index key. NULL observer_idx cannot +// conflict, so the caller skips it. Only an uncheckpointed old row needs work. +const legacyAdvertObservationSQL = `SELECT raw_hex FROM observations + WHERE transmission_id=$1 AND observer_idx=$2 AND COALESCE(path_json,'')=$3 + AND id > COALESCE((SELECT obs_cursor FROM advert_evidence_backfill WHERE id=1),0)` + +// Caller holds writerMu, including through the subsequent observation UPSERT. +// The backfill commits evidence and obs_cursor together under that same lock. +func (s *Store) preserveLegacyAdvertObservation(tx *sql.Tx, txID, observerIdx int64, path string) error { + if s.advertEvidenceComplete.Load() { + return nil + } + var raw sql.NullString + err := tx.Stmt(s.stmtGetLegacyAdvertObservation).QueryRow(txID, observerIdx, path).Scan(&raw) + if err == sql.ErrNoRows { + return nil + } + if err != nil { + return err + } + if bit := packetpath.AdvertRouteEvidence(raw.String); bit != 0 { + _, err = tx.Stmt(s.stmtInsertAdvertEvidence).Exec(txID, bit, txID, bit) + } + return err +} + +// backfillAdvertEvidence recovers only evidence still present in canonical and +// observation frames. An overwritten middle frame is unrecoverable. Persisted +// cursors and 500-row transactions make this cancellable/resumable; new traffic +// records evidence synchronously, so rows beyond either scan need no replay. +func (s *Store) backfillAdvertEvidence(ctx context.Context, db *sql.DB) error { + if _, err := db.ExecContext(ctx, `INSERT INTO advert_evidence_backfill(id) VALUES(1) ON CONFLICT DO NOTHING`); err != nil { + return err + } + for _, source := range []struct{ cursor, table, query string }{ + {"tx_cursor", "transmissions", `SELECT id,id,COALESCE(raw_hex,''),payload_type FROM transmissions WHERE id>` + s.parameter(1) + ` AND id<=` + s.parameter(2) + ` ORDER BY id LIMIT 500`}, + {"obs_cursor", "observations", `SELECT o.id,o.transmission_id,COALESCE(o.raw_hex,''),t.payload_type FROM observations o JOIN transmissions t ON t.id=o.transmission_id WHERE o.id>` + s.parameter(1) + ` AND o.id<=` + s.parameter(2) + ` ORDER BY o.id LIMIT 500`}, + } { + // A finite horizon prevents live traffic from extending this scan. + var upper int64 + if err := db.QueryRowContext(ctx, `SELECT COALESCE(MAX(id),0) FROM `+source.table).Scan(&upper); err != nil { + return err + } + for { + if err := ctx.Err(); err != nil { + return err + } + var cursor int64 + if err := db.QueryRowContext(ctx, `SELECT `+source.cursor+` FROM advert_evidence_backfill WHERE id=1`).Scan(&cursor); err != nil { + return err + } + rows, err := db.QueryContext(ctx, source.query, cursor, upper) + if err != nil { + return err + } + type evidence struct { + txID int64 + bit uint8 + } + batch := make([]evidence, 0, 500) + lastID := cursor + for rows.Next() { + var id, txID int64 + var raw string + var payloadType sql.NullInt64 + if err := rows.Scan(&id, &txID, &raw, &payloadType); err != nil { + rows.Close() + return err + } + lastID = id + if bit := packetpath.AdvertRouteEvidence(raw); bit != 0 && payloadType.Valid && payloadType.Int64 == 4 { + batch = append(batch, evidence{txID, bit}) + } + } + err = rows.Err() + rows.Close() + if err != nil { + return err + } + if lastID == cursor { + break + } + // The read cursor is closed before the single writer is acquired. + // Retention may have removed a transmission meanwhile; do not + // resurrect evidence for it or violate its foreign key. + err = func() error { + writerMu.Lock() + defer writerMu.Unlock() + tx, err := beginWriteContext(ctx, db, s.Backend()) + if err != nil { + return err + } + defer tx.Rollback() + for _, item := range batch { + if _, err := tx.ExecContext(ctx, insertAdvertEvidenceSQL+` AND EXISTS(SELECT 1 FROM transmissions WHERE id=`+s.parameter(5)+`)`, item.txID, item.bit, item.txID, item.bit, item.txID); err != nil { + return err + } + } + if _, err := tx.ExecContext(ctx, `UPDATE advert_evidence_backfill SET `+source.cursor+`=`+s.parameter(1)+` WHERE id=1`, lastID); err != nil { + return err + } + return tx.Commit() + }() + if err != nil { + return fmt.Errorf("advert evidence backfill: %w", err) + } + // Yield between bounded batches so live ingestion gets the writer. + timer := time.NewTimer(10 * time.Millisecond) + select { + case <-ctx.Done(): + timer.Stop() + return ctx.Err() + case <-timer.C: + } + } + } + // RunAsyncMigration persists "done" after this returns. A crash before + // that status write simply re-enables protection and resumes the cursors. + s.advertEvidenceComplete.Store(true) + return nil +} diff --git a/cmd/ingestor/advert_route_evidence_test.go b/cmd/ingestor/advert_route_evidence_test.go new file mode 100644 index 000000000..10494fa8a --- /dev/null +++ b/cmd/ingestor/advert_route_evidence_test.go @@ -0,0 +1,374 @@ +package main + +import ( + "context" + "errors" + "fmt" + "path/filepath" + "strings" + "testing" +) + +// These frames have the same advert payload. Firmware hashes payload/type, +// excluding the route header and path, so they belong to one transmission. +func TestAdvertRouteEvidenceSurvivesObservationUpsert(t *testing.T) { + for _, sequence := range []struct { + name string + raws []string + }{ + {"flood_zero", []string{"1100aa", "1200aa"}}, + {"zero_flood", []string{"1200aa", "1100aa"}}, + {"flood_zero_flood", []string{"1100aa", "1200aa", "1100aa"}}, + {"zero_flood_zero", []string{"1200aa", "1100aa", "1200aa"}}, + {"flood_empty_2byte_flood", []string{"1100aa", "1240aa", "1100aa"}}, + {"empty_3byte_flood_empty", []string{"1280aa", "1100aa", "1280aa"}}, + {"transport_empty_2byte_flood", []string{"130102030440aa", "100102030400aa"}}, + {"transport_flood_empty_3byte", []string{"100102030400aa", "130102030480aa"}}, + {"transport_flood_zero_flood", []string{"100102030400aa", "130102030400aa", "100102030400aa"}}, + {"transport_zero_flood_zero", []string{"130102030400aa", "100102030400aa", "130102030400aa"}}, + } { + t.Run(sequence.name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "evidence.db") + s, err := openPostgresTestStore(t, path) + if err != nil { + t.Fatal(err) + } + defer func() { s.Close() }() + if err := s.UpsertObserver("test-observer", "Fixture observer", "", nil); err != nil { + t.Fatal(err) + } + data := &PacketData{Hash: "advert-evidence", PayloadType: 4, ObserverID: "test-observer", PathJSON: "[]", DecodedJSON: `{"type":"ADVERT"}`} + for i, raw := range sequence.raws { + data.RawHex = raw + data.RouteType = int(raw[1]-'0') & 3 + // Same second, then an older receive-time: timestamps cannot be + // used as a reliable cursor for evidence changes. + data.Timestamp = "2026-01-02T00:00:00Z" + if i == 2 { + data.Timestamp = "2026-01-01T00:00:00Z" + } + if _, err := s.InsertTransmission(data); err != nil { + t.Fatal(err) + } + } + var count, obsID int + var canonical, surviving string + if err := s.db.QueryRow(`SELECT COUNT(*), MIN(id), MIN(raw_hex) FROM observations`).Scan(&count, &obsID, &surviving); err != nil { + t.Fatal(err) + } + if count != 1 || obsID != 1 || surviving != sequence.raws[len(sequence.raws)-1] { + t.Fatalf("fixture must overwrite one observation in place: count=%d id=%d raw=%s", count, obsID, surviving) + } + if err := s.db.QueryRow(`SELECT raw_hex FROM transmissions`).Scan(&canonical); err != nil { + t.Fatal(err) + } + if canonical != sequence.raws[0] { + t.Fatalf("canonical raw changed: %s", canonical) + } + if err := s.db.QueryRow(testNativeSQL(`SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='advert_route_evidence'`, `SELECT COUNT(*) FROM information_schema.tables WHERE table_schema=current_schema() AND table_name='advert_route_evidence'`)).Scan(&count); err != nil { + t.Fatal(err) + } + if count != 1 { + t.Fatal("missing durable advert route evidence: expected both route families to survive same-row observation replacement") + } + assertMixed := func() int64 { + t.Helper() + var rows, mask int + var maxID int64 + if err := s.db.QueryRow(`SELECT COUNT(*), COALESCE(SUM(bit),0), COALESCE(MAX(id),0) FROM advert_route_evidence`).Scan(&rows, &mask, &maxID); err != nil { + t.Fatal(err) + } + if rows != 2 || mask != 3 { + t.Fatalf("durable evidence rows=%d mask=%d, want exactly two rows and mixed mask=3", rows, mask) + } + return maxID + } + maxID := assertMixed() + for i := 0; i < 100; i++ { + data.RawHex = sequence.raws[i%len(sequence.raws)] + data.RouteType = int(data.RawHex[1]-'0') & 3 + if _, err := s.InsertTransmission(data); err != nil { + t.Fatal(err) + } + } + if assertMixed() != maxID { + t.Fatal("duplicate traffic appended route evidence") + } + var sequenceID int64 + if err := s.db.QueryRow(testNativeSQL(`SELECT seq FROM sqlite_sequence WHERE name='advert_route_evidence'`, `SELECT last_value FROM advert_route_evidence_id_seq`)).Scan(&sequenceID); err != nil { + t.Fatal(err) + } + if sequenceID != maxID { + t.Fatalf("duplicate traffic advanced evidence sequence: %d, want %d", sequenceID, maxID) + } + s.Close() + s, err = openPostgresTestStore(t, path) + if err != nil { + t.Fatal(err) + } + assertMixed() + // The evidence lifetime is bounded by the retained transmission. + if _, err := s.db.Exec(`DELETE FROM observations; DELETE FROM transmissions`); err != nil { + t.Fatal(err) + } + if err := s.db.QueryRow(`SELECT COUNT(*) FROM advert_route_evidence`).Scan(&count); err != nil { + t.Fatal(err) + } + if count != 0 { + t.Fatalf("retention left %d orphan evidence rows", count) + } + }) + } +} + +func TestAdvertRouteEvidenceBackfillResumeAndLiveUnion(t *testing.T) { + s, err := openPostgresTestStore(t, filepath.Join(t.TempDir(), "backfill.db")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + s.WaitForAsyncMigrations() + if err := s.UpsertObserver("fixture-observer", "Fixture observer", "", nil); err != nil { + t.Fatal(err) + } + // Simulate pre-upgrade history without going through the new writer. + tx, err := s.db.Begin() + if err != nil { + t.Fatal(err) + } + for id := 1; id <= 1200; id++ { + if _, err := tx.Exec(`INSERT INTO transmissions(id,hash,raw_hex,first_seen,payload_type,route_type) VALUES($1,$2, '1100aa','2026-01-01T00:00:00Z',4,1)`, id, fmt.Sprintf("history-%d", id)); err != nil { + t.Fatal(err) + } + if _, err := tx.Exec(`INSERT INTO observations(transmission_id,observer_idx,raw_hex,path_json,timestamp) VALUES($1,(SELECT rowid FROM observers WHERE id='fixture-observer'),'1200aa','[]',1)`, id); err != nil { + t.Fatal(err) + } + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if err := s.backfillAdvertEvidence(ctx, s.db); !errors.Is(err, context.Canceled) { + t.Fatalf("cancelled backfill returned %v", err) + } + // Abort after one committed batch; the failed batch must not move its + // persisted cursor, so a retry can recover every remaining frame. + if _, err := testAdmin(t, s).Exec(testNativeSQL(`CREATE TRIGGER fail_evidence_batch BEFORE INSERT ON advert_route_evidence WHEN NEW.tx_id=501 BEGIN SELECT RAISE(ABORT,'fixture failure'); END`, `CREATE OR REPLACE FUNCTION fail_evidence_batch_fn() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.tx_id=501 THEN RAISE EXCEPTION 'fixture failure'; END IF; RETURN NEW; END $$; CREATE TRIGGER fail_evidence_batch BEFORE INSERT ON advert_route_evidence FOR EACH ROW EXECUTE FUNCTION fail_evidence_batch_fn()`)); err != nil { + t.Fatal(err) + } + if err := s.backfillAdvertEvidence(context.Background(), s.db); err == nil { + t.Fatal("expected injected batch failure") + } + var cursor, count int + if err := s.db.QueryRow(`SELECT tx_cursor FROM advert_evidence_backfill WHERE id=1`).Scan(&cursor); err != nil { + t.Fatal(err) + } + if cursor != 500 { + t.Fatalf("cursor=%d after failure, want committed batch boundary 500", cursor) + } + if _, err := testAdmin(t, s).Exec(testNativeSQL(`DROP TRIGGER fail_evidence_batch`, `DROP TRIGGER fail_evidence_batch ON advert_route_evidence`)); err != nil { + t.Fatal(err) + } + done := make(chan error, 1) + go func() { done <- s.backfillAdvertEvidence(context.Background(), s.db) }() + // Live processing can overwrite the only surviving zero-hop raw before + // the backfill reaches it; its synchronous evidence must preserve it. + data := &PacketData{Hash: "history-1200", ObserverID: "fixture-observer", PayloadType: 4, RouteType: 1, RawHex: "1100aa", Timestamp: "2026-01-01T00:00:00Z", PathJSON: "[]"} + if _, err := s.InsertTransmission(data); err != nil { + t.Fatal(err) + } + if err := <-done; err != nil { + t.Fatal(err) + } + if err := s.db.QueryRow(`SELECT COUNT(*) FROM advert_route_evidence`).Scan(&count); err != nil { + t.Fatal(err) + } + if count != 2400 { + t.Fatalf("got %d evidence rows, want two per history transmission", count) + } + var before, after int64 + if err := s.db.QueryRow(testNativeSQL(`SELECT seq FROM sqlite_sequence WHERE name='advert_route_evidence'`, `SELECT last_value FROM advert_route_evidence_id_seq`)).Scan(&before); err != nil { + t.Fatal(err) + } + if err := s.backfillAdvertEvidence(context.Background(), s.db); err != nil { + t.Fatal(err) + } + if err := s.db.QueryRow(testNativeSQL(`SELECT seq FROM sqlite_sequence WHERE name='advert_route_evidence'`, `SELECT last_value FROM advert_route_evidence_id_seq`)).Scan(&after); err != nil { + t.Fatal(err) + } + if before != after { + t.Fatalf("idempotent backfill changed evidence sequence %d -> %d", before, after) + } +} + +// Analytics is best effort: failure must not prevent the core observation, +// resolved relay path, relay liveness, or transmission timestamp from landing. +func TestAdvertRouteEvidenceFailureKeepsCoreIngestion(t *testing.T) { + for _, failure := range []string{"incoming-write", "legacy-read", "legacy-write"} { + t.Run(failure, func(t *testing.T) { + s, err := openPostgresTestStore(t, filepath.Join(t.TempDir(), "write-failure.db")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + const relay = "bbbbbbbbbb" + seedRelayNode(t, s, relay, "Fixture relay", "2026-01-01T00:00:00Z") + if err := s.RefreshPrefixIndex(); err != nil { + t.Fatal(err) + } + if err := s.UpsertObserver("fixture-observer", "Fixture observer", "", nil); err != nil { + t.Fatal(err) + } + data := &PacketData{Hash: "write-failure", PayloadType: 4, RouteType: 1, RawHex: "1101bbaa", PathJSON: `["bb"]`, ObserverID: "fixture-observer", Timestamp: "2026-01-01T00:00:00Z"} + if _, err := s.InsertTransmission(data); err != nil { + t.Fatal(err) + } + if _, err := s.db.Exec(`DELETE FROM advert_route_evidence; UPDATE observations SET raw_hex='1200aa',resolved_path=NULL`); err != nil { + t.Fatal(err) + } + stmt := testNativeSQL(`CREATE TRIGGER fail_evidence BEFORE INSERT ON advert_route_evidence WHEN NEW.bit=1 BEGIN SELECT RAISE(ABORT,'fixture evidence failure'); END`, `CREATE OR REPLACE FUNCTION fail_evidence_fn() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.bit=1 THEN RAISE EXCEPTION 'fixture evidence failure'; END IF; RETURN NEW; END $$; CREATE TRIGGER fail_evidence BEFORE INSERT ON advert_route_evidence FOR EACH ROW EXECUTE FUNCTION fail_evidence_fn()`) + if failure == "legacy-read" { + stmt = `DROP TABLE advert_evidence_backfill` + } + if failure == "legacy-write" { + stmt = testNativeSQL(`CREATE TRIGGER fail_evidence BEFORE INSERT ON advert_route_evidence WHEN NEW.bit=2 BEGIN SELECT RAISE(ABORT,'fixture evidence failure'); END`, `CREATE OR REPLACE FUNCTION fail_evidence_fn() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.bit=2 THEN RAISE EXCEPTION 'fixture evidence failure'; END IF; RETURN NEW; END $$; CREATE TRIGGER fail_evidence BEFORE INSERT ON advert_route_evidence FOR EACH ROW EXECUTE FUNCTION fail_evidence_fn()`) + } + if _, err := testAdmin(t, s).Exec(stmt); err != nil { + t.Fatal(err) + } + before := s.Stats.WriteErrors.Load() + data.Timestamp = "2026-01-02T00:00:00Z" + if _, err := s.InsertTransmission(data); err != nil { + t.Errorf("analytics failure aborted core ingestion: %v", err) + } + // The observation UPSERT preserves its original timestamp; tx last_seen advances. + var raw, resolved string + var ts, lastSeen int64 + if err := s.db.QueryRow(`SELECT raw_hex,COALESCE(resolved_path,''),timestamp FROM observations`).Scan(&raw, &resolved, &ts); err != nil { + t.Fatal(err) + } + if raw != data.RawHex || resolved != `["bbbbbbbbbb"]` || ts != 1767225600 { + t.Errorf("core observation not updated: raw=%s resolved=%s timestamp=%d", raw, resolved, ts) + } + if err := s.db.QueryRow(`SELECT last_seen FROM transmissions`).Scan(&lastSeen); err != nil { + t.Fatal(err) + } + if lastSeen != 1767312000 || nodeLastSeen(t, s, relay) != data.Timestamp { + t.Errorf("core liveness not updated: tx=%d relay=%s", lastSeen, nodeLastSeen(t, s, relay)) + } + if s.Stats.WriteErrors.Load() != before+1 { + t.Errorf("analytics failure not counted once: before=%d after=%d", before, s.Stats.WriteErrors.Load()) + } + var mask int + if err := s.db.QueryRow(`SELECT COALESCE(SUM(bit),0) FROM advert_route_evidence`).Scan(&mask); err != nil { + t.Fatal(err) + } + want := 1 + if failure == "incoming-write" { + want = 2 + } + if mask != want { + t.Errorf("independent evidence not preserved: mask=%d want %d", mask, want) + } + }) + } +} + +func TestAdvertRouteEvidenceConstraintsAndFeedIndex(t *testing.T) { + s, err := openPostgresTestStore(t, filepath.Join(t.TempDir(), "constraints.db")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + if _, err := s.InsertTransmission(&PacketData{Hash: "constraints", PayloadType: 4, RouteType: 1, RawHex: "1100aa", PathJSON: "[]"}); err != nil { + t.Fatal(err) + } + for _, stmt := range []string{ + `INSERT INTO advert_route_evidence(tx_id,bit) VALUES(1,1)`, + `INSERT INTO advert_route_evidence(tx_id,bit) VALUES(1,3)`, + `INSERT INTO advert_route_evidence(tx_id,bit) VALUES(999,2)`, + } { + if _, err := testAdmin(t, s).Exec(stmt); err == nil { + t.Errorf("constraint accepted: %s", stmt) + } + } + if s.Backend() == "postgres" { + s.db.Exec(`SET enable_seqscan=off`) + } + rows, err := s.db.Query(testNativeSQL(`EXPLAIN QUERY PLAN SELECT id,tx_id,bit FROM advert_route_evidence WHERE id>1 ORDER BY id LIMIT 500`, `EXPLAIN (COSTS OFF) SELECT id,tx_id,bit FROM advert_route_evidence WHERE id>1 ORDER BY id LIMIT 500`)) + if err != nil { + t.Fatal(err) + } + var plan string + for rows.Next() { + var line string + if err := scanTestPlan(rows, &line); err != nil { + t.Fatal(err) + } + plan += line + } + rows.Close() + if !strings.Contains(plan, testNativeSQL("INTEGER PRIMARY KEY", "advert_route_evidence_pkey")) { + t.Fatalf("feed must seek by primary key: %s", plan) + } + if _, err := s.db.Exec(`DELETE FROM observations; DELETE FROM transmissions`); err != nil { + t.Fatal(err) + } + if _, err := s.InsertTransmission(&PacketData{Hash: "after-retention", PayloadType: 4, RouteType: 1, RawHex: "1100aa", PathJSON: "[]"}); err != nil { + t.Fatal(err) + } + var nextID int + if err := s.db.QueryRow(`SELECT MIN(id) FROM advert_route_evidence`).Scan(&nextID); err != nil { + t.Fatal(err) + } + if nextID <= 1 { + t.Fatalf("feed ID reused after retention: %d", nextID) + } +} + +func BenchmarkAdvertEvidenceRepeatedWrite(b *testing.B) { + s, err := openPostgresTestStore(b, filepath.Join(b.TempDir(), "repeat.db")) + if err != nil { + b.Fatal(err) + } + defer s.Close() + s.WaitForAsyncMigrations() + if _, err := s.InsertTransmission(&PacketData{Hash: "benchmark-repeat", PayloadType: 4, RouteType: 1, RawHex: "1100aa", PathJSON: "[]"}); err != nil { + b.Fatal(err) + } + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err := s.stmtInsertAdvertEvidence.Exec(1, 1, 1, 1); err != nil { + b.Fatal(err) + } + } +} + +func TestAdvertRouteEvidenceRejectsUnprovenFrames(t *testing.T) { + s, err := openPostgresTestStore(t, filepath.Join(t.TempDir(), "evidence.db")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + for _, raw := range []string{"", "12", "1200", "1200a", "12zzaa", "12c0aa", "1201ffaa", "130102030401ffaa", "13zz00000000aa", "1600aa", "1100zz", "1101"} { + data := &PacketData{Hash: "unproven-" + raw, PayloadType: 4, RouteType: 2, RawHex: raw, Timestamp: "2026-01-01T00:00:00Z", PathJSON: "[]"} + if _, err := s.InsertTransmission(data); err != nil { + t.Fatal(err) + } + } + var count int + if err := s.db.QueryRow(testNativeSQL(`SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='advert_route_evidence'`, `SELECT COUNT(*) FROM information_schema.tables WHERE table_schema=current_schema() AND table_name='advert_route_evidence'`)).Scan(&count); err != nil { + t.Fatal(err) + } + if count != 1 { + t.Fatal("missing durable advert route evidence table") + } + if err := s.db.QueryRow(`SELECT COUNT(*) FROM advert_route_evidence`).Scan(&count); err != nil { + t.Fatal(err) + } + if count != 0 { + t.Fatalf("malformed/nonzero-path/unrelated frames contributed %d evidence rows", count) + } +} diff --git a/cmd/ingestor/approved_channels.go b/cmd/ingestor/approved_channels.go new file mode 100644 index 000000000..fe88f6fcd --- /dev/null +++ b/cmd/ingestor/approved_channels.go @@ -0,0 +1,233 @@ +package main + +import ( + "context" + "database/sql" + "errors" + "log" + "net/url" + "strings" + "sync/atomic" + "time" + + _ "github.com/mattn/go-sqlite3" + "github.com/meshcore-analyzer/channel" + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/pgutil" +) + +// Approved hashtag channels (docs/specs/2026-10-07-channel-proposals-design.md). +// The server owns account data; the ingestor only reads its approved view +// SQL, once per approvedChannelsRefresh, and never writes it (AGENTS.md, +// read/write separation). + +const approvedChannelsRefresh = time.Minute + +// approvedChannelsQuery is pinned in internal/users/proposals_test.go +// (ingestorApprovedQuery), which checks it against ApprovedSubjects. +const approvedChannelsQuery = `SELECT subject FROM approved_channels WHERE kind = 'hashtag_channel' ORDER BY decided_at, id LIMIT $1` + +const approvedChannelsSQLiteQuery = `SELECT subject FROM proposals WHERE kind = 'hashtag_channel' AND status = 'approved' ORDER BY decided_at, id LIMIT ?1` + +var errUsersDBMissing = errors.New("approved channel account storage is not configured") + +// channelKeySet hands each message the current channel key map. Snapshots +// are never mutated after they are stored, so decoders read them without a +// lock. refresh is called from one goroutine only (startup, then the ticker). +type channelKeySet struct { + configured map[string]string + path string + backend dbconfig.Backend + max int + cur atomic.Pointer[map[string]string] + + db *sql.DB // opened lazily, read-only + failing bool // the last read failed; logged once per failure streak + approved int // approved keys in the current snapshot +} + +// newChannelKeySet starts with the configured keys (the very map +// loadChannelKeys built, so the feature off changes nothing). +// newChannelKeySet retains compatibility for an unambiguous native target. +func newChannelKeySet(configured map[string]string, target string, max int) *channelKeySet { + storage := dbconfig.Storage{Backend: dbconfig.SQLite, UsersDBPath: target} + if strings.HasPrefix(strings.ToLower(target), "postgres:") || strings.HasPrefix(strings.ToLower(target), "postgresql:") { + storage.Backend = dbconfig.Postgres + storage.ApprovedChannelsDatabaseURL = target + } + return newChannelKeySetStorage(configured, storage, max) +} + +// newChannelKeySetStorage consumes the same resolved storage choice as startup. +// Inactive addresses never cause fallback to a different account store. +func newChannelKeySetStorage(configured map[string]string, storage dbconfig.Storage, max int) *channelKeySet { + target := storage.UsersDBPath + if storage.Backend == dbconfig.Postgres { + target = storage.ApprovedChannelsDatabaseURL + } + s := &channelKeySet{configured: configured, path: target, backend: storage.Backend, max: max} + s.cur.Store(&configured) + return s +} + +// Snapshot returns the key map to decode one message with. +func (s *channelKeySet) Snapshot() map[string]string { return *s.cur.Load() } + +// refresh re-reads the approved names and swaps in configured + approved. +// A failed read keeps the current snapshot: only a successful read changes +// the set, so an unavailable or incomplete account store never drops a key. +func (s *channelKeySet) refresh() { + names, err := s.readApproved() + if err != nil { + if !s.failing { + log.Printf("[channels] approved channels unavailable (%v); keeping the %d keys in force", err, len(s.Snapshot())) + s.failing = true + } + return + } + if s.failing { + log.Print("[channels] approved channels readable again") + s.failing = false + } + next, added := mergeApprovedKeys(s.configured, names) + if added != s.approved { + log.Printf("[channels] %d approved hashtag channel(s) in force", added) + } + s.approved = added + s.cur.Store(&next) +} + +func (s *channelKeySet) readApproved() ([]string, error) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if s.backend != dbconfig.SQLite && s.backend != dbconfig.Postgres { + return nil, errors.New("approved channel storage backend must be selected") + } + if s.db == nil { + if s.path == "" { + return nil, errUsersDBMissing + } + var db *sql.DB + var err error + if s.backend == dbconfig.SQLite { + uri, e := dbconfig.SQLiteURI(s.path, url.Values{"mode": {"ro"}, "_busy_timeout": {"5000"}}) + if e != nil { + return nil, e + } + db, err = sql.Open("sqlite3", uri) + } else { + db, err = pgutil.Open(s.path, true) + if err == nil { + err = pgutil.AssertReadOnly(db) + } + if err == nil { + var credentialAccess bool + err = db.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname=current_schema() AND c.relkind IN ('r','p','m','f') AND c.relname <> 'corescope_schema' AND has_table_privilege(c.oid,'SELECT'))`).Scan(&credentialAccess) + if err == nil && credentialAccess { + err = errors.New("approved-channel reader has account-table access") + } + } + } + if err != nil { + if db != nil { + db.Close() + } + return nil, err + } + db.SetMaxOpenConns(1) + db.SetMaxIdleConns(1) + s.db = db + } + if s.backend == dbconfig.SQLite { + if err := dbconfig.AssertSQLiteImportComplete(s.db); err != nil { + return nil, err + } + } + // Check schema and rows within one native read-only snapshot. PostgreSQL's + // projection role can read only the readiness marker and approved view. + opts := &sql.TxOptions{ReadOnly: true} + query := approvedChannelsSQLiteQuery + if s.backend == dbconfig.Postgres { + opts.Isolation = sql.LevelRepeatableRead + query = approvedChannelsQuery + } + tx, err := s.db.BeginTx(ctx, opts) + if err != nil { + return nil, err + } + defer tx.Rollback() + var version int + if s.backend == dbconfig.Postgres { + var ready bool + if err := tx.QueryRowContext(ctx, `SELECT version,ready FROM corescope_schema WHERE kind='accounts'`).Scan(&version, &ready); err != nil { + return nil, errors.New("approved-channel account schema is not initialized") + } + if version != 1 || !ready { + return nil, errors.New("approved-channel account schema is incomplete or unsupported") + } + } else { + var count int + if err := tx.QueryRowContext(ctx, `SELECT count(*),COALESCE(MAX(version),0) FROM schema_version`).Scan(&count, &version); err != nil || count != 1 || version < 4 || version > 6 { + return nil, errors.New("approved-channel SQLite account schema is incomplete or unsupported") + } + } + rows, err := tx.QueryContext(ctx, query, s.max) + if err != nil { + return nil, err + } + defer rows.Close() + var out []string + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return nil, err + } + out = append(out, name) + } + if err := rows.Err(); err != nil { + return nil, err + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := tx.Commit(); err != nil { + return nil, err + } + return out, nil +} + +// mergeApprovedKeys returns a new map: the configured keys plus a derived key +// per approved name that passes channel.ValidateHashtagName unchanged and is +// not configured already (a configured name always wins). The second result +// counts the approved keys added. +func mergeApprovedKeys(configured map[string]string, names []string) (map[string]string, int) { + next := make(map[string]string, len(configured)+len(names)) + for k, v := range configured { + next[k] = v + } + added := 0 + for _, raw := range names { + name, err := channel.ValidateHashtagName(raw) + if err != nil || name != raw { + continue + } + if _, ok := next[name]; ok { + continue + } + next[name] = deriveHashtagChannelKey(name) + added++ + } + return next, added +} + +// Close closes the read-only handle, if any. +func (s *channelKeySet) Close() { + if s.db != nil { + s.db.Close() + } +} + +// Connection strings contain credentials and must never appear in logs. +func logApprovedChannelsSource(databaseURL string) { + log.Print("[proposals] reading approved channels from the selected account store") +} diff --git a/cmd/ingestor/approved_channels_import_test.go b/cmd/ingestor/approved_channels_import_test.go new file mode 100644 index 000000000..718168ba1 --- /dev/null +++ b/cmd/ingestor/approved_channels_import_test.go @@ -0,0 +1,31 @@ +package main + +import ( + "errors" + "path/filepath" + "testing" + + "github.com/meshcore-analyzer/dbconfig" +) + +func TestApprovedChannelsPendingSQLiteImportKeepsLastGoodKeys(t *testing.T) { + t.Setenv("CORESCOPE_TEST_BACKEND", "sqlite") + path := filepath.Join(t.TempDir(), "accounts.db") + writeUsersDB(t, path, testProposal{"#complete", "approved", 1}) + keys := newApprovedTestKeySet(t, configuredKeys(), path, 128) + defer keys.Close() + keys.refresh() + want := keys.Snapshot()["#complete"] + if want == "" { + t.Fatal("complete initial approval was not loaded") + } + execUsersDB(t, path, `CREATE TABLE corescope_reverse_progress (table_name TEXT)`) + execUsersDB(t, path, `UPDATE proposals SET subject='#partial'`) + if _, err := keys.readApproved(); !errors.Is(err, dbconfig.ErrSQLiteImportIncomplete) { + t.Fatalf("partial import was accepted: %v", err) + } + keys.refresh() + if keys.Snapshot()["#complete"] != want || keys.Snapshot()["#partial"] != "" { + t.Fatal("partial import replaced the last verified approval set") + } +} diff --git a/cmd/ingestor/approved_channels_test.go b/cmd/ingestor/approved_channels_test.go new file mode 100644 index 000000000..978e58683 --- /dev/null +++ b/cmd/ingestor/approved_channels_test.go @@ -0,0 +1,558 @@ +package main + +import ( + "bytes" + "crypto/sha256" + "database/sql" + "encoding/hex" + "fmt" + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/pgutil" + "github.com/meshcore-analyzer/pgutil/pgtest" + "log" + "net/url" + "os" + "path/filepath" + "reflect" + "strings" + "sync" + "testing" +) + +// The columns cmd/server's users.db (internal/users schema v4) has; the +// ingestor reads kind, subject, status, decided_at and id. +const testProposalsDDL = `CREATE TABLE IF NOT EXISTS proposals ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + kind TEXT NOT NULL, + subject TEXT NOT NULL, + status TEXT NOT NULL, + proposer_id INTEGER, + reviewer_id INTEGER, + note TEXT NOT NULL DEFAULT '', + created_at INTEGER NOT NULL, + decided_at INTEGER +)` + +type testProposal struct { + subject, status string + decidedAt int64 +} + +func execUsersDB(t *testing.T, path string, stmts ...string) { + t.Helper() + db, err := openApprovedFixture(t, approvedFixture(t, path).owner) + if err != nil { + t.Fatal(err) + } + defer db.Close() + for _, s := range stmts { + if approvedTestBackend(t) == dbconfig.SQLite && s == testProposalsDDL { + s = strings.Replace(s, "BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY", "INTEGER PRIMARY KEY", 1) + } + if _, err := db.Exec(s); err != nil { + t.Fatalf("%s: %v", s, err) + } + } + if stmts[0] == testProposalsDDL { + if approvedTestBackend(t) == dbconfig.SQLite { + if _, err := db.Exec(`CREATE TABLE IF NOT EXISTS schema_version(version INTEGER NOT NULL); DELETE FROM schema_version; INSERT INTO schema_version VALUES(5)`); err != nil { + t.Fatal(err) + } + return + } + f := approvedFixture(t, path) + for _, stmt := range []string{ + `CREATE TABLE IF NOT EXISTS corescope_schema(version INTEGER NOT NULL, kind TEXT PRIMARY KEY, ready BOOLEAN NOT NULL)`, + `INSERT INTO corescope_schema VALUES(1,'accounts',true) ON CONFLICT(kind) DO NOTHING`, + `CREATE OR REPLACE VIEW approved_channels AS SELECT kind,subject,decided_at,id FROM proposals WHERE status='approved'`, + `GRANT SELECT ON approved_channels,corescope_schema TO ` + f.role, + } { + if _, err := db.Exec(stmt); err != nil { + t.Fatal(err) + } + } + } + +} + +func writeUsersDB(t *testing.T, path string, rows ...testProposal) { + t.Helper() + stmts := []string{testProposalsDDL, `DELETE FROM proposals`} + for i, r := range rows { + stmts = append(stmts, fmt.Sprintf(`INSERT INTO proposals (kind, subject, status, created_at, decided_at) VALUES ('hashtag_channel', '%s', '%s', %d, %d)`, + r.subject, r.status, i, r.decidedAt)) + } + execUsersDB(t, path, stmts...) +} + +func configuredKeys() map[string]string { + return map[string]string{"Public": "8b3387e9c5cdea6ac9e5edbaa115cd72", "#mesh": deriveHashtagChannelKey("#mesh")} +} + +func TestApprovedChannelsQueryMatchesProjection(t *testing.T) { + path := filepath.Join(t.TempDir(), "projection.db") + writeUsersDB(t, path, testProposal{"#fixture", "approved", 1}) + reader := newApprovedTestKeySet(t, configuredKeys(), path, 128) + defer reader.Close() + if _, err := reader.readApproved(); err != nil { + t.Fatal(err) + } +} + +func TestApprovedChannelsRejectsIncompleteAccountImport(t *testing.T) { + if approvedTestBackend(t) != dbconfig.Postgres { + t.Skip("PostgreSQL import readiness marker; SQLite schema completeness is tested separately") + } + path := filepath.Join(t.TempDir(), "account-import-gate.db") + writeUsersDB(t, path, testProposal{"#fixture", "approved", 1}) + reader := newApprovedTestKeySet(t, configuredKeys(), path, 128) + defer reader.Close() + reader.refresh() + if _, ok := reader.Snapshot()["#fixture"]; !ok { + t.Fatal("projection reader with readiness metadata could not load a verified account store") + } + execUsersDB(t, path, `UPDATE corescope_schema SET ready=false WHERE kind='accounts'`, `DELETE FROM proposals`) + if _, err := reader.readApproved(); err == nil { + t.Fatal("partially imported account data was accepted") + } + reader.refresh() + if _, ok := reader.Snapshot()["#fixture"]; !ok { + t.Fatal("incomplete account import discarded the last good approved keys") + } + execUsersDB(t, path, `UPDATE corescope_schema SET ready=true,version=2 WHERE kind='accounts'`) + if _, err := reader.readApproved(); err == nil { + t.Fatal("future account schema version was accepted") + } +} + +func TestChannelKeySetOffReturnsConfiguredMap(t *testing.T) { + cfgd := configuredKeys() + s := newApprovedTestKeySet(t, cfgd, filepath.Join(t.TempDir(), "users.db"), 128) + if reflect.ValueOf(s.Snapshot()).Pointer() != reflect.ValueOf(cfgd).Pointer() { + t.Fatal("before any refresh the decoder must get the very map loadChannelKeys built") + } +} + +func TestChannelKeySetLoadsApproved(t *testing.T) { + path := filepath.Join(t.TempDir(), "users.db") + writeUsersDB(t, path, + testProposal{"#mycity", "approved", 10}, testProposal{"#waiting", "pending", 0}, + testProposal{"#nope", "rejected", 11}, testProposal{"#gone", "revoked", 12}) + s := newApprovedTestKeySet(t, configuredKeys(), path, 128) + defer s.Close() + s.refresh() + snap := s.Snapshot() + if snap["#mycity"] != deriveHashtagChannelKey("#mycity") { + t.Fatalf("#mycity key = %q", snap["#mycity"]) + } + for _, n := range []string{"#waiting", "#nope", "#gone"} { + if _, ok := snap[n]; ok { + t.Errorf("%s decrypted without approval", n) + } + } + if snap["Public"] == "" || snap["#mesh"] == "" || len(snap) != 3 { + t.Fatalf("snapshot = %v; want the configured keys plus #mycity", snap) + } +} + +func TestChannelKeySetConfiguredNameWins(t *testing.T) { + path := filepath.Join(t.TempDir(), "users.db") + writeUsersDB(t, path, testProposal{"#mesh", "approved", 1}) + cfgd := configuredKeys() + cfgd["#mesh"] = "00112233445566778899aabbccddeeff" + s := newApprovedTestKeySet(t, cfgd, path, 128) + defer s.Close() + s.refresh() + if got := s.Snapshot()["#mesh"]; got != "00112233445566778899aabbccddeeff" { + t.Fatalf("#mesh = %q; the configured key must win", got) + } + // Revoking the proposal leaves the configured channel decrypted. + execUsersDB(t, path, `UPDATE proposals SET status = 'revoked'`) + s.refresh() + if got := s.Snapshot()["#mesh"]; got != "00112233445566778899aabbccddeeff" { + t.Fatalf("after revoke #mesh = %q", got) + } +} + +func TestChannelKeySetRevokeRemovesKeyOnNextRefresh(t *testing.T) { + path := filepath.Join(t.TempDir(), "users.db") + writeUsersDB(t, path, testProposal{"#mycity", "approved", 1}) + s := newApprovedTestKeySet(t, configuredKeys(), path, 128) + defer s.Close() + s.refresh() + before := s.Snapshot() + execUsersDB(t, path, `UPDATE proposals SET status = 'revoked' WHERE subject = '#mycity'`) + s.refresh() + if _, ok := s.Snapshot()["#mycity"]; ok { + t.Fatal("#mycity still decrypted after revoke") + } + if _, ok := before["#mycity"]; !ok { + t.Fatal("an earlier snapshot must stay unchanged (decoders may still hold it)") + } +} + +func TestChannelKeySetMissingFileKeepsConfiguredAndLogsOnce(t *testing.T) { + var buf bytes.Buffer + log.SetOutput(&buf) + defer log.SetOutput(os.Stderr) + path := filepath.Join(t.TempDir(), "users.db") + s := newApprovedTestKeySet(t, configuredKeys(), path, 128) + defer s.Close() + s.refresh() + s.refresh() + if len(s.Snapshot()) != 2 { + t.Fatalf("snapshot = %v; want the configured keys", s.Snapshot()) + } + if n := strings.Count(buf.String(), "approved channels unavailable"); n != 1 { + t.Fatalf("logged %d times; want once:\n%s", n, buf.String()) + } + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Fatal("the ingestor created users.db") + } + writeUsersDB(t, path, testProposal{"#mycity", "approved", 1}) + s.refresh() + if _, ok := s.Snapshot()["#mycity"]; !ok { + t.Fatal("users.db appearing later is picked up") + } +} + +func TestChannelKeySetReadErrorKeepsLastGoodSet(t *testing.T) { + path := filepath.Join(t.TempDir(), "users.db") + writeUsersDB(t, path, testProposal{"#mycity", "approved", 1}) + s := newApprovedTestKeySet(t, configuredKeys(), path, 128) + defer s.Close() + s.refresh() + statement := `DROP VIEW approved_channels` + if approvedTestBackend(t) == dbconfig.SQLite { + statement = `DROP TABLE proposals` + } + execUsersDB(t, path, statement) + s.refresh() + if _, ok := s.Snapshot()["#mycity"]; !ok { + t.Fatal("a failing read removed an approved key") + } +} + +func TestChannelKeySetSkipsNamesThatFailTheRules(t *testing.T) { + path := filepath.Join(t.TempDir(), "users.db") + writeUsersDB(t, path, + testProposal{"#public", "approved", 1}, testProposal{"nohash", "approved", 2}, + testProposal{"#" + strings.Repeat("a", 31), "approved", 3}, testProposal{"#ok", "approved", 4}) + s := newApprovedTestKeySet(t, configuredKeys(), path, 128) + defer s.Close() + s.refresh() + snap := s.Snapshot() + if len(snap) != 3 || snap["#ok"] == "" { + t.Fatalf("snapshot = %v; want configured + #ok only", snap) + } +} + +func TestChannelKeySetCapKeepsOldestApprovals(t *testing.T) { + path := filepath.Join(t.TempDir(), "users.db") + writeUsersDB(t, path, testProposal{"#c", "approved", 30}, testProposal{"#a", "approved", 10}, testProposal{"#b", "approved", 20}) + s := newApprovedTestKeySet(t, configuredKeys(), path, 2) + defer s.Close() + s.refresh() + snap := s.Snapshot() + if snap["#a"] == "" || snap["#b"] == "" || snap["#c"] != "" { + t.Fatalf("snapshot = %v; want #a and #b (oldest decisions) only", snap) + } +} + +func TestChannelKeySetIsReadOnly(t *testing.T) { + path := filepath.Join(t.TempDir(), "users.db") + writeUsersDB(t, path, testProposal{"#mycity", "approved", 1}) + s := newApprovedTestKeySet(t, configuredKeys(), path, 128) + defer s.Close() + s.refresh() + _, err := s.db.Exec(`INSERT INTO proposals (kind, subject, status, created_at) VALUES ('hashtag_channel', '#x', 'approved', 1)`) + if err == nil || !strings.Contains(strings.ToLower(err.Error()), "read") { + t.Fatalf("write through the ingestor's users.db handle: err = %v; want a read-only refusal", err) + } + if approvedTestBackend(t) == dbconfig.Postgres { + if _, err := s.db.Exec(`SET default_transaction_read_only=off`); err != nil { + t.Fatal(err) + } + if _, err := s.db.Query(`SELECT * FROM proposals`); err == nil { + t.Fatal("projection reader can read proposals table") + } + } + +} + +// The server keeps users.db in WAL mode with a writer connection open. +func TestChannelKeySetReadsWALWithOpenWriter(t *testing.T) { + path := filepath.Join(t.TempDir(), "users.db") + writeUsersDB(t, path, testProposal{"#mycity", "approved", 1}) + w, err := openApprovedFixture(t, approvedFixture(t, path).owner) + if err != nil { + t.Fatal(err) + } + defer w.Close() + w.SetMaxOpenConns(1) + + if _, err := w.Exec(`INSERT INTO proposals (kind, subject, status, created_at, decided_at) VALUES ('hashtag_channel', '#walcity', 'approved', 9, 9)`); err != nil { + t.Fatal(err) + } + s := newApprovedTestKeySet(t, configuredKeys(), path, 128) + defer s.Close() + s.refresh() + if s.Snapshot()["#walcity"] == "" || s.Snapshot()["#mycity"] == "" { + t.Fatalf("snapshot = %v; want both approved names", s.Snapshot()) + } +} + +func grpTxtFor(t testing.TB, channelName, text string) []byte { + key := deriveHashtagChannelKey(channelName) + inner := append([]byte{1, 0, 0, 0, 0}, []byte(text)...) + ctHex, macHex := buildChannelEncrypted(key, inner) + kb, _ := hex.DecodeString(key) + ct, _ := hex.DecodeString(ctHex) + mac, _ := hex.DecodeString(macHex) + h := sha256.Sum256(kb) + return append(append([]byte{h[0]}, mac...), ct...) +} + +// Run with -race: decoders read snapshots while refresh swaps them. +func TestChannelKeySetConcurrentDecodeWhileSwapping(t *testing.T) { + path := filepath.Join(t.TempDir(), "users.db") + writeUsersDB(t, path, testProposal{"#mycity", "approved", 1}) + s := newApprovedTestKeySet(t, configuredKeys(), path, 128) + defer s.Close() + s.refresh() + buf := grpTxtFor(t, "#mycity", "alice: hi") + stop := make(chan struct{}) + var wg sync.WaitGroup + for i := 0; i < 4; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for { + select { + case <-stop: + return + default: + } + if p := decodeGrpTxt(buf, s.Snapshot()); p.Type == "CHAN" && p.Channel != "#mycity" { + t.Errorf("decoded as %q", p.Channel) + return + } + } + }() + } + for i := 0; i < 50; i++ { + status := "revoked" + if i%2 == 1 { + status = "approved" + } + execUsersDB(t, path, `UPDATE proposals SET status = '`+status+`'`) + s.refresh() + } + close(stop) + wg.Wait() + if p := decodeGrpTxt(buf, s.Snapshot()); p.Type != "CHAN" || p.Channel != "#mycity" { + t.Fatalf("final decode = %+v; want #mycity (last state approved)", p) + } +} + +// Perf evidence for the PR: worst case (no key opens the packet), a +// realistic configured set (320 keys, the size of channel-rainbow.json plus +// Public) alone vs plus 128 approved keys (the maxApproved default). +func BenchmarkDecodeGrpTxtApprovedKeys(b *testing.B) { + buf := grpTxtFor(b, "#nobody-has-this", "x: y") + base := configuredKeys() + for i := len(base); i < 320; i++ { + name := fmt.Sprintf("#rainbow%03d", i) + base[name] = deriveHashtagChannelKey(name) + } + names := make([]string, 128) + for i := range names { + names[i] = fmt.Sprintf("#bench%03d", i) + } + withApproved, _ := mergeApprovedKeys(base, names) + for _, c := range []struct { + name string + keys map[string]string + }{{"configured", base}, {"plus128approved", withApproved}} { + b.Run(c.name, func(b *testing.B) { + for i := 0; i < b.N; i++ { + decodeGrpTxt(buf, c.keys) + } + }) + } +} + +func TestApprovedChannelsConfig(t *testing.T) { + t.Setenv("DB_PATH", "") + dir := t.TempDir() + dbPath := filepath.ToSlash(filepath.Join(dir, "data", "meshcore.db")) + load := func(um string) *Config { + t.Helper() + p := filepath.Join(dir, "config.json") + if err := os.WriteFile(p, []byte(`{"dbPath": "`+dbPath+`"`+um+`}`), 0o644); err != nil { + t.Fatal(err) + } + cfg, err := LoadConfig(p) + if err != nil { + t.Fatal(err) + } + return cfg + } + off := load(``) + if off.ApprovedChannelsEnabled() || off.ApprovedChannelsMax() != 128 || off.ApprovedChannelsURL() != "" { + t.Fatalf("no block: enabled=%v max=%d path=%q", off.ApprovedChannelsEnabled(), off.ApprovedChannelsMax(), off.ApprovedChannelsURL()) + } + on := load(`, "userManagement": {"enabled": true, "adminEmails": ["a@example.org"], "mail": {"provider": "brevo"}, "channelProposals": {"enabled": true}}`) + if !on.ApprovedChannelsEnabled() || on.ApprovedChannelsMax() != 128 { + t.Fatalf("on: enabled=%v max=%d", on.ApprovedChannelsEnabled(), on.ApprovedChannelsMax()) + } + for _, um := range []string{ + `, "userManagement": {"enabled": false, "channelProposals": {"enabled": true}}`, + `, "userManagement": {"enabled": true}`, + `, "userManagement": {"enabled": true, "channelProposals": {"enabled": false}}`, + } { + if load(um).ApprovedChannelsEnabled() { + t.Errorf("%s resolved as on", um) + } + } + custom := load(`, "userManagement": {"enabled": true, "approvedChannelsDatabaseURL": "postgres://reader@localhost/accounts", "channelProposals": {"enabled": true, "maxApproved": 7}}`) + if custom.ApprovedChannelsURL() != "postgres://reader@localhost/accounts" || custom.ApprovedChannelsMax() != 7 { + t.Fatalf("custom: path=%q max=%d", custom.ApprovedChannelsURL(), custom.ApprovedChannelsMax()) + } + if load(`, "userManagement": {"enabled": true, "channelProposals": {"enabled": true, "maxApproved": -3}}`).ApprovedChannelsMax() != 128 { + t.Fatal("negative maxApproved must fall back to 128") + } +} + +// The startup line shows the absolute users.db path, so a relative path +// resolved against another working directory than the server's is visible. +func TestApprovedChannelsSourceLogRedactsCredentials(t *testing.T) { + var buf bytes.Buffer + prev := log.Writer() + log.SetOutput(&buf) + defer log.SetOutput(prev) + logApprovedChannelsSource("postgres://reader:secret-password@localhost/accounts") + if strings.Contains(buf.String(), "secret-password") || strings.Contains(buf.String(), "postgres://") { + t.Fatal("database URL leaked in log") + } +} + +type approvedFixtureDB struct{ owner, reader, role string } + +var approvedFixtures sync.Map + +func approvedFixture(t *testing.T, key string) approvedFixtureDB { + t.Helper() + if approvedTestBackend(t) == dbconfig.SQLite { + return approvedFixtureDB{owner: key, reader: key} + } + key = t.Name() + "|" + key + if v, ok := approvedFixtures.Load(key); ok { + return v.(approvedFixtureDB) + } + owner := pgtest.NewSchema(t) + u, _ := url.Parse(owner) + schema := u.Query().Get("search_path") + role := schema + "_reader" + db, err := pgutil.Open(owner, false) + if err != nil { + t.Fatal(err) + } + for _, stmt := range []string{`CREATE ROLE ` + role + ` LOGIN PASSWORD '` + role + `'`, `GRANT USAGE ON SCHEMA ` + schema + ` TO ` + role} { + if _, err := db.Exec(stmt); err != nil { + t.Fatal(err) + } + } + u.User = url.UserPassword(role, role) + f := approvedFixtureDB{owner, u.String(), role} + approvedFixtures.Store(key, f) + t.Cleanup(func() { + approvedFixtures.Delete(key) + db.Exec(`DROP OWNED BY ` + role) + db.Exec(`DROP ROLE ` + role) + db.Close() + }) + return f +} +func newApprovedTestKeySet(t *testing.T, configured map[string]string, key string, max int) *channelKeySet { + return newChannelKeySet(configured, approvedFixture(t, key).reader, max) +} + +func approvedTestBackend(t *testing.T) dbconfig.Backend { + t.Helper() + switch os.Getenv("CORESCOPE_TEST_BACKEND") { + case "", "sqlite": + return dbconfig.SQLite + case "postgres": + return dbconfig.Postgres + default: + t.Fatal("invalid CORESCOPE_TEST_BACKEND") + return "" + } +} +func openApprovedFixture(t *testing.T, path string) (*sql.DB, error) { + t.Helper() + if approvedTestBackend(t) == dbconfig.Postgres { + return pgutil.Open(path, false) + } + uri, err := dbconfig.SQLiteURI(path, url.Values{"_journal_mode": {"WAL"}, "_busy_timeout": {"5000"}}) + if err != nil { + return nil, err + } + return sql.Open("sqlite3", uri) +} + +func TestApprovedChannelsStorageChoiceDoesNotFallback(t *testing.T) { + path := filepath.Join(t.TempDir(), "accounts.db") + // A selected PostgreSQL target cannot fall back even when a local file exists. + if approvedTestBackend(t) == dbconfig.SQLite { + writeUsersDB(t, path, testProposal{"#local", "approved", 1}) + } + s := newChannelKeySetStorage(configuredKeys(), dbconfig.Storage{Backend: dbconfig.Postgres, UsersDBPath: path}, 128) + defer s.Close() + if _, err := s.readApproved(); err != errUsersDBMissing { + t.Fatalf("missing PostgreSQL target error=%v", err) + } + s.refresh() + if len(s.Snapshot()) != 2 { + t.Fatal("invalid backend selection added account keys") + } +} + +func TestApprovedChannelsSQLiteVersionsAndLiteralPath(t *testing.T) { + if approvedTestBackend(t) != dbconfig.SQLite { + t.Skip("native SQLite contract") + } + path := filepath.Join(t.TempDir(), "account # % ü.db") + writeUsersDB(t, path, testProposal{"#local", "approved", 1}) + s := newChannelKeySetStorage(configuredKeys(), dbconfig.Storage{Backend: dbconfig.SQLite, UsersDBPath: path, ApprovedChannelsDatabaseURL: "postgres://unused.invalid/accounts"}, 128) + defer s.Close() + for _, version := range []int{4, 5, 6} { + execUsersDB(t, path, fmt.Sprintf(`UPDATE schema_version SET version=%d`, version)) + names, err := s.readApproved() + if err != nil || len(names) != 1 || names[0] != "#local" { + t.Fatalf("version%d names=%v,%v", version, names, err) + } + } + s.refresh() + for _, version := range []int{3, 7} { + execUsersDB(t, path, fmt.Sprintf(`UPDATE schema_version SET version=%d`, version)) + if _, err := s.readApproved(); err == nil { + t.Fatalf("unsupported version%d accepted", version) + } + s.refresh() + if s.Snapshot()["#local"] == "" { + t.Fatal("unsupported schema removed last good key") + } + } + if s.db.Stats().MaxOpenConnections != 1 { + t.Fatal("account reader pool is not bounded") + } + // mode=ro remains authoritative even when a caller disables query_only. + if _, err := s.db.Exec(`PRAGMA query_only=OFF`); err != nil { + t.Fatal(err) + } + if _, err := s.db.Exec(`DELETE FROM proposals`); err == nil { + t.Fatal("SQLite account reader became writable") + } +} diff --git a/cmd/ingestor/async_migration.go b/cmd/ingestor/async_migration.go index 32133db4e..e631af65e 100644 --- a/cmd/ingestor/async_migration.go +++ b/cmd/ingestor/async_migration.go @@ -1,31 +1,7 @@ -// Async migration helper — runs schema/backfill work that may take minutes on -// large prod tables WITHOUT blocking ingestor startup. -// -// MIGRATION ANNOTATION CONVENTION (read this before touching migrations): -// -// Sync schema/data migrations (CREATE INDEX, ALTER TABLE, UPDATE ... WHERE) -// that run inline during OpenStore() block the ingestor from accepting -// packets until they finish. On an empty dev DB they return in milliseconds; -// at prod scale (1.9M+ observations, 80K+ adverts) they can pin the boot -// for minutes and trigger restart loops. This regression class has bitten us -// repeatedly (#791 resolved_path backfill, #1483 obs_observer_ts_idx_v1). -// -// ANY new CREATE INDEX / ALTER TABLE / data-rewrite migration MUST EITHER: -// 1. Run via Store.RunAsyncMigration(...) below (preferred for backfills -// and any work that may touch >1K rows). The migration is recorded as -// `pending_async` immediately, returns to the caller (boot proceeds), -// and completes in a goroutine. Status flips to `done` (or `failed` -// with an error message) when fn returns. -// 2. Carry the preflight annotation comment immediately above the -// migration block, e.g. -// // PREFLIGHT: async=true reason="" -// Use this for migrations that are genuinely cheap at any scale -// (e.g. ALTER TABLE ADD COLUMN, CREATE INDEX on a known-bounded -// table). The annotation is grepped by -// ~/.openclaw/skills/pr-preflight/scripts/check-async-migrations.sh -// — its absence on a touched migration block is a hard-fail gate. -// -// See MIGRATIONS.md in the repo root for the full policy and examples. +// Async data backfills resume through durable bookkeeping. Schema DDL belongs +// to the SQLite writer or PostgreSQL owner; PostgreSQL callbacks must not +// create or alter tables/indexes. Long backfills use bounded transactions and +// yield between batches so live ingestion can continue. package main @@ -33,22 +9,21 @@ import ( "context" "database/sql" "fmt" + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/dbschema" "log" ) -// ensureAsyncMigrationsTable creates the bookkeeping table used by -// RunAsyncMigration / AsyncMigrationStatus. Idempotent. -func ensureAsyncMigrationsTable(db *sql.DB) error { - _, err := db.Exec(` - CREATE TABLE IF NOT EXISTS _async_migrations ( - name TEXT PRIMARY KEY, - status TEXT NOT NULL, -- pending_async | done | failed - started_at TEXT NOT NULL DEFAULT (datetime('now')), - ended_at TEXT, - error TEXT - ) - `) - return err +// SQLite owns its ledger; PostgreSQL checks the bootstrap-owned ledger without DDL. +func ensureAsyncMigrationsTable(db *sql.DB, backend dbconfig.Backend) error { + if backend == dbconfig.SQLite { + return dbschema.EnsureSQLiteAsyncMigrations(db) + } + rows, err := db.Query(`SELECT name FROM _async_migrations LIMIT 0`) + if err != nil { + return err + } + return rows.Close() } // RunAsyncMigration registers `name` as a pending async migration and @@ -65,27 +40,30 @@ func ensureAsyncMigrationsTable(db *sql.DB) error { // - The caller's WaitGroup tracks the goroutine so tests/shutdown can // wait via Store.WaitForAsyncMigrations(). func (s *Store) RunAsyncMigration(ctx context.Context, name string, fn func(context.Context, *sql.DB) error) error { - if err := ensureAsyncMigrationsTable(s.db); err != nil { + if err := ensureAsyncMigrationsTable(s.db, s.Backend()); err != nil { return fmt.Errorf("ensure _async_migrations: %w", err) } var existing string - row := s.db.QueryRow(`SELECT status FROM _async_migrations WHERE name = ?`, name) + row := s.db.QueryRow(`SELECT status FROM _async_migrations WHERE name = `+s.parameter(1), name) switch err := row.Scan(&existing); err { case nil: if existing == "done" { return nil // already complete, nothing to do } // pending_async or failed → reset and retry. - if _, err := s.db.Exec(` + if _, err := s.db.Exec(s.nativeSQL(` UPDATE _async_migrations SET status = 'pending_async', started_at = datetime('now'), ended_at = NULL, error = NULL - WHERE name = ?`, name); err != nil { + WHERE name = ?1`, ` + UPDATE _async_migrations + SET status = 'pending_async', started_at = to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC','YYYY-MM-DD HH24:MI:SS'), ended_at = NULL, error = NULL + WHERE name = `+s.parameter(1)), name); err != nil { return fmt.Errorf("reset async migration %q: %w", name, err) } case sql.ErrNoRows: if _, err := s.db.Exec(` - INSERT INTO _async_migrations (name, status) VALUES (?, 'pending_async')`, + INSERT INTO _async_migrations (name, status) VALUES (`+s.parameter(1)+`, 'pending_async')`, name); err != nil { return fmt.Errorf("register async migration %q: %w", name, err) } @@ -103,19 +81,25 @@ func (s *Store) RunAsyncMigration(ctx context.Context, name string, fn func(cont log.Printf("[async-migration] %q panic recovered: %v", name, r) } if runErr != nil { - if _, err := s.db.Exec(` + if _, err := s.db.Exec(s.nativeSQL(` UPDATE _async_migrations - SET status = 'failed', ended_at = datetime('now'), error = ? - WHERE name = ?`, runErr.Error(), name); err != nil { + SET status = 'failed', ended_at = datetime('now'), error = ?1 + WHERE name = ?2`, ` + UPDATE _async_migrations + SET status = 'failed', ended_at = to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC','YYYY-MM-DD HH24:MI:SS'), error = `+s.parameter(1)+` + WHERE name = `+s.parameter(2)), runErr.Error(), name); err != nil { log.Printf("[async-migration] failed to record failure for %q: %v", name, err) } log.Printf("[async-migration] %q FAILED: %v", name, runErr) return } - if _, err := s.db.Exec(` + if _, err := s.db.Exec(s.nativeSQL(` UPDATE _async_migrations SET status = 'done', ended_at = datetime('now'), error = NULL - WHERE name = ?`, name); err != nil { + WHERE name = ?1`, ` + UPDATE _async_migrations + SET status = 'done', ended_at = to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC','YYYY-MM-DD HH24:MI:SS'), error = NULL + WHERE name = `+s.parameter(1)), name); err != nil { log.Printf("[async-migration] failed to mark %q done: %v", name, err) return } @@ -132,11 +116,11 @@ func (s *Store) RunAsyncMigration(ctx context.Context, name string, fn func(cont // (one of "pending_async", "done", "failed") or sql.ErrNoRows if no such // migration has been registered. func (s *Store) AsyncMigrationStatus(name string) (string, error) { - if err := ensureAsyncMigrationsTable(s.db); err != nil { + if err := ensureAsyncMigrationsTable(s.db, s.Backend()); err != nil { return "", err } var status string - err := s.db.QueryRow(`SELECT status FROM _async_migrations WHERE name = ?`, name).Scan(&status) + err := s.db.QueryRow(`SELECT status FROM _async_migrations WHERE name = `+s.parameter(1), name).Scan(&status) return status, err } diff --git a/cmd/ingestor/async_migration_test.go b/cmd/ingestor/async_migration_test.go index 2a8a942d2..4e31d412d 100644 --- a/cmd/ingestor/async_migration_test.go +++ b/cmd/ingestor/async_migration_test.go @@ -110,7 +110,7 @@ func TestRunAsyncMigration_PanicCapture(t *testing.T) { } var errMsg sql.NullString - if err := s.db.QueryRow(`SELECT error FROM _async_migrations WHERE name = ?`, name).Scan(&errMsg); err != nil { + if err := s.db.QueryRow(`SELECT error FROM _async_migrations WHERE name = $1`, name).Scan(&errMsg); err != nil { t.Fatalf("error column lookup: %v", err) } if !errMsg.Valid || errMsg.String == "" { @@ -157,10 +157,10 @@ func TestRunAsyncMigration_RestartSafetyFailedIsRetried(t *testing.T) { s := newTestStore(t) const name = "test_restart_failed_v1" - if err := ensureAsyncMigrationsTable(s.db); err != nil { + if err := ensureAsyncMigrationsTable(s.db, s.Backend()); err != nil { t.Fatalf("ensure table: %v", err) } - if _, err := s.db.Exec(`INSERT INTO _async_migrations (name, status, error) VALUES (?, 'failed', 'simulated prior crash')`, name); err != nil { + if _, err := s.db.Exec(`INSERT INTO _async_migrations (name, status, error) VALUES ($1, 'failed', 'simulated prior crash')`, name); err != nil { t.Fatalf("seed failed row: %v", err) } @@ -181,7 +181,7 @@ func TestRunAsyncMigration_RestartSafetyFailedIsRetried(t *testing.T) { // And the error column must be cleared on success. var errCol sql.NullString - if err := s.db.QueryRow(`SELECT error FROM _async_migrations WHERE name = ?`, name).Scan(&errCol); err != nil { + if err := s.db.QueryRow(`SELECT error FROM _async_migrations WHERE name = $1`, name).Scan(&errCol); err != nil { t.Fatalf("error col: %v", err) } if errCol.Valid && errCol.String != "" { @@ -197,10 +197,10 @@ func TestRunAsyncMigration_RestartSafetyPendingIsRetried(t *testing.T) { s := newTestStore(t) const name = "test_restart_pending_v1" - if err := ensureAsyncMigrationsTable(s.db); err != nil { + if err := ensureAsyncMigrationsTable(s.db, s.Backend()); err != nil { t.Fatalf("ensure table: %v", err) } - if _, err := s.db.Exec(`INSERT INTO _async_migrations (name, status) VALUES (?, 'pending_async')`, name); err != nil { + if _, err := s.db.Exec(`INSERT INTO _async_migrations (name, status) VALUES ($1, 'pending_async')`, name); err != nil { t.Fatalf("seed pending row: %v", err) } @@ -243,7 +243,7 @@ func TestRunAsyncMigration_FnErrorRecorded(t *testing.T) { } var errCol sql.NullString - if err := s.db.QueryRow(`SELECT error FROM _async_migrations WHERE name = ?`, name).Scan(&errCol); err != nil { + if err := s.db.QueryRow(`SELECT error FROM _async_migrations WHERE name = $1`, name).Scan(&errCol); err != nil { t.Fatalf("error col: %v", err) } if !errCol.Valid || errCol.String == "" { @@ -297,3 +297,20 @@ func TestRunAsyncMigration_ConcurrentSameNameSerialized(t *testing.T) { t.Fatalf("fn invoked %d times, want 1..5 inclusive (bounded by caller count)", got) } } + +func TestAsyncMigrationMinimalSQLiteStore(t *testing.T) { + db, err := sql.Open("sqlite3", ":memory:") + if err != nil { + t.Fatal(err) + } + defer db.Close() + db.SetMaxOpenConns(1) + s := &Store{db: db} + if err := s.RunAsyncMigration(context.Background(), "minimal", func(context.Context, *sql.DB) error { return nil }); err != nil { + t.Fatal(err) + } + s.WaitForAsyncMigrations() + if status, err := s.AsyncMigrationStatus("minimal"); err != nil || status != "done" { + t.Fatal(status, err) + } +} diff --git a/cmd/ingestor/boot_migration_wait_test.go b/cmd/ingestor/boot_migration_wait_test.go new file mode 100644 index 000000000..59792f462 --- /dev/null +++ b/cmd/ingestor/boot_migration_wait_test.go @@ -0,0 +1,70 @@ +package main + +import ( + "bytes" + "log" + "strings" + "testing" +) + +// Issue #2065: the race detector failed on master with +// +// Write by RunAsyncMigration.func1 (async_migration.go, log.Printf) +// Read by TestHandleMessageDecodeErrorLog_PII_Issue1211 (buf.String()) +// +// OpenStore schedules two async migrations whose goroutines log while they +// run, and that test points the standard logger at a bytes.Buffer and reads +// it. Both touch one buffer with no synchronisation. +// +// Close() already waits on backfillWg, so the goroutines do not outlive their +// test: the race is inside a single test, not across tests. newTestStore +// therefore waits for the boot migrations before handing the store over, so +// no test body can run while one is in flight. +// +// These two tests pin that, deterministically — the race detector only catches +// the fault when the scheduler cooperates, which is why it sat latent from +// 2026-09-03 until it surfaced three weeks later. + +// TestNewTestStoreWaitsForBootMigrations fails if the wait is removed from the +// helper. tx_last_seen_backfill_v1 is scheduled unconditionally by OpenStore +// (db.go), so on a fresh temp database it is always pending at that moment and +// can only read "done" here if something waited. +func TestNewTestStoreWaitsForBootMigrations(t *testing.T) { + s := newTestStore(t) + + status, err := s.AsyncMigrationStatus("tx_last_seen_backfill_v1") + if err != nil { + t.Fatalf("AsyncMigrationStatus: %v", err) + } + if status != "done" { + t.Errorf("tx_last_seen_backfill_v1 = %q immediately after newTestStore, want %q — "+ + "the helper must wait for the boot migrations, or a test capturing the standard "+ + "logger races their goroutines (#2065)", status, "done") + } +} + +// TestCapturedLogIsFreeOfMigrationOutput is the assertion from the failing +// test's own point of view: once the store is handed over, nothing else is +// writing to the logger, so a captured buffer holds only what the test put +// there. Without the wait this is the buffer two goroutines fight over. +func TestCapturedLogIsFreeOfMigrationOutput(t *testing.T) { + newTestStore(t) + + var buf bytes.Buffer + orig := log.Writer() + log.SetOutput(&buf) + defer log.SetOutput(orig) + + log.Printf("sentinel line") + + out := buf.String() + if !strings.Contains(out, "sentinel line") { + t.Fatalf("capture did not work at all; got %q", out) + } + // "[migration/async]" is what the two boot migrations print. Their + // appearance here means a goroutine was still running when the capture + // started, which is the race, not merely untidy output. + if strings.Contains(out, "[migration/async]") || strings.Contains(out, "[async-migration]") { + t.Errorf("migration output landed in a captured buffer: %q", out) + } +} diff --git a/cmd/ingestor/client_reception.go b/cmd/ingestor/client_reception.go index c42c3560a..ecdc7f39d 100644 --- a/cmd/ingestor/client_reception.go +++ b/cmd/ingestor/client_reception.go @@ -1,6 +1,7 @@ package main import ( + "database/sql" "encoding/json" "log" "regexp" @@ -22,7 +23,7 @@ var clientPubkeyRe = regexp.MustCompile(`^[0-9a-f]{2,64}$`) // companion reports WHERE it directly heard a node, so we write a // client_receptions row and never touch the observers/observations tables. // rxPubkey is the companion pubkey from the topic (ACL-bound by the broker). -func handleClientPacket(store *Store, cfg *Config, tag, rxPubkey string, msg map[string]interface{}, channelKeys map[string]string, regionKeys map[string][]byte) { +func handleClientPacket(store *Store, cfg *Config, tag, rxPubkey string, msg map[string]interface{}, channelKeys map[string]string, regionSet *regionKeySet) { // The companion identity IS the (ACL-bound) topic pubkey. Reject non-hex // topic segments so a no-ACL broker can't pollute the coverage tables, and // never fall back to a payload-supplied id (that would defeat the ACL trust @@ -83,6 +84,7 @@ func handleClientPacket(store *Store, cfg *Config, tag, rxPubkey string, msg map rxAt := rxTime.Format(time.RFC3339) ingestedAt := time.Now().UTC().Format(time.RFC3339) isAdvert := decoded.Header.PayloadTypeName == "ADVERT" + isDiscoverResp := decoded.Header.PayloadTypeName == "CONTROL" && decoded.Payload.CtrlSubtype == "DISCOVER_RESP" if cfg.ClientRxObservationsEnabled() { // rxAtMillis: UNIQUE(rx_pubkey, pkt_hash, rx_at) needs sub-second @@ -91,7 +93,7 @@ func handleClientPacket(store *Store, cfg *Config, tag, rxPubkey string, msg map // collapse them and ON CONFLICT DO NOTHING would silently drop all but // the first. rxAtMillis := rxTime.Format(rxTimeMillisLayout) - if obs := buildClientRxObservation(direction, rxPubkey, rawHex, rxAtMillis, ingestedAt, decoded, regionKeys, snrPtr, rssiPtr, lat, lon, accPtr); obs != nil { + if obs := buildClientRxObservation(direction, rxPubkey, rawHex, rxAtMillis, ingestedAt, decoded, regionSet, snrPtr, rssiPtr, lat, lon, accPtr); obs != nil { if _, err := store.InsertClientRxObservation(obs); err != nil { log.Printf("MQTT [%s] client observation insert: %v", tag, err) } @@ -101,6 +103,7 @@ func handleClientPacket(store *Store, cfg *Config, tag, rxPubkey string, msg map rec, ok := buildClientReception( rxPubkey, direction, decoded.Header.RouteType, decoded.Header.PayloadType, decoded.Path.Hops, decoded.Payload.PubKey, isAdvert, + isDiscoverResp, decoded.Payload.CtrlPubKey, snrPtr, rssiPtr, lat, lon, accPtr, rxAt, ingestedAt, ) if !ok { @@ -125,7 +128,7 @@ func (s *Store) UpsertClientObserver(pubkey, name, ts string) error { return nil } _, err := s.db.Exec(` - INSERT INTO client_observers (pubkey, name, last_seen) VALUES (?,?,?) + INSERT INTO client_observers (pubkey, name, last_seen) VALUES (`+s.parameter(1)+`,`+s.parameter(2)+`,`+s.parameter(3)+`) ON CONFLICT(pubkey) DO UPDATE SET name = excluded.name, last_seen = excluded.last_seen`, strings.ToLower(pubkey), name, ts) return err @@ -178,10 +181,16 @@ type ClientReception struct { // consume the next hop from the FRONT (firmware Mesh.cpp removeSelfFromPath), // so path[len-1] is the route's destination-side end, not who was heard. // - hops empty + isAdvert → the 0-hop advertiser, by its full pubkey. +// - hops empty + isDiscoverResp → the 0-hop responder, by its pubkey (8 or +// 32 bytes — CTL_TYPE_NODE_DISCOVER_RESP always carries an 8-byte prefix +// or the full 32-byte key; anything else is rejected outright, not +// truncated). A node-discover reply is always a direct, zero-hop RX — +// stronger identification than a path hash, since it carries the +// responder's own identity rather than an inferred forwarder. // - otherwise → not attributable (ok=false). // // Returns (heardKey lowercased, keylenBytes, src, ok). -func deriveHeardKey(direction string, routeType, payloadType int, hops []string, advertPubkey string, isAdvert bool) (string, int, string, bool) { +func deriveHeardKey(direction string, routeType, payloadType int, hops []string, advertPubkey string, isAdvert bool, isDiscoverResp bool, discoverPubkey string) (string, int, string, bool) { if !strings.EqualFold(direction, "rx") { return "", 0, "", false } @@ -207,6 +216,14 @@ func deriveHeardKey(direction string, routeType, payloadType int, hops []string, pk := strings.ToLower(strings.TrimSpace(advertPubkey)) return pk, len(pk) / 2, "advert", true } + if isDiscoverResp && discoverPubkey != "" { + pk := strings.ToLower(strings.TrimSpace(discoverPubkey)) + keylen := len(pk) / 2 + if keylen != 8 && keylen != 32 { // discover pubkeys are always 8B (prefix) or 32B (full) — no floor, an exact set + return "", 0, "", false + } + return pk, keylen, "discover", true + } return "", 0, "", false } @@ -214,6 +231,7 @@ func deriveHeardKey(direction string, routeType, payloadType int, hops []string, // returns ok=false when the packet is not attributable / out of range. func buildClientReception( rxPubkey, direction string, routeType, payloadType int, hops []string, advertPubkey string, isAdvert bool, + isDiscoverResp bool, discoverPubkey string, snr *float64, rssi *int, lat, lon float64, posAccM *float64, rxAt, ingestedAt string, ) (*ClientReception, bool) { if rxPubkey == "" || rxAt == "" { @@ -222,7 +240,7 @@ func buildClientReception( if lat < -90 || lat > 90 || lon < -180 || lon > 180 { return nil, false } - heardKey, keylen, src, ok := deriveHeardKey(direction, routeType, payloadType, hops, advertPubkey, isAdvert) + heardKey, keylen, src, ok := deriveHeardKey(direction, routeType, payloadType, hops, advertPubkey, isAdvert, isDiscoverResp, discoverPubkey) if !ok { return nil, false } @@ -240,7 +258,7 @@ func (s *Store) InsertClientReception(r *ClientReception) (bool, error) { res, err := s.db.Exec(` INSERT INTO client_receptions (rx_pubkey, heard_key, heard_keylen, rssi, snr, lat, lon, pos_acc_m, rx_at, ingested_at, src) - VALUES (?,?,?,?,?,?,?,?,?,?,?) + VALUES (`+s.parameter(1)+`,`+s.parameter(2)+`,`+s.parameter(3)+`,`+s.parameter(4)+`,`+s.parameter(5)+`,`+s.parameter(6)+`,`+s.parameter(7)+`,`+s.parameter(8)+`,`+s.parameter(9)+`,`+s.parameter(10)+`,`+s.parameter(11)+`) ON CONFLICT(rx_pubkey, heard_key, rx_at) DO NOTHING`, r.RxPubkey, r.HeardKey, r.HeardKeyLen, r.RSSI, r.SNR, r.Lat, r.Lon, r.PosAccM, r.RxAt, r.IngestedAt, r.Src) if err != nil { @@ -281,7 +299,7 @@ func (s *Store) InsertClientRfSample(o *ClientRfSample) (bool, error) { uptime_secs, battery_mv, queue_len, errors, noise_floor, last_rssi, last_snr, tx_air_secs, rx_air_secs, recv, sent, flood_rx, direct_rx, flood_tx, direct_tx, recv_errors) - VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) + VALUES (`+s.parameter(1)+`,`+s.parameter(2)+`,`+s.parameter(3)+`,`+s.parameter(4)+`,`+s.parameter(5)+`,`+s.parameter(6)+`,`+s.parameter(7)+`,`+s.parameter(8)+`,`+s.parameter(9)+`,`+s.parameter(10)+`,`+s.parameter(11)+`,`+s.parameter(12)+`,`+s.parameter(13)+`,`+s.parameter(14)+`,`+s.parameter(15)+`,`+s.parameter(16)+`,`+s.parameter(17)+`,`+s.parameter(18)+`,`+s.parameter(19)+`,`+s.parameter(20)+`,`+s.parameter(21)+`,`+s.parameter(22)+`,`+s.parameter(23)+`) ON CONFLICT(rx_pubkey, sampled_at) DO NOTHING`, o.RxPubkey, o.SampledAt, o.IngestedAt, o.Lat, o.Lon, o.PosAccM, boolToInt(o.Stationary), o.UptimeSecs, o.BatteryMV, o.QueueLen, o.Errors, o.NoiseFloor, o.LastRSSI, @@ -328,7 +346,7 @@ type ClientRxObservation struct { // signal — per-flood row multiplicity meant to measure forwarder // amplification of traffic actually heard over the air. func buildClientRxObservation( - direction, rxPubkey, rawHex, rxAt, ingestedAt string, decoded *DecodedPacket, regionKeys map[string][]byte, + direction, rxPubkey, rawHex, rxAt, ingestedAt string, decoded *DecodedPacket, regionSet *regionKeySet, snr *float64, rssi *int, lat, lon float64, posAccM *float64, ) *ClientRxObservation { if !strings.EqualFold(direction, "rx") { @@ -353,7 +371,7 @@ func buildClientRxObservation( obs.Code1 = &decoded.TransportCodes.Code1 obs.Code2 = &decoded.TransportCodes.Code2 if decoded.TransportCodes.Code1 != "0000" { - sn := matchScope(regionKeys, byte(decoded.Header.PayloadType), decoded.payloadRaw, decoded.TransportCodes.Code1) + sn := regionSet.matchScopeName(byte(decoded.Header.PayloadType), decoded.payloadRaw, decoded.TransportCodes.Code1) obs.ScopeName = &sn } } @@ -400,7 +418,7 @@ func (s *Store) InsertClientRxObservation(o *ClientRxObservation) (bool, error) (rx_pubkey, rx_at, ingested_at, pkt_hash, route_type, payload_type, code1, code2, scope_name, hash_size, hop_count, path_json, forwarder, snr, rssi, lat, lon, pos_acc_m) - VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) + VALUES (`+s.parameter(1)+`,`+s.parameter(2)+`,`+s.parameter(3)+`,`+s.parameter(4)+`,`+s.parameter(5)+`,`+s.parameter(6)+`,`+s.parameter(7)+`,`+s.parameter(8)+`,`+s.parameter(9)+`,`+s.parameter(10)+`,`+s.parameter(11)+`,`+s.parameter(12)+`,`+s.parameter(13)+`,`+s.parameter(14)+`,`+s.parameter(15)+`,`+s.parameter(16)+`,`+s.parameter(17)+`,`+s.parameter(18)+`) ON CONFLICT(rx_pubkey, pkt_hash, rx_at) DO NOTHING`, o.RxPubkey, o.RxAt, o.IngestedAt, o.PktHash, o.RouteType, o.PayloadType, o.Code1, o.Code2, o.ScopeName, o.HashSize, o.HopCount, o.PathJSON, o.Forwarder, @@ -411,3 +429,65 @@ func (s *Store) InsertClientRxObservation(o *ClientRxObservation) (bool, error) n, _ := res.RowsAffected() return n > 0, nil } + +// ClientDeclaredRegions is one repeater's answer to a flood-allowed region +// list request (ANON_REQ_TYPE_REGIONS), relayed by a mobile companion. Every +// answer is a row — this table stores observations, never state. An empty +// RegionsCSV is a valid, deliberate answer ("nothing flood-allowed"); a +// repeater that did not answer produces no row at all, never a row with an +// empty list standing in for silence. Truncated is a hint, not a detection: +// the firmware's exportNamesTo skips names that do not fit and continues, so +// an overflowing reply has holes rather than a truncated prefix, with no +// marker distinguishing the two. +type ClientDeclaredRegions struct { + Target string + RxPubkey string + ObservedAt string + IngestedAt string + RegionsCSV string + Truncated bool + Lat, Lon *float64 + PosAccM *float64 + RepeaterClock *int64 +} + +// InsertClientDeclaredRegions writes one declared-region observation. +// Idempotent via UNIQUE(target, rx_pubkey, observed_at); returns ins=false +// when the row already existed. +func (s *Store) InsertClientDeclaredRegions(o *ClientDeclaredRegions) (bool, error) { + res, err := s.db.Exec(` + INSERT INTO node_declared_regions + (target, rx_pubkey, observed_at, ingested_at, regions_csv, truncated, lat, lon, pos_acc_m, repeater_clock) + VALUES (`+s.parameter(1)+`,`+s.parameter(2)+`,`+s.parameter(3)+`,`+s.parameter(4)+`,`+s.parameter(5)+`,`+s.parameter(6)+`,`+s.parameter(7)+`,`+s.parameter(8)+`,`+s.parameter(9)+`,`+s.parameter(10)+`) + ON CONFLICT(target, rx_pubkey, observed_at) DO NOTHING`, + o.Target, o.RxPubkey, o.ObservedAt, o.IngestedAt, o.RegionsCSV, boolToInt(o.Truncated), o.Lat, o.Lon, o.PosAccM, o.RepeaterClock) + if err != nil { + return false, err + } + n, _ := res.RowsAffected() + return n > 0, nil +} + +// CurrentDeclaredRegions returns the most recent declared-region observation +// for a target, or nil if none exist. "Most recent" is the greatest +// observed_at, NOT the greatest ingested_at: a drive buffered offline can +// arrive days late, and ordering by arrival would let that stale reading +// overwrite a fresher one. +func (s *Store) CurrentDeclaredRegions(target string) (*ClientDeclaredRegions, error) { + target = strings.ToLower(strings.TrimSpace(target)) + row := s.db.QueryRow(` + SELECT target, rx_pubkey, observed_at, ingested_at, regions_csv, truncated, lat, lon, pos_acc_m, repeater_clock + FROM node_declared_regions + WHERE target = `+s.parameter(1)+` + ORDER BY observed_at DESC LIMIT 1`, target) + var o ClientDeclaredRegions + var truncated int + if err := row.Scan(&o.Target, &o.RxPubkey, &o.ObservedAt, &o.IngestedAt, &o.RegionsCSV, &truncated, &o.Lat, &o.Lon, &o.PosAccM, &o.RepeaterClock); err != nil { + if err == sql.ErrNoRows { + return nil, nil + } + return nil, err + } + o.Truncated = truncated == 1 + return &o, nil +} diff --git a/cmd/ingestor/client_reception_test.go b/cmd/ingestor/client_reception_test.go index 8ff513a1f..65c5d3d02 100644 --- a/cmd/ingestor/client_reception_test.go +++ b/cmd/ingestor/client_reception_test.go @@ -54,16 +54,20 @@ func TestPruneOldClientReceptions(t *testing.T) { // this is a regression guard. func TestPruneClientRxObservationsUsesIndex(t *testing.T) { s := newTestStore(t) - rows, err := s.db.Query(`EXPLAIN QUERY PLAN DELETE FROM client_rx_observations WHERE rx_at < ?`, "2026-01-01T00:00:00Z") + if testBackendValue() == "postgres" { + if _, err := s.db.Exec(`SET enable_seqscan=off`); err != nil { + t.Fatal(err) + } + } + rows, err := s.db.Query(testNativeSQL(`EXPLAIN QUERY PLAN DELETE FROM client_rx_observations WHERE rx_at < $1`, `EXPLAIN (COSTS OFF) DELETE FROM client_rx_observations WHERE rx_at < $1`), "2026-01-01T00:00:00Z") if err != nil { t.Fatal(err) } defer rows.Close() plan := "" for rows.Next() { - var id, parent, notused int var detail string - if err := rows.Scan(&id, &parent, ¬used, &detail); err != nil { + if err := scanTestPlan(rows, &detail); err != nil { t.Fatal(err) } plan += detail + "\n" @@ -125,12 +129,12 @@ func TestPruneOldClientRxObservations(t *testing.T) { t.Fatalf("expected 2 observations remaining (recent + boundary), got %d", remaining) } var recentSurvived int - s.db.QueryRow(`SELECT COUNT(*) FROM client_rx_observations WHERE pkt_hash = ?`, "hash-recent").Scan(&recentSurvived) + s.db.QueryRow(`SELECT COUNT(*) FROM client_rx_observations WHERE pkt_hash = $1`, "hash-recent").Scan(&recentSurvived) if recentSurvived != 1 { t.Fatalf("the recent row must survive the prune; got %d", recentSurvived) } var boundarySurvived int - s.db.QueryRow(`SELECT COUNT(*) FROM client_rx_observations WHERE pkt_hash = ?`, "hash-boundary").Scan(&boundarySurvived) + s.db.QueryRow(`SELECT COUNT(*) FROM client_rx_observations WHERE pkt_hash = $1`, "hash-boundary").Scan(&boundarySurvived) if boundarySurvived != 1 { t.Fatalf("boundary row (500ms after cutoff instant) must survive; an RFC3339 (no-ms) cutoff would wrongly delete it because '.' < 'Z' lexicographically — got %d", boundarySurvived) } @@ -139,7 +143,7 @@ func TestPruneOldClientRxObservations(t *testing.T) { func TestClientReceptionsTableExists(t *testing.T) { s := newTestStore(t) cols := map[string]bool{} - rows, err := s.db.Query(`PRAGMA table_info(client_receptions)`) + rows, err := s.db.Query(testNativeSQL(`SELECT cid,name,type,"notnull",dflt_value,pk FROM pragma_table_info('client_receptions') ORDER BY cid`, `SELECT ordinal_position::int,column_name,CASE data_type WHEN 'bigint' THEN 'INTEGER' WHEN 'double precision' THEN 'REAL' ELSE upper(data_type) END,CASE is_nullable WHEN 'NO' THEN 1 ELSE 0 END,column_default,0 FROM information_schema.columns WHERE table_schema=current_schema() AND table_name='client_receptions' ORDER BY ordinal_position`)) if err != nil { t.Fatalf("PRAGMA failed: %v", err) } @@ -168,12 +172,15 @@ func crI(i int) *int { return &i } // per-node coverage query (sargable heard_key IN-list + bbox, mirroring // cmd/server coverageHeardKeyCandidates) seeks the heard_key composite index // rather than scanning the table. Without idx_client_recept_heard_geo the plan -// is "SCAN client_receptions". +// is "Seq Scan on client_receptions". func TestClientReceptionsCoverageQueryUsesIndex(t *testing.T) { s := newTestStore(t) - q := `EXPLAIN QUERY PLAN SELECT lat, lon, snr, rssi, heard_key, rx_at + q := testNativeSQL(`EXPLAIN QUERY PLAN SELECT lat, lon, snr, rssi, heard_key, rx_at FROM client_receptions - WHERE heard_key IN (?,?,?) AND lat BETWEEN ? AND ? AND lon BETWEEN ? AND ?` + WHERE heard_key IN ($1,$2,$3) AND lat BETWEEN $4 AND $5 AND lon BETWEEN $6 AND $7`, `EXPLAIN (COSTS OFF) SELECT lat, lon, snr, rssi, heard_key, rx_at + FROM client_receptions + WHERE heard_key IN ($1,$2,$3) AND lat BETWEEN $4 AND $5 AND lon BETWEEN $6 AND $7`) + s.db.Exec(`SET enable_seqscan=off`) rows, err := s.db.Query(q, "aabbccddeeff00112233", "aabbcc", "aabb", 50.0, 52.0, 3.0, 4.0) if err != nil { t.Fatal(err) @@ -181,17 +188,16 @@ func TestClientReceptionsCoverageQueryUsesIndex(t *testing.T) { defer rows.Close() plan := "" for rows.Next() { - var id, parent, notused int var detail string - if err := rows.Scan(&id, &parent, ¬used, &detail); err != nil { + if err := scanTestPlan(rows, &detail); err != nil { t.Fatal(err) } plan += detail + "\n" } - if !strings.Contains(plan, "USING INDEX idx_client_recept") { + if !strings.Contains(plan, "idx_client_recept") { t.Fatalf("coverage query should use a client_recept index, plan was:\n%s", plan) } - if strings.Contains(plan, "SCAN client_receptions") { + if strings.Contains(plan, "Seq Scan on client_receptions") { t.Fatalf("coverage query should not full-scan, plan was:\n%s", plan) } } @@ -201,16 +207,20 @@ func TestClientReceptionsCoverageQueryUsesIndex(t *testing.T) { // index rather than full-scanning under the writer lock (polish review). func TestClientReceptionsRetentionUsesRxAtIndex(t *testing.T) { s := newTestStore(t) - rows, err := s.db.Query(`EXPLAIN QUERY PLAN DELETE FROM client_receptions WHERE rx_at < ?`, "2026-01-01T00:00:00Z") + if testBackendValue() == "postgres" { + if _, err := s.db.Exec(`SET enable_seqscan=off`); err != nil { + t.Fatal(err) + } + } + rows, err := s.db.Query(testNativeSQL(`EXPLAIN QUERY PLAN DELETE FROM client_receptions WHERE rx_at < $1`, `EXPLAIN (COSTS OFF) DELETE FROM client_receptions WHERE rx_at < $1`), "2026-01-01T00:00:00Z") if err != nil { t.Fatal(err) } defer rows.Close() plan := "" for rows.Next() { - var id, parent, notused int var detail string - if err := rows.Scan(&id, &parent, ¬used, &detail); err != nil { + if err := scanTestPlan(rows, &detail); err != nil { t.Fatal(err) } plan += detail + "\n" @@ -230,22 +240,32 @@ func TestClientReceptionsRetentionUsesRxAtIndex(t *testing.T) { // index-backed). func TestRxLeaderboardQueryIsIndexBacked(t *testing.T) { s := newTestStore(t) - rows, err := s.db.Query(`EXPLAIN QUERY PLAN + if testBackendValue() == "postgres" { + if _, err := s.db.Exec(`SET enable_seqscan=off`); err != nil { + t.Fatal(err) + } + } + rows, err := s.db.Query(testNativeSQL(`EXPLAIN QUERY PLAN SELECT cr.rx_pubkey, COUNT(*), COUNT(DISTINCT cr.heard_key) FROM client_receptions cr - WHERE cr.rx_at >= ? + WHERE cr.rx_at >= $1 GROUP BY cr.rx_pubkey ORDER BY COUNT(*) DESC - LIMIT ?`, "2026-01-01T00:00:00Z", 100) + LIMIT $2`, `EXPLAIN (COSTS OFF) + SELECT cr.rx_pubkey, COUNT(*), COUNT(DISTINCT cr.heard_key) + FROM client_receptions cr + WHERE cr.rx_at >= $1 + GROUP BY cr.rx_pubkey + ORDER BY COUNT(*) DESC + LIMIT $2`), "2026-01-01T00:00:00Z", 100) if err != nil { t.Fatal(err) } defer rows.Close() plan := "" for rows.Next() { - var id, parent, notused int var detail string - if err := rows.Scan(&id, &parent, ¬used, &detail); err != nil { + if err := scanTestPlan(rows, &detail); err != nil { t.Fatal(err) } plan += detail + "\n" @@ -254,63 +274,96 @@ func TestRxLeaderboardQueryIsIndexBacked(t *testing.T) { // The concern is a bare table-heap scan, not which specific index wins. The // plan must stay index-backed (covering or search) — a regression to a bare // "SCAN cr" without an index fails here. - if !strings.Contains(plan, "INDEX") { + if !strings.Contains(strings.ToLower(plan), "index") { t.Fatalf("leaderboard SELECT must stay index-backed (no full table-heap scan), plan was:\n%s", plan) } } func TestDeriveHeardKey(t *testing.T) { full := "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789" - k, l, src, ok := deriveHeardKey("rx", packetpath.RouteFlood, PayloadADVERT, nil, strings.ToUpper(full), true) + k, l, src, ok := deriveHeardKey("rx", packetpath.RouteFlood, PayloadADVERT, nil, strings.ToUpper(full), true, false, "") if !ok || l != 32 || src != "advert" || k != full { t.Fatalf("0-hop advert: got k=%q l=%d src=%q ok=%v", k, l, src, ok) } - k, l, src, ok = deriveHeardKey("rx", packetpath.RouteFlood, PayloadGRP_TXT, []string{"aa", "bbccdd"}, "", false) + k, l, src, ok = deriveHeardKey("rx", packetpath.RouteFlood, PayloadGRP_TXT, []string{"aa", "bbccdd"}, "", false, false, "") if !ok || k != "bbccdd" || l != 3 || src != "rxlog" { t.Fatalf("flood path: got k=%q l=%d src=%q ok=%v", k, l, src, ok) } // DIRECT route: path[last] is the route's far end, not the transmitter — must be rejected. - if _, _, _, ok = deriveHeardKey("rx", packetpath.RouteDirect, PayloadGRP_TXT, []string{"aa", "bbccdd"}, "", false); ok { + if _, _, _, ok = deriveHeardKey("rx", packetpath.RouteDirect, PayloadGRP_TXT, []string{"aa", "bbccdd"}, "", false, false, ""); ok { t.Fatalf("direct-route path must be rejected") } - if _, _, _, ok = deriveHeardKey("rx", packetpath.RouteTransportDirect, PayloadGRP_TXT, []string{"aa", "bbccdd"}, "", false); ok { + if _, _, _, ok = deriveHeardKey("rx", packetpath.RouteTransportDirect, PayloadGRP_TXT, []string{"aa", "bbccdd"}, "", false, false, ""); ok { t.Fatalf("transport-direct-route path must be rejected") } - if _, _, _, ok = deriveHeardKey("rx", packetpath.RouteFlood, PayloadGRP_TXT, []string{"aa", "bb"}, "", false); ok { + if _, _, _, ok = deriveHeardKey("rx", packetpath.RouteFlood, PayloadGRP_TXT, []string{"aa", "bb"}, "", false, false, ""); ok { t.Fatalf("1-byte last hop should be rejected") } - if _, _, _, ok = deriveHeardKey("tx", packetpath.RouteFlood, PayloadGRP_TXT, []string{"aabbcc"}, "", false); ok { + if _, _, _, ok = deriveHeardKey("tx", packetpath.RouteFlood, PayloadGRP_TXT, []string{"aabbcc"}, "", false, false, ""); ok { t.Fatalf("tx must be rejected") } - if _, _, _, ok = deriveHeardKey("rx", packetpath.RouteFlood, PayloadGRP_TXT, nil, "", false); ok { + if _, _, _, ok = deriveHeardKey("rx", packetpath.RouteFlood, PayloadGRP_TXT, nil, "", false, false, ""); ok { t.Fatalf("no hops + non-advert must be rejected") } // TRACE repurposes the header path bytes as per-hop SNR values, not node // hashes — a FLOOD-routed TRACE must never be attributable, even though the // route type and hop shape are otherwise identical to the accepted case above. - if _, _, _, ok = deriveHeardKey("rx", packetpath.RouteFlood, PayloadTRACE, []string{"aa", "bbccdd"}, "", false); ok { + if _, _, _, ok = deriveHeardKey("rx", packetpath.RouteFlood, PayloadTRACE, []string{"aa", "bbccdd"}, "", false, false, ""); ok { t.Fatalf("FLOOD-routed TRACE must be rejected (path bytes are SNR values, not node hashes)") } + + // --- discover-response cases (0x90 CTL_TYPE_NODE_DISCOVER_RESP) --- + discover32 := strings.Repeat("ab", 32) + discover8 := "abcdef0123456789" + + k, l, src, ok = deriveHeardKey("rx", packetpath.RouteFlood, PayloadCONTROL, nil, "", false, true, discover8) + if !ok || l != 8 || src != "discover" || k != discover8 { + t.Fatalf("8-byte discover pubkey: got k=%q l=%d src=%q ok=%v", k, l, src, ok) + } + k, l, src, ok = deriveHeardKey("rx", packetpath.RouteFlood, PayloadCONTROL, nil, "", false, true, strings.ToUpper(discover32)) + if !ok || l != 32 || src != "discover" || k != discover32 { + t.Fatalf("32-byte discover pubkey: got k=%q l=%d src=%q ok=%v", k, l, src, ok) + } + // Malformed/short discover payload (neither 8 nor 32 bytes) must be rejected outright, not truncated. + if _, _, _, ok = deriveHeardKey("rx", packetpath.RouteFlood, PayloadCONTROL, nil, "", false, true, "aabbcc"); ok { + t.Fatalf("short discover pubkey (3 bytes) must be rejected") + } + if _, _, _, ok = deriveHeardKey("rx", packetpath.RouteFlood, PayloadCONTROL, nil, "", false, true, ""); ok { + t.Fatalf("empty discover pubkey must be rejected") + } + // Wrong control type (not a DISCOVER_RESP) must be rejected even with a well-formed pubkey. + if _, _, _, ok = deriveHeardKey("rx", packetpath.RouteFlood, PayloadCONTROL, nil, "", false, false, discover8); ok { + t.Fatalf("non-discover CONTROL payload must be rejected") + } + // A discover response that arrived with hops is not a direct reception. + if _, _, _, ok = deriveHeardKey("rx", packetpath.RouteDirect, PayloadCONTROL, []string{"aa", "bbccdd"}, "", false, true, discover8); ok { + t.Fatalf("discover response with hops (DIRECT route) must be rejected") + } } func TestBuildClientReception(t *testing.T) { acc := 8.0 - rec, ok := buildClientReception("companionpk", "rx", packetpath.RouteFlood, PayloadGRP_TXT, []string{"aa", "bbccdd"}, "", false, + rec, ok := buildClientReception("companionpk", "rx", packetpath.RouteFlood, PayloadGRP_TXT, []string{"aa", "bbccdd"}, "", false, false, "", crF(-7.5), crI(-92), 51.05, 3.72, &acc, "2026-06-09T12:00:00Z", "2026-06-09T12:00:01Z") if !ok || rec.HeardKey != "bbccdd" || rec.HeardKeyLen != 3 || rec.Src != "rxlog" { t.Fatalf("bad reception: %+v ok=%v", rec, ok) } - if _, ok := buildClientReception("c", "rx", packetpath.RouteDirect, PayloadGRP_TXT, []string{"bbccdd"}, "", false, + if _, ok := buildClientReception("c", "rx", packetpath.RouteDirect, PayloadGRP_TXT, []string{"bbccdd"}, "", false, false, "", crF(-7.5), crI(-92), 51.05, 3.72, nil, "t", "t"); ok { t.Fatal("direct-route path must be rejected (not the transmitter)") } - if _, ok := buildClientReception("c", "rx", packetpath.RouteFlood, PayloadGRP_TXT, []string{"bbccdd"}, "", false, nil, nil, 99.0, 3.72, nil, "t", "t"); ok { + if _, ok := buildClientReception("c", "rx", packetpath.RouteFlood, PayloadGRP_TXT, []string{"bbccdd"}, "", false, false, "", nil, nil, 99.0, 3.72, nil, "t", "t"); ok { t.Fatal("out-of-range lat must be rejected") } - if _, ok := buildClientReception("c", "rx", packetpath.RouteFlood, PayloadTRACE, []string{"aa", "bbccdd"}, "", false, + if _, ok := buildClientReception("c", "rx", packetpath.RouteFlood, PayloadTRACE, []string{"aa", "bbccdd"}, "", false, false, "", crF(-7.5), crI(-92), 51.05, 3.72, nil, "t", "t"); ok { t.Fatal("FLOOD-routed TRACE must be rejected (path bytes are SNR values, not node hashes)") } + acc32 := strings.Repeat("ab", 32) + if rec, ok := buildClientReception("c", "rx", packetpath.RouteFlood, PayloadCONTROL, nil, "", false, true, acc32, + crF(-7.5), crI(-92), 51.05, 3.72, nil, "t", "t"); !ok || rec.HeardKey != acc32 || rec.HeardKeyLen != 32 || rec.Src != "discover" { + t.Fatalf("discover reception: %+v ok=%v", rec, ok) + } } func TestInsertClientReceptionRoundTripAndIdempotent(t *testing.T) { @@ -347,7 +400,7 @@ func TestHandleClientPacketRelayedAdvertWritesReception(t *testing.T) { handleClientPacket(s, &Config{}, "test", testCompanionPK, msg, nil, nil) var obsName string - s.db.QueryRow(`SELECT name FROM client_observers WHERE pubkey=?`, testCompanionPK).Scan(&obsName) + s.db.QueryRow(`SELECT name FROM client_observers WHERE pubkey=$1`, testCompanionPK).Scan(&obsName) if obsName != "MyMob" { t.Fatalf("expected client_observers name 'MyMob', got %q", obsName) } @@ -357,7 +410,7 @@ func TestHandleClientPacketRelayedAdvertWritesReception(t *testing.T) { // The 0-hop advert→full-pubkey branch is covered by TestDeriveHeardKey. var n, keylen int var src string - if err := s.db.QueryRow(`SELECT COUNT(*), COALESCE(MAX(heard_keylen),0), COALESCE(MAX(src),'') FROM client_receptions WHERE rx_pubkey=?`, testCompanionPK).Scan(&n, &keylen, &src); err != nil { + if err := s.db.QueryRow(`SELECT COUNT(*), COALESCE(MAX(heard_keylen),0), COALESCE(MAX(src),'') FROM client_receptions WHERE rx_pubkey=$1`, testCompanionPK).Scan(&n, &keylen, &src); err != nil { t.Fatal(err) } if n != 1 || keylen < 2 || src != "rxlog" { @@ -368,7 +421,7 @@ func TestHandleClientPacketRelayedAdvertWritesReception(t *testing.T) { const companion2 = "b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3" handleClientPacket(s, &Config{}, "test", companion2, map[string]interface{}{"raw": advertHex, "direction": "rx"}, nil, nil) var n2 int - s.db.QueryRow(`SELECT COUNT(*) FROM client_receptions WHERE rx_pubkey=?`, companion2).Scan(&n2) + s.db.QueryRow(`SELECT COUNT(*) FROM client_receptions WHERE rx_pubkey=$1`, companion2).Scan(&n2) if n2 != 0 { t.Fatalf("packet without gps must be dropped, got %d rows", n2) } @@ -399,7 +452,7 @@ func TestHandleClientPacketZeroHopAdvertWritesReception(t *testing.T) { var keylen int var snr sql.NullFloat64 var lat, lon float64 - if err := s.db.QueryRow(`SELECT heard_key, heard_keylen, src, snr, lat, lon FROM client_receptions WHERE rx_pubkey=?`, testCompanionPK). + if err := s.db.QueryRow(`SELECT heard_key, heard_keylen, src, snr, lat, lon FROM client_receptions WHERE rx_pubkey=$1`, testCompanionPK). Scan(&heardKey, &keylen, &src, &snr, &lat, &lon); err != nil { t.Fatalf("expected a 0-hop advert reception: %v", err) } @@ -504,6 +557,38 @@ func TestClientPacketFloodWritesBoth(t *testing.T) { } } +// TestHandleClientPacketDiscoverRespWritesReception is the end-to-end +// regression test for the discover-response gap: a zero-hop CONTROL packet +// carrying a CTL_TYPE_NODE_DISCOVER_RESP (firmware +// examples/simple_repeater/MyMesh.cpp:780) must attribute a client_receptions +// row keyed on the responder's own pubkey, src='discover' — not be silently +// dropped as it was before this fix. +func TestHandleClientPacketDiscoverRespWritesReception(t *testing.T) { + s := newTestStore(t) + // header 0x2D = route_type 1 (FLOOD), payload_type 0x0B (CONTROL). + // path byte 0x00 = 0 hops (a discover response is always heard direct). + // payload: flags(0x92 = CTL_TYPE_NODE_DISCOVER_RESP|ADV_TYPE_REPEATER) + + // snr(0x0A) + tag(4B LE) + 32-byte pubkey. + pubkey := strings.Repeat("ab", 32) + raw := "2d" + "00" + "920a44332211" + pubkey + msg := map[string]interface{}{ + "raw": raw, "direction": "rx", "SNR": 4.5, "RSSI": -101.0, + "timestamp": "2026-08-17T10:00:00.123Z", + "gps": map[string]interface{}{"lat": 51.2, "lon": 4.4, "acc_m": 8.0}, + } + handleClientPacket(s, cfgWithObservations(), "test", "aa11", msg, nil, nil) + + var heardKey, src string + var keylen int + if err := s.db.QueryRow(`SELECT heard_key, heard_keylen, src FROM client_receptions WHERE rx_pubkey=$1`, "aa11"). + Scan(&heardKey, &keylen, &src); err != nil { + t.Fatalf("expected a discover-response reception: %v", err) + } + if heardKey != pubkey || keylen != 32 || src != "discover" { + t.Fatalf("discover reception: want heard_key=%s/32/discover, got %s/%d/%s", pubkey, heardKey, keylen, src) + } +} + // TestClientObservationsGateOff verifies that with clientRxObservations // disabled, a client packet writes zero observation rows. func TestClientObservationsGateOff(t *testing.T) { @@ -638,13 +723,13 @@ func TestClientObservationScopeNameFromTransportCode(t *testing.T) { "timestamp": ts1, "gps": map[string]interface{}{"lat": 51.2, "lon": 4.4}, } - handleClientPacket(s, cfgWithObservations(), "test", "aa11", msg, nil, regionKeys) + handleClientPacket(s, cfgWithObservations(), "test", "aa11", msg, nil, regionKeySetFromKeys(regionKeys)) // ComputeContentHash deliberately excludes the transport-code bytes (so the // same content dedups across scopes), so raw and raw2 below share one // pkt_hash — distinguish the two inserted rows by rx_at, not pkt_hash. var code1, code2, scopeName sql.NullString - if err := s.db.QueryRow(`SELECT code1, code2, scope_name FROM client_rx_observations WHERE rx_at = ?`, ts1). + if err := s.db.QueryRow(`SELECT code1, code2, scope_name FROM client_rx_observations WHERE rx_at = $1`, ts1). Scan(&code1, &code2, &scopeName); err != nil { t.Fatal(err) } @@ -667,10 +752,10 @@ func TestClientObservationScopeNameFromTransportCode(t *testing.T) { "timestamp": ts2, "gps": map[string]interface{}{"lat": 51.2, "lon": 4.4}, } - handleClientPacket(s, cfgWithObservations(), "test", "aa11", msg2, nil, regionKeys) + handleClientPacket(s, cfgWithObservations(), "test", "aa11", msg2, nil, regionKeySetFromKeys(regionKeys)) var code1b, scopeNameB sql.NullString - if err := s.db.QueryRow(`SELECT code1, scope_name FROM client_rx_observations WHERE rx_at = ?`, ts2). + if err := s.db.QueryRow(`SELECT code1, scope_name FROM client_rx_observations WHERE rx_at = $1`, ts2). Scan(&code1b, &scopeNameB); err != nil { t.Fatal(err) } diff --git a/cmd/ingestor/client_regions.go b/cmd/ingestor/client_regions.go new file mode 100644 index 000000000..b78cd9f2c --- /dev/null +++ b/cmd/ingestor/client_regions.go @@ -0,0 +1,106 @@ +package main + +import ( + "log" + "regexp" + "strings" + "time" +) + +// targetPubkeyRe validates the `target` field of a /regions payload: the +// repeater the companion asked. Unlike the topic pubkey (clientPubkeyRe, +// 2-64 hex — a separate contract with its own history), target has one +// legitimate shape only. Task 1's buildRegionsRequest throws a TypeError +// unless the pubkey it sends is exactly 64 hex characters, so the app can +// only ever ask a full pubkey and can only ever report one back. A target +// shorter than that is a malformed or forged payload, not a legitimate +// variant — and accepting it anyway would be a silent-accumulation bug, not +// a loud one: CurrentDeclaredRegions matches on the exact target, so a row +// stored against a short prefix is invisible to every query for the real +// node and just accumulates as junk that looks like data. +var targetPubkeyRe = regexp.MustCompile(`^[0-9a-f]{64}$`) + +// handleClientRegions processes one region-discovery answer from +// meshcore/client/{PUBLIC_KEY}/regions: a mobile companion asked a repeater +// (target) which regions it declares flood-allowed, and relays the answer. +// The topic pubkey (rxPubkey) is the reporting companion's identity +// (ACL-bound by the broker) and is never taken from the payload. `target` is +// the repeater that was asked; it arrives as payload data and is validated +// as a full 64-hex pubkey (targetPubkeyRe) before being trusted. An empty +// `regions` list is a valid, deliberate answer ("nothing flood-allowed") and +// is stored; a missing or malformed `regions` field is not an answer at all +// and the whole message is dropped rather than half-stored as an empty list. +// GPS is optional here, unlike client coverage: a declared-regions answer is +// meaningful without a position, so a missing/invalid fix stores NULL +// lat/lon instead of dropping the row. +func handleClientRegions(store *Store, cfg *Config, tag, rxPubkey string, msg map[string]interface{}) { + rxPubkey = strings.ToLower(strings.TrimSpace(rxPubkey)) + if !clientPubkeyRe.MatchString(rxPubkey) { + log.Printf("MQTT [%s] regions: invalid pubkey %.8q, dropping", tag, rxPubkey) + return + } + target, _ := msg["target"].(string) + target = strings.ToLower(strings.TrimSpace(target)) + if !targetPubkeyRe.MatchString(target) { + log.Printf("MQTT [%s] regions: invalid target %.8q, dropping", tag, target) + return + } + regionsRaw, ok := msg["regions"].([]interface{}) + if !ok { + log.Printf("MQTT [%s] regions: missing/malformed regions field, dropping", tag) + return + } + regions := make([]string, 0, len(regionsRaw)) + for _, r := range regionsRaw { + s, ok := r.(string) + if !ok { + log.Printf("MQTT [%s] regions: non-string region entry, dropping", tag) + return + } + // The repeater's reply is encrypted with a block cipher, so its plaintext + // is NUL-padded to a 16-byte boundary and the padding lands inside the LAST + // region name. Clients trim it since coredrive-rx v1.10.2, but a PWA keeps + // running its cached build until the user reloads, and this instance has no + // control over when eight-odd independent clients do that. Trimming here + // closes it once, for every client version: "belml\x00\x00" stored verbatim + // matches no observed scope and reads as a region the repeater declares but + // never forwards. + s = strings.TrimRight(s, "\x00") + if s == "" { + continue // padding-only entry — not a declared region + } + regions = append(regions, s) + } + truncated, _ := msg["truncated"].(bool) + + rxTime, _ := resolveRxTimeCore(msg, tag) + observedAt := rxTime.Format(rxTimeMillisLayout) + + var latPtr, lonPtr, posAccPtr *float64 + if gps, ok := msg["gps"].(map[string]interface{}); ok { + lat, latOK := toFloat64(gps["lat"]) + lon, lonOK := toFloat64(gps["lon"]) + if latOK && lonOK && lat >= -90 && lat <= 90 && lon >= -180 && lon <= 180 { + latPtr, lonPtr = &lat, &lon + if acc, ok := toFloat64(gps["acc_m"]); ok { + posAccPtr = &acc + } + } + } + + o := &ClientDeclaredRegions{ + Target: target, + RxPubkey: rxPubkey, + ObservedAt: observedAt, + IngestedAt: time.Now().UTC().Format(time.RFC3339), + RegionsCSV: strings.Join(regions, ","), + Truncated: truncated, + Lat: latPtr, + Lon: lonPtr, + PosAccM: posAccPtr, + RepeaterClock: optInt64(msg, "repeater_clock"), + } + if _, err := store.InsertClientDeclaredRegions(o); err != nil { + log.Printf("MQTT [%s] node_declared_regions insert: %v", tag, err) + } +} diff --git a/cmd/ingestor/client_regions_test.go b/cmd/ingestor/client_regions_test.go new file mode 100644 index 000000000..7ceb1daa9 --- /dev/null +++ b/cmd/ingestor/client_regions_test.go @@ -0,0 +1,654 @@ +package main + +import ( + "strings" + "testing" + "time" +) + +// declaredRegionsCount returns the row count in node_declared_regions. +func declaredRegionsCount(t *testing.T, s *Store) int { + t.Helper() + var n int + if err := s.db.QueryRow(`SELECT COUNT(*) FROM node_declared_regions`).Scan(&n); err != nil { + t.Fatal(err) + } + return n +} + +// testTargetPK is a valid, exactly-64-hex-char target pubkey (the repeater +// asked), distinct from testCompanionPK (the reporting companion) so a test +// that mixes them up is caught. Must stay exactly 64 characters — that is +// what targetPubkeyRe requires, and a shortened value here would make the +// happy-path tests pass for the wrong reason. +const testTargetPK = "bb11223344556677889900aabbccddeeff00112233445566778899aabbccddaa" + +// TestHandleClientRegionsWritesRow proves a well-formed /regions message +// writes exactly one node_declared_regions row and touches nothing else. +func TestHandleClientRegionsWritesRow(t *testing.T) { + s := newTestStore(t) + msg := map[string]interface{}{ + "type": "REGIONS", "timestamp": "2026-08-18T10:00:00.000Z", + "target": testTargetPK, + "regions": []interface{}{"be", "be-vlg"}, + "truncated": false, + "gps": map[string]interface{}{"lat": 51.05, "lon": 3.72, "acc_m": 8.0}, + } + handleClientRegions(s, &Config{}, "test", testCompanionPK, msg) + + if n := declaredRegionsCount(t, s); n != 1 { + t.Fatalf("rows = %d, want 1", n) + } + cur, err := s.CurrentDeclaredRegions(testTargetPK) + if err != nil { + t.Fatal(err) + } + if cur == nil { + t.Fatal("current = nil, want a row") + } + if cur.RegionsCSV != "be,be-vlg" { + t.Errorf("RegionsCSV = %q, want %q", cur.RegionsCSV, "be,be-vlg") + } + if cur.RxPubkey != testCompanionPK { + t.Errorf("RxPubkey = %q, want the topic pubkey %q", cur.RxPubkey, testCompanionPK) + } + if cur.Lat == nil || cur.Lon == nil || *cur.Lat != 51.05 || *cur.Lon != 3.72 { + t.Errorf("lat/lon = %v/%v, want 51.05/3.72", cur.Lat, cur.Lon) + } + if _, err := time.Parse(rxTimeMillisLayout, cur.ObservedAt); err != nil { + t.Errorf("observed_at = %q, want rxTimeMillisLayout format: %v", cur.ObservedAt, err) + } +} + +// TestHandleClientRegionsInvalidTopicPubkeyDropped proves an invalid topic +// pubkey (the reporter's identity) is rejected before any write, the same +// guard used by every other client sub-topic handler. +func TestHandleClientRegionsInvalidTopicPubkeyDropped(t *testing.T) { + s := newTestStore(t) + msg := map[string]interface{}{ + "timestamp": "2026-08-18T10:00:00.000Z", + "target": testTargetPK, + "regions": []interface{}{"be"}, + } + handleClientRegions(s, &Config{}, "test", "NOT-HEX!", msg) + + if n := declaredRegionsCount(t, s); n != 0 { + t.Fatalf("rows = %d, want 0 for an invalid topic pubkey", n) + } +} + +// TestHandleClientRegionsRejectsInvalidTarget proves `target` is validated +// before being trusted — it is payload data, not ACL-bound identity. +func TestHandleClientRegionsRejectsInvalidTarget(t *testing.T) { + s := newTestStore(t) + base := func() map[string]interface{} { + return map[string]interface{}{ + "timestamp": "2026-08-18T10:00:00.000Z", + "regions": []interface{}{"be"}, + } + } + + missing := base() + handleClientRegions(s, &Config{}, "test", testCompanionPK, missing) + + invalid := base() + invalid["target"] = "not-hex!" + handleClientRegions(s, &Config{}, "test", testCompanionPK, invalid) + + empty := base() + empty["target"] = "" + handleClientRegions(s, &Config{}, "test", testCompanionPK, empty) + + if n := declaredRegionsCount(t, s); n != 0 { + t.Fatalf("rows = %d, want 0 — all three target values must be rejected", n) + } +} + +// TestHandleClientRegionsRejectsWrongLengthTarget proves a target that is +// valid hex but the wrong length is rejected, not silently accepted as a +// prefix. Task 1's buildRegionsRequest throws unless the pubkey it sends is +// exactly 64 hex characters, so the app can only ever ask (and report back) +// a full pubkey — anything shorter is a malformed/forged payload. Without +// this check, a short target would still insert a row (clientPubkeyRe alone +// accepts 2-64 hex), and CurrentDeclaredRegions matches on the exact target +// string, so that row would never surface for the real node's queries — it +// would just accumulate silently as junk that looks like data. +func TestHandleClientRegionsRejectsWrongLengthTarget(t *testing.T) { + s := newTestStore(t) + msg := map[string]interface{}{ + "timestamp": "2026-08-18T10:00:00.000Z", + "target": testTargetPK[:8], // valid hex, but only 8 of the required 64 chars + "regions": []interface{}{"be"}, + } + handleClientRegions(s, &Config{}, "test", testCompanionPK, msg) + + if n := declaredRegionsCount(t, s); n != 0 { + t.Fatalf("rows = %d, want 0 — a target shorter than 64 hex chars must be rejected", n) + } +} + +// TestHandleClientRegionsEmptyListIsValid proves an empty regions array is a +// genuine, storable observation ("nothing flood-allowed"), not treated as a +// malformed/missing field. +func TestHandleClientRegionsEmptyListIsValid(t *testing.T) { + s := newTestStore(t) + msg := map[string]interface{}{ + "timestamp": "2026-08-18T10:00:00.000Z", + "target": testTargetPK, + "regions": []interface{}{}, + } + handleClientRegions(s, &Config{}, "test", testCompanionPK, msg) + + if n := declaredRegionsCount(t, s); n != 1 { + t.Fatalf("rows = %d, want 1 — an empty regions list is a valid observation", n) + } + cur, err := s.CurrentDeclaredRegions(testTargetPK) + if err != nil { + t.Fatal(err) + } + if cur == nil { + t.Fatal("current = nil, want a row for the empty-list observation") + } + if cur.RegionsCSV != "" { + t.Errorf("RegionsCSV = %q, want empty string", cur.RegionsCSV) + } +} + +// TestHandleClientRegionsMalformedPayloadDropped proves a malformed `regions` +// field (absent, wrong type, or containing a non-string element) is dropped +// with a log — never half-stored as an empty list standing in for "no +// answer", which would blur "declared nothing" with "did not answer". +func TestHandleClientRegionsMalformedPayloadDropped(t *testing.T) { + s := newTestStore(t) + base := func() map[string]interface{} { + return map[string]interface{}{ + "timestamp": "2026-08-18T10:00:00.000Z", + "target": testTargetPK, + } + } + + missing := base() + handleClientRegions(s, &Config{}, "test", testCompanionPK, missing) + + wrongType := base() + wrongType["regions"] = "be,be-vlg" + handleClientRegions(s, &Config{}, "test", testCompanionPK, wrongType) + + badElement := base() + badElement["regions"] = []interface{}{"be", 42.0} + handleClientRegions(s, &Config{}, "test", testCompanionPK, badElement) + + if n := declaredRegionsCount(t, s); n != 0 { + t.Fatalf("rows = %d, want 0 — all three malformed payloads must be dropped, not half-stored", n) + } +} + +// TestHandleClientRegionsNoGPSStoresNullLatLon proves a declared-regions +// answer without a position is still meaningful (unlike a coverage point) and +// is stored with NULL lat/lon rather than dropped. +func TestHandleClientRegionsNoGPSStoresNullLatLon(t *testing.T) { + s := newTestStore(t) + msg := map[string]interface{}{ + "timestamp": "2026-08-18T10:00:00.000Z", + "target": testTargetPK, + "regions": []interface{}{"be"}, + } + handleClientRegions(s, &Config{}, "test", testCompanionPK, msg) + + if n := declaredRegionsCount(t, s); n != 1 { + t.Fatalf("rows = %d, want 1 — no GPS must not drop the observation", n) + } + cur, err := s.CurrentDeclaredRegions(testTargetPK) + if err != nil { + t.Fatal(err) + } + if cur == nil { + t.Fatal("current = nil, want a row") + } + if cur.Lat != nil || cur.Lon != nil { + t.Errorf("lat/lon = %v/%v, want nil/nil when gps is absent", cur.Lat, cur.Lon) + } +} + +// TestHandleClientRegionsTruncatedCarriedThrough proves `truncated` is +// stored as reported (it is a hint from the app, not something the handler +// re-detects). +func TestHandleClientRegionsTruncatedCarriedThrough(t *testing.T) { + s := newTestStore(t) + msg := map[string]interface{}{ + "timestamp": "2026-08-18T10:00:00.000Z", + "target": testTargetPK, + "regions": []interface{}{"be"}, + "truncated": true, + } + handleClientRegions(s, &Config{}, "test", testCompanionPK, msg) + + cur, err := s.CurrentDeclaredRegions(testTargetPK) + if err != nil { + t.Fatal(err) + } + if cur == nil || !cur.Truncated { + t.Errorf("Truncated = %v, want true (carried through from payload)", cur) + } +} + +// TestHandleClientRegionsRepeaterClockRoundTrips proves repeater_clock is +// read from the payload (optInt64, same helper handleClientRfSample uses for +// this shape) and stored — it is the only signal in this stream that would +// reveal a repeater with a broken RTC, and was previously discarded even +// though the app publishes it and the docs show it in the payload example. +func TestHandleClientRegionsRepeaterClockRoundTrips(t *testing.T) { + s := newTestStore(t) + msg := map[string]interface{}{ + "timestamp": "2026-08-18T10:00:00.000Z", + "target": testTargetPK, + "regions": []interface{}{"be"}, + "repeater_clock": 1755518096.0, + } + handleClientRegions(s, &Config{}, "test", testCompanionPK, msg) + + cur, err := s.CurrentDeclaredRegions(testTargetPK) + if err != nil { + t.Fatal(err) + } + if cur == nil { + t.Fatal("current = nil, want a row") + } + if cur.RepeaterClock == nil || *cur.RepeaterClock != 1755518096 { + t.Errorf("RepeaterClock = %v, want 1755518096", cur.RepeaterClock) + } +} + +// TestHandleClientRegionsOmittedRepeaterClockIsNull proves a payload that +// omits repeater_clock stores NULL, not 0 — absent is not zero, the same +// rule this codebase enforces for recv_errors (client_rf_sample.go). +func TestHandleClientRegionsOmittedRepeaterClockIsNull(t *testing.T) { + s := newTestStore(t) + msg := map[string]interface{}{ + "timestamp": "2026-08-18T10:00:00.000Z", + "target": testTargetPK, + "regions": []interface{}{"be"}, + } + handleClientRegions(s, &Config{}, "test", testCompanionPK, msg) + + cur, err := s.CurrentDeclaredRegions(testTargetPK) + if err != nil { + t.Fatal(err) + } + if cur == nil { + t.Fatal("current = nil, want a row") + } + if cur.RepeaterClock != nil { + t.Errorf("RepeaterClock = %v, want nil when the field is absent from the payload", *cur.RepeaterClock) + } +} + +// TestHandleClientRegionsPosAccMRoundTrips proves gps.acc_m is read and +// stored the same way handleClientRfSample stores it as pos_acc_m — it was +// previously dropped even though the app publishes it. +func TestHandleClientRegionsPosAccMRoundTrips(t *testing.T) { + s := newTestStore(t) + msg := map[string]interface{}{ + "timestamp": "2026-08-18T10:00:00.000Z", + "target": testTargetPK, + "regions": []interface{}{"be"}, + "gps": map[string]interface{}{"lat": 51.05, "lon": 3.72, "acc_m": 8.5}, + } + handleClientRegions(s, &Config{}, "test", testCompanionPK, msg) + + cur, err := s.CurrentDeclaredRegions(testTargetPK) + if err != nil { + t.Fatal(err) + } + if cur == nil { + t.Fatal("current = nil, want a row") + } + if cur.PosAccM == nil || *cur.PosAccM != 8.5 { + t.Errorf("PosAccM = %v, want 8.5", cur.PosAccM) + } +} + +// TestHandleClientRegionsOmittedPosAccMIsNull proves a payload whose gps +// object omits acc_m stores NULL, not 0. +func TestHandleClientRegionsOmittedPosAccMIsNull(t *testing.T) { + s := newTestStore(t) + msg := map[string]interface{}{ + "timestamp": "2026-08-18T10:00:00.000Z", + "target": testTargetPK, + "regions": []interface{}{"be"}, + "gps": map[string]interface{}{"lat": 51.05, "lon": 3.72}, + } + handleClientRegions(s, &Config{}, "test", testCompanionPK, msg) + + cur, err := s.CurrentDeclaredRegions(testTargetPK) + if err != nil { + t.Fatal(err) + } + if cur == nil { + t.Fatal("current = nil, want a row") + } + if cur.PosAccM != nil { + t.Errorf("PosAccM = %v, want nil when gps.acc_m is absent", *cur.PosAccM) + } +} + +// TestHandleClientRegionsRejectedPositionDropsPosAccM proves an accuracy +// radius is not stored when the coordinates it qualifies were rejected: a +// pos_acc_m beside a NULL lat/lon describes the precision of no position. +func TestHandleClientRegionsRejectedPositionDropsPosAccM(t *testing.T) { + s := newTestStore(t) + msg := map[string]interface{}{ + "timestamp": "2026-08-18T10:00:00.000Z", + "target": testTargetPK, + "regions": []interface{}{"be"}, + "gps": map[string]interface{}{"lat": 151.05, "lon": 3.72, "acc_m": 8.5}, + } + handleClientRegions(s, &Config{}, "test", testCompanionPK, msg) + + cur, err := s.CurrentDeclaredRegions(testTargetPK) + if err != nil { + t.Fatal(err) + } + if cur == nil { + t.Fatal("current = nil, want a row") + } + if cur.Lat != nil || cur.Lon != nil { + t.Fatalf("Lat/Lon = %v/%v, want nil for an out-of-range latitude", cur.Lat, cur.Lon) + } + if cur.PosAccM != nil { + t.Errorf("PosAccM = %v, want nil when the position was rejected", *cur.PosAccM) + } +} + +func TestDeclaredRegionsCurrentIsLatestObservation(t *testing.T) { + s := newTestStore(t) + ins := func(at, csv string) { + if _, err := s.InsertClientDeclaredRegions(&ClientDeclaredRegions{ + Target: "bb11", RxPubkey: "aa11", ObservedAt: at, IngestedAt: "2026-08-18T00:00:00.000Z", + RegionsCSV: csv, + }); err != nil { + t.Fatalf("insert %s: %v", at, err) + } + } + // Deliberately inserted newest-first, so a query ordering by rowid or by + // ingest order would pick the wrong one. + ins("2026-08-18T10:00:00.000Z", "be,be-vlg,be-van") + ins("2026-08-17T10:00:00.000Z", "be,be-vlg") + + cur, err := s.CurrentDeclaredRegions("bb11") + if err != nil { + t.Fatal(err) + } + if cur.RegionsCSV != "be,be-vlg,be-van" { + t.Errorf("current = %q, want the 18th's list — a late-arriving older drive must not win", cur.RegionsCSV) + } +} + +func TestDeclaredRegionsIdempotent(t *testing.T) { + s := newTestStore(t) + o := &ClientDeclaredRegions{Target: "bb11", RxPubkey: "aa11", + ObservedAt: "2026-08-18T10:00:00.000Z", IngestedAt: "2026-08-18T10:00:01.000Z", RegionsCSV: "be"} + if ins, err := s.InsertClientDeclaredRegions(o); err != nil || !ins { + t.Fatalf("first: ins=%v err=%v", ins, err) + } + if ins, err := s.InsertClientDeclaredRegions(o); err != nil || ins { + t.Fatalf("duplicate must be a no-op: ins=%v err=%v", ins, err) + } +} + +// TestDeclaredRegionsEmptyListRoundTrips proves an empty regions_csv is a +// valid, retrievable observation — "a repeater with nothing flood-allowed +// genuinely declares nothing, and that gets a row." Nothing in the insert or +// query path may special-case the empty string into a NULL or a dropped row; +// this test would fail against a later "helpful" `if regionsCSV == "" { +// return }` guard. +func TestDeclaredRegionsEmptyListRoundTrips(t *testing.T) { + s := newTestStore(t) + o := &ClientDeclaredRegions{Target: "bb11", RxPubkey: "aa11", + ObservedAt: "2026-08-18T10:00:00.000Z", IngestedAt: "2026-08-18T10:00:01.000Z", RegionsCSV: ""} + if ins, err := s.InsertClientDeclaredRegions(o); err != nil || !ins { + t.Fatalf("insert: ins=%v err=%v", ins, err) + } + + cur, err := s.CurrentDeclaredRegions("bb11") + if err != nil { + t.Fatal(err) + } + if cur == nil { + t.Fatal("current = nil, want a row for the empty-list observation") + } + if cur.RegionsCSV != "" { + t.Errorf("RegionsCSV = %q, want empty string — an empty list is a valid observation, not absence", cur.RegionsCSV) + } +} + +// TestDeclaredRegionsDifferentReportersPersistBoth pins the UNIQUE +// constraint's shape from the permitting side: two different companions +// (rx_pubkey) reporting the same target's regions at the same observed_at +// are two distinct observations and must both persist. Only a genuine +// duplicate delivery from the SAME reporter (same target, rx_pubkey AND +// observed_at — see TestDeclaredRegionsIdempotent) collapses. A later +// constraint change to (target, observed_at) would fail this test loudly +// instead of silently discarding the second reporter's reading. +func TestDeclaredRegionsDifferentReportersPersistBoth(t *testing.T) { + s := newTestStore(t) + at := "2026-08-18T10:00:00.000Z" + for _, rx := range []string{"aa11", "aa22"} { + if _, err := s.InsertClientDeclaredRegions(&ClientDeclaredRegions{ + Target: "bb11", RxPubkey: rx, ObservedAt: at, IngestedAt: at, RegionsCSV: "be", + }); err != nil { + t.Fatalf("insert %s: %v", rx, err) + } + } + + var n int + if err := s.db.QueryRow(`SELECT COUNT(*) FROM node_declared_regions WHERE target = $1 AND observed_at = $2`, "bb11", at).Scan(&n); err != nil { + t.Fatal(err) + } + if n != 2 { + t.Errorf("rows = %d, want 2 — two different reporters observing the same target at the same time must both persist", n) + } +} + +// TestCurrentDeclaredRegionsNormalizesTargetCase proves CurrentDeclaredRegions +// lowercases its target argument before querying — mirroring every write +// path's normalization. Without it, a target pasted in uppercase (e.g. from a +// URL) silently returns zero rows and no error, exactly the sibling failure +// this table's read query must not repeat. +func TestCurrentDeclaredRegionsNormalizesTargetCase(t *testing.T) { + s := newTestStore(t) + if _, err := s.InsertClientDeclaredRegions(&ClientDeclaredRegions{ + Target: "bb11", RxPubkey: "aa11", + ObservedAt: "2026-08-18T10:00:00.000Z", IngestedAt: "2026-08-18T10:00:01.000Z", RegionsCSV: "be", + }); err != nil { + t.Fatal(err) + } + + cur, err := s.CurrentDeclaredRegions("BB11") + if err != nil { + t.Fatal(err) + } + if cur == nil { + t.Fatal("current = nil, want a hit for uppercase target BB11") + } + if cur.RegionsCSV != "be" { + t.Errorf("current.RegionsCSV = %q, want %q", cur.RegionsCSV, "be") + } +} + +// TestPruneClientDeclaredRegionsUsesIndex pins the retention DELETE to an +// index seek on idx_ndr_prune rather than a full table scan. +func TestPruneClientDeclaredRegionsUsesIndex(t *testing.T) { + s := newTestStore(t) + if testBackendValue() == "postgres" { + if _, err := s.db.Exec(`SET enable_seqscan=off`); err != nil { + t.Fatal(err) + } + } + rows, err := s.db.Query(testNativeSQL(`EXPLAIN QUERY PLAN DELETE FROM node_declared_regions WHERE observed_at < $1`, `EXPLAIN (COSTS OFF) DELETE FROM node_declared_regions WHERE observed_at < $1`), "2026-01-01T00:00:00Z") + if err != nil { + t.Fatal(err) + } + defer rows.Close() + plan := "" + for rows.Next() { + var detail string + if err := scanTestPlan(rows, &detail); err != nil { + t.Fatal(err) + } + plan += detail + "\n" + } + if !strings.Contains(plan, "idx_ndr_prune") { + t.Fatalf("retention DELETE should use idx_ndr_prune, plan was:\n%s", plan) + } +} + +// TestPruneOldClientDeclaredRegions verifies the declared-regions reaper +// deletes rows older than the window and keeps recent ones, and that days=0 +// disables it. It only exercises the days-to-cutoff arithmetic (a row well +// outside vs. well inside the window) — the millisecond-precision boundary +// that distinguishes a correct cutoff from an RFC3339 one is covered +// separately by TestPruneOldClientDeclaredRegionsAtMillisecondBoundary, which +// pins its own cutoff instant instead of deriving one from time.Now(); see +// that test's comment for why this one deliberately does not attempt it. +func TestPruneOldClientDeclaredRegions(t *testing.T) { + s := newTestStore(t) + now := time.Now().UTC() + recent := now.AddDate(0, 0, -1).Format(rxTimeMillisLayout) + old := now.AddDate(0, 0, -40).Format(rxTimeMillisLayout) + + mk := func(observedAt string) *ClientDeclaredRegions { + return &ClientDeclaredRegions{ + Target: "bb11", RxPubkey: "aa11", ObservedAt: observedAt, IngestedAt: observedAt, + RegionsCSV: "be", + } + } + if _, err := s.InsertClientDeclaredRegions(mk(recent)); err != nil { + t.Fatal(err) + } + if _, err := s.InsertClientDeclaredRegions(mk(old)); err != nil { + t.Fatal(err) + } + + if n, _ := s.PruneOldClientDeclaredRegions(0); n != 0 { + t.Fatalf("days=0 must be a no-op, got %d", n) + } + n, err := s.PruneOldClientDeclaredRegions(7) + if err != nil { + t.Fatal(err) + } + if n != 1 { + t.Fatalf("expected 1 old row pruned, got %d", n) + } + if got := declaredRegionsCount(t, s); got != 1 { + t.Fatalf("expected 1 row remaining (recent), got %d", got) + } +} + +// TestPruneOldClientDeclaredRegionsAtMillisecondBoundary proves the pruner's +// cutoff comparison distinguishes a millisecond-precision stored value from +// an RFC3339 (whole-second) cutoff: observed_at is written via +// rxTimeMillisLayout, and SQLite compares it lexicographically as a string, +// so a cutoff formatted without the ".mmm" component sorts AFTER a +// chronologically-newer row that has one (since '.' is 0x2E and 'Z' is +// 0x5A) and would wrongly delete it. +// +// It calls the unexported pruneOldClientDeclaredRegionsAt seam with a cutoff +// instant the test controls, rather than deriving one from time.Now() the +// way PruneOldClientDeclaredRegions(days) does. That is deliberate: the +// obvious construction — take "now", subtract N days via AddDate (which +// shifts only the calendar date), then add a sub-second offset — inherits +// whatever sub-second component "now" happened to have. If that leftover +// fraction is e.g. 700ms, adding another 500ms rolls into the next second +// and the boundary row stops testing what it claims to (it would survive +// under the buggy RFC3339 cutoff too, so the assertion passes either way). +// Truncating "now" first avoids that but reintroduces the opposite failure: +// the boundary can then land BEFORE the real cutoff and the test fails +// against correct code. That is the same flake already fixed once on the +// sibling client_rf_samples table (~50% of runs, in one direction or the +// other depending on which fix is chosen). Pinning both the cutoff and the +// offset removes time.Now() from the test entirely, so the result no longer +// depends on wall-clock phase. +func TestPruneOldClientDeclaredRegionsAtMillisecondBoundary(t *testing.T) { + s := newTestStore(t) + cutoffInstant := time.Date(2026, 8, 4, 10, 0, 0, 0, time.UTC).Add(400 * time.Millisecond) + before := cutoffInstant.Add(-1 * time.Millisecond).Format(rxTimeMillisLayout) + boundary := cutoffInstant.Add(1 * time.Millisecond).Format(rxTimeMillisLayout) + + mk := func(observedAt string) *ClientDeclaredRegions { + return &ClientDeclaredRegions{ + Target: "bb11", RxPubkey: "aa11", ObservedAt: observedAt, IngestedAt: observedAt, + RegionsCSV: "be", + } + } + if _, err := s.InsertClientDeclaredRegions(mk(before)); err != nil { + t.Fatal(err) + } + if _, err := s.InsertClientDeclaredRegions(mk(boundary)); err != nil { + t.Fatal(err) + } + + n, err := s.pruneOldClientDeclaredRegionsAt(cutoffInstant) + if err != nil { + t.Fatal(err) + } + if n != 1 { + t.Fatalf("expected 1 row pruned (1ms before the cutoff), got %d", n) + } + var boundarySurvived int + s.db.QueryRow(`SELECT COUNT(*) FROM node_declared_regions WHERE observed_at = $1`, boundary).Scan(&boundarySurvived) + if boundarySurvived != 1 { + t.Fatalf("boundary row (1ms after a cutoff with a non-zero millisecond component) must survive; an RFC3339 (no-ms) cutoff would wrongly delete it because '.' < 'Z' lexicographically — got %d", boundarySurvived) + } +} + +// TestHandleClientRegionsTrimsBlockPadding proves the ingestor strips the AES +// block padding a repeater's reply carries, whatever the client version sent it. +// coredrive-rx trims it since v1.10.2, but a PWA runs its cached build until the +// user reloads and this instance controls none of the independent clients — the +// padding was observed arriving from a client 40 minutes AFTER that release. +// Stored verbatim it lands inside the LAST region name, so "be-vli\x00\x00" matches +// no observed scope and reads as a region declared but never forwarded. +func TestHandleClientRegionsTrimsBlockPadding(t *testing.T) { + s := newTestStore(t) + msg := map[string]interface{}{ + "timestamp": "2026-08-19T10:00:00.000Z", + "target": testTargetPK, + "regions": []interface{}{"*", "be", "be-vli\x00\x00\x00"}, + } + handleClientRegions(s, &Config{}, "test", testCompanionPK, msg) + + cur, err := s.CurrentDeclaredRegions(testTargetPK) + if err != nil { + t.Fatal(err) + } + if cur == nil { + t.Fatal("current = nil, want a row") + } + if cur.RegionsCSV != "*,be,be-vli" { + t.Errorf("regions_csv = %q, want %q", cur.RegionsCSV, "*,be,be-vli") + } +} + +// TestHandleClientRegionsPaddingOnlyEntryDropped: an entry that is nothing but +// padding is not a declared region and must not become an empty-named one. +func TestHandleClientRegionsPaddingOnlyEntryDropped(t *testing.T) { + s := newTestStore(t) + msg := map[string]interface{}{ + "timestamp": "2026-08-19T10:00:00.000Z", + "target": testTargetPK, + "regions": []interface{}{"*", "\x00\x00\x00\x00"}, + } + handleClientRegions(s, &Config{}, "test", testCompanionPK, msg) + + cur, err := s.CurrentDeclaredRegions(testTargetPK) + if err != nil { + t.Fatal(err) + } + if cur == nil { + t.Fatal("current = nil, want a row") + } + if cur.RegionsCSV != "*" { + t.Errorf("regions_csv = %q, want %q", cur.RegionsCSV, "*") + } +} diff --git a/cmd/ingestor/client_rf_sample.go b/cmd/ingestor/client_rf_sample.go index 46a8f5a26..ad307f112 100644 --- a/cmd/ingestor/client_rf_sample.go +++ b/cmd/ingestor/client_rf_sample.go @@ -134,7 +134,7 @@ func (s *Store) ClientRfDeltas(rxPubkey, from, to string) ([]ClientRfDelta, erro LAG(recv) OVER w AS prev_recv, LAG(recv_errors) OVER w AS prev_errs FROM client_rf_samples - WHERE rx_pubkey = ? AND sampled_at >= ? AND sampled_at < ? + WHERE rx_pubkey = `+s.parameter(1)+` AND sampled_at >= `+s.parameter(2)+` AND sampled_at < `+s.parameter(3)+` WINDOW w AS (ORDER BY sampled_at) ORDER BY sampled_at`, rxPubkey, from, to) if err != nil { diff --git a/cmd/ingestor/client_rf_sample_test.go b/cmd/ingestor/client_rf_sample_test.go index bbc0dc480..1aae0a194 100644 --- a/cmd/ingestor/client_rf_sample_test.go +++ b/cmd/ingestor/client_rf_sample_test.go @@ -5,10 +5,40 @@ import ( "time" ) +// rfFixtureTime returns a sample timestamp offset seconds from a base two +// hours in the past, in the layout the handler round-trips. +// +// The times have to be anchored to now, not to a calendar date. +// resolveRxTimeCore (main.go:1527) replaces any timestamp more than 30 days +// old with the ingest time, so a fixed fixture date stops round-tripping 30 +// days after it is written: the seeded rows land at time.Now() while the query +// window stays in the past, and every delta disappears. The 2026-08-17 +// fixtures this file used to carry did exactly that on 2026-09-16T10:00Z, +// which turned all three delta tests red on master and on every open PR. +// The base is computed once for the whole test binary, not per call: two +// calls straddling a wall-clock second boundary would otherwise sit 16 s +// apart instead of 15 s, and TestRfDeltaBreaksOnReboot asserts the exact +// WallMillis. Truncating to the second keeps the formatted values ending in +// ".000Z", the shape these fixtures have always had. +var rfFixtureBase = time.Now().UTC().Add(-2 * time.Hour).Truncate(time.Second) + +func rfFixtureTime(offset time.Duration) string { + return rfFixtureBase.Add(offset).Format(rxTimeMillisLayout) +} + +// rfFixtureWindow brackets the rfFixtureTime series. Bounds are compared +// lexicographically against millisecond-precision sampled_at values, so they +// must themselves be in the millisecond layout: a second-resolution +// "...:00:00Z" bound would lexicographically exclude "...:00:00.000Z", +// because `.` sorts before `Z`. +func rfFixtureWindow() (from, to string) { + return rfFixtureTime(-time.Hour), rfFixtureTime(time.Hour) +} + func TestHandleClientRfSample(t *testing.T) { s := newTestStore(t) msg := map[string]interface{}{ - "type": "RF_SAMPLE", "timestamp": "2026-08-17T10:00:00.000Z", + "type": "RF_SAMPLE", "timestamp": rfFixtureTime(0), "gps": map[string]interface{}{"lat": 51.2, "lon": 4.4, "acc_m": 8.0}, "stationary": false, "uptime_secs": 84213.0, "noise_floor": -119.0, "rx_air_secs": 20877.0, @@ -46,7 +76,7 @@ func TestHandleClientRfSampleRejects(t *testing.T) { s := newTestStore(t) base := func() map[string]interface{} { return map[string]interface{}{ - "timestamp": "2026-08-17T10:00:00.000Z", + "timestamp": rfFixtureTime(0), "gps": map[string]interface{}{"lat": 51.2, "lon": 4.4}, "uptime_secs": 1.0, } @@ -89,14 +119,12 @@ func TestRfDeltaBreaksOnReboot(t *testing.T) { "rx_air_secs": rxAir, }) } - seed("2026-08-17T10:00:00.000Z", 1000, 500) - seed("2026-08-17T10:00:15.000Z", 1015, 512) // +12 s of RX air over 15 s - seed("2026-08-17T10:00:30.000Z", 10, 3) // rebooted: uptime dropped + seed(rfFixtureTime(0), 1000, 500) + seed(rfFixtureTime(15*time.Second), 1015, 512) // +12 s of RX air over 15 s + seed(rfFixtureTime(30*time.Second), 10, 3) // rebooted: uptime dropped - // Bounds are compared lexicographically against millisecond-precision - // sampled_at values, so they must themselves be in the millisecond layout - // — "10:00:00Z" would lexicographically exclude "10:00:00.000Z" (`.` < `Z`). - deltas, err := s.ClientRfDeltas("aa11", "2026-08-17T00:00:00.000Z", "2026-08-18T00:00:00.000Z") + from, to := rfFixtureWindow() + deltas, err := s.ClientRfDeltas("aa11", from, to) if err != nil { t.Fatalf("deltas: %v", err) } @@ -126,10 +154,11 @@ func TestRfDeltaNilWhenEitherEndpointMissingCounter(t *testing.T) { handleClientRfSample(s, "test", "aa11", msg) } five := int64(5) - seed("2026-08-17T10:00:00.000Z", 1000, nil) // firmware without the counter - seed("2026-08-17T10:00:15.000Z", 1015, &five) + seed(rfFixtureTime(0), 1000, nil) // firmware without the counter + seed(rfFixtureTime(15*time.Second), 1015, &five) - deltas, err := s.ClientRfDeltas("aa11", "2026-08-17T00:00:00.000Z", "2026-08-18T00:00:00.000Z") + from, to := rfFixtureWindow() + deltas, err := s.ClientRfDeltas("aa11", from, to) if err != nil { t.Fatalf("deltas: %v", err) } @@ -156,10 +185,11 @@ func TestClientRfDeltasNormalizesPubkeyCase(t *testing.T) { "rx_air_secs": rxAir, }) } - seed("2026-08-17T10:00:00.000Z", 1000, 500) - seed("2026-08-17T10:00:15.000Z", 1015, 512) + seed(rfFixtureTime(0), 1000, 500) + seed(rfFixtureTime(15*time.Second), 1015, 512) - deltas, err := s.ClientRfDeltas("AA11", "2026-08-17T00:00:00.000Z", "2026-08-18T00:00:00.000Z") + from, to := rfFixtureWindow() + deltas, err := s.ClientRfDeltas("AA11", from, to) if err != nil { t.Fatalf("deltas: %v", err) } diff --git a/cmd/ingestor/config.go b/cmd/ingestor/config.go index fb1026532..657a3a393 100644 --- a/cmd/ingestor/config.go +++ b/cmd/ingestor/config.go @@ -8,6 +8,7 @@ import ( "os" "strings" "sync" + "time" "github.com/meshcore-analyzer/dbconfig" "github.com/meshcore-analyzer/geofilter" @@ -25,6 +26,8 @@ type MQTTSource struct { IATAFilter []string `json:"iataFilter,omitempty"` ConnectTimeoutSec int `json:"connectTimeoutSec,omitempty"` Region string `json:"region,omitempty"` + // ClientID is the MQTT ClientID; empty means corescope--. + ClientID string `json:"clientId,omitempty"` } // ConnectTimeoutOrDefault returns the per-source connect timeout in seconds, @@ -42,8 +45,10 @@ type MQTTLegacy struct { Topic string `json:"topic"` } -// Config holds the ingestor configuration, compatible with the Node.js config.json format. +// Config holds the shared Go-service configuration and ingestor-only settings. type Config struct { + DatabaseURL string `json:"databaseURL"` + StateDir string `json:"stateDir"` DBPath string `json:"dbPath"` MQTT *MQTTLegacy `json:"mqtt,omitempty"` MQTTSources []MQTTSource `json:"mqttSources,omitempty"` @@ -52,12 +57,14 @@ type Config struct { ChannelKeys map[string]string `json:"channelKeys,omitempty"` HashChannels []string `json:"hashChannels,omitempty"` HashRegions []string `json:"hashRegions,omitempty"` + AutoRegionKeys *AutoRegionKeysConfig `json:"autoRegionKeys,omitempty"` Retention *RetentionConfig `json:"retention,omitempty"` Metrics *MetricsConfig `json:"metrics,omitempty"` Runtime *RuntimeConfig `json:"runtime,omitempty"` ClientRxCoverage *ClientRxCoverageConfig `json:"clientRxCoverage,omitempty"` ClientRxObservations *ClientRxObservationsConfig `json:"clientRxObservations,omitempty"` ClientRfSamples *ClientRfSamplesConfig `json:"clientRfSamples,omitempty"` + ClientRegions *ClientRegionsConfig `json:"clientRegions,omitempty"` GeoFilter *GeoFilterConfig `json:"geo_filter,omitempty"` // PathTrust configures the minimum path-hash prefix length trusted as // mapping/topology evidence (issue #1784). @@ -76,6 +83,17 @@ type Config struct { obsIATAWhitelistCached map[string]bool obsIATAWhitelistOnce sync.Once + // IATAWarnIntervalSec throttles the one-line-per-region warning emitted when + // ObserverIATAWhitelist rejects a region. 0 => defaultIATAWarnIntervalSec. + IATAWarnIntervalSec int `json:"iataWarnIntervalSec,omitempty"` + + // iataWarnLast tracks when each dropped region was last logged. + iataWarnMu sync.Mutex + iataWarnLast map[string]time.Time + // iataWarnOverflowLast throttles the shared warning used once + // iataWarnLast has reached iataWarnMaxTracked. + iataWarnOverflowLast time.Time + // ObserverBlacklist is a list of observer public keys to drop at ingest. // Messages from blacklisted observers are silently discarded — no DB writes, // no UpsertObserver, no observations, no metrics. @@ -90,10 +108,15 @@ type Config struct { NeighborEdgesMaxAgeDays int `json:"neighborEdgesMaxAgeDays,omitempty"` // IngestBufferSize caps the in-memory queue (number of MQTT messages) held - // while the single SQLite writer is blocked by startup migrations/prunes + // while the single serialized writer is blocked by startup migrations/prunes // (#1608). Received messages are drained once the write path is ready. // 0 / unset => default. Bounded memory. IngestBufferSize int `json:"ingestBufferSize,omitempty"` + + // UserManagement is the part of the shared userManagement block the + // ingestor reads: where users.db is and whether approved hashtag channels + // are decrypted. Everything else in the block belongs to the server. + UserManagement *UserManagementConfig `json:"userManagement,omitempty"` } // NeighborEdgesDaysOrDefault returns the configured pruning window or 5. @@ -173,6 +196,17 @@ func (c *Config) ClientRfSamplesEnabled() bool { return c.ClientRfSamples != nil && c.ClientRfSamples.Enabled } +// ClientRegionsConfig controls the opt-in region-discovery answer stream. +type ClientRegionsConfig struct { + Enabled bool `json:"enabled"` +} + +// ClientRegionsEnabled reports whether declared-region answers are recorded. +// Default false. +func (c *Config) ClientRegionsEnabled() bool { + return c.ClientRegions != nil && c.ClientRegions.Enabled +} + // RetentionConfig controls how long stale nodes are kept before being moved to inactive_nodes. type RetentionConfig struct { NodeDays int `json:"nodeDays"` @@ -199,6 +233,11 @@ type RetentionConfig struct { // client_rf_samples table; 0 disables. Bounds the table the opt-in RF // sample stream would otherwise grow without limit. ClientRfDays int `json:"clientRfDays"` + // ClientRegionsDays is the retention window (by observed_at) for the + // node_declared_regions table; 0 disables. Bounds the table the opt-in + // region-discovery stream would otherwise grow without limit. Consumed by + // Task 6's pruner. + ClientRegionsDays int `json:"clientRegionsDays"` } // PacketDaysOrZero returns the configured retention.packetDays or 0 @@ -244,6 +283,15 @@ func (c *Config) ClientRfDaysOrZero() int { return 0 } +// ClientRegionsDaysOrZero returns retention.clientRegionsDays or 0 +// (disabled) if not set. +func (c *Config) ClientRegionsDaysOrZero() int { + if c.Retention != nil && c.Retention.ClientRegionsDays > 0 { + return c.Retention.ClientRegionsDays + } + return 0 +} + // MetricsConfig controls observer metrics collection. type MetricsConfig struct { SampleIntervalSec int `json:"sampleIntervalSec"` @@ -258,10 +306,10 @@ type RuntimeConfig struct { MaxMemoryMB int `json:"maxMemoryMB"` } -// DBConfig is the shared SQLite vacuum/maintenance config (#919, #921). +// DBConfig shares startup-load settings and accepts obsolete SQLite config keys. type DBConfig = dbconfig.DBConfig -// IncrementalVacuumPages returns the configured pages per vacuum or 1024 default. +// IncrementalVacuumPages preserves legacy config decoding; PostgreSQL does not use it. func (c *Config) IncrementalVacuumPages() int { if c.DB != nil && c.DB.IncrementalVacuumPages > 0 { return c.DB.IncrementalVacuumPages @@ -269,6 +317,15 @@ func (c *Config) IncrementalVacuumPages() int { return 1024 } +// AnalysisLimit preserves the old SQLite setting for configuration compatibility. +// PostgreSQL owns planner statistics; runtime maintenance does not use this limit. +func (c *Config) AnalysisLimit() int { + if c.DB != nil && c.DB.AnalysisLimit != 0 { + return c.DB.AnalysisLimit + } + return 10000 +} + // ShouldValidateSignatures returns true (default) unless explicitly disabled. func (c *Config) ShouldValidateSignatures() bool { if c.ValidateSignatures != nil { @@ -356,6 +413,92 @@ func (c *Config) IsObserverIATAAllowed(iata string) bool { return c.obsIATAWhitelistCached[strings.ToUpper(strings.TrimSpace(iata))] } +// iataWarnMaxTracked bounds the per-region throttle map. The key is +// publisher-controlled, so it cannot be allowed to grow with traffic. 512 is +// far above any real deployment's region count (the reference instance has 43 +// observers across a handful of regions) and small enough that a hostile feed +// buys nothing: the map stops growing and the warning keeps coming. +const iataWarnMaxTracked = 512 + +// defaultIATAWarnIntervalSec is the re-log interval for whitelist drops (6h). +const defaultIATAWarnIntervalSec = 21600 + +// IATAWarnInterval returns how often a dropped region is re-logged. +func (c *Config) IATAWarnInterval() time.Duration { + if c == nil || c.IATAWarnIntervalSec <= 0 { + return defaultIATAWarnIntervalSec * time.Second + } + return time.Duration(c.IATAWarnIntervalSec) * time.Second +} + +// ShouldWarnIATADrop reports whether a whitelist drop for this region should be +// logged now, recording the decision when it returns true. +// +// Logging every dropped message is not an option: a foreign feed runs to +// thousands of messages a day and would flood the container log. But dropping +// in silence is worse — an allow-list fails in the dangerous direction, where a +// legitimate but unlisted region simply vanishes with nothing to show for it. +// So: one line per region, re-logged at most every IATAWarnInterval for as long +// as that region keeps arriving. The re-log is deliberate — a strict log-once +// would emit a single edge event that any scrape window eventually rolls past, +// leaving an actively-dropping region looking identical to a healthy one. +func (c *Config) ShouldWarnIATADrop(iata string) bool { + if c == nil { + return false + } + code := strings.ToUpper(strings.TrimSpace(iata)) + if code == "" { + return false + } + interval := c.IATAWarnInterval() + now := time.Now() + + c.iataWarnMu.Lock() + defer c.iataWarnMu.Unlock() + if last, ok := c.iataWarnLast[code]; ok && now.Sub(last) < interval { + return false + } + if c.iataWarnLast == nil { + c.iataWarnLast = make(map[string]time.Time) + } + // The key comes from a topic segment the publisher controls, so an + // unbounded map here is a remote memory sink: 200k distinct codes retained + // 200k entries and 15.1 MB of heap when this was measured. Nothing in the + // codebase constrains an IATA code's shape (it is uppercased and trimmed, + // never validated), so rejecting by shape would invent a rule operators + // have not agreed to. Bound the map instead. + // + // Past the cap the drop is still logged, throttled on one shared timestamp + // rather than a per-code one. That keeps the flood-protection the feature + // exists for while making the overflow itself visible: silently dropping + // the warning would reintroduce the bug this PR fixes, one level up. + if len(c.iataWarnLast) >= iataWarnMaxTracked { + // Sweep first. An entry older than the interval holds nothing back: + // the code would be re-logged on its next sighting anyway, so dropping + // it changes no behaviour and frees the slot. Without this the first + // iataWarnMaxTracked codes ever seen own the map forever, and a + // legitimate region that starts arriving later is stuck sharing the + // overflow throttle with whatever transient junk got there first. + // Only runs at the cap, so the normal path pays nothing. + for k, t := range c.iataWarnLast { + if now.Sub(t) >= interval { + delete(c.iataWarnLast, k) + } + } + } + if len(c.iataWarnLast) >= iataWarnMaxTracked { + if _, known := c.iataWarnLast[code]; !known { + if now.Sub(c.iataWarnOverflowLast) < interval { + return false + } + c.iataWarnOverflowLast = now + return true + } + } + c.iataWarnLast[code] = now + return true +} + // LoadConfig reads configuration from a JSON file, with env var overrides. // If the config file does not exist, sensible defaults are used (zero-config startup). func LoadConfig(path string) (*Config, error) { @@ -391,10 +534,23 @@ func LoadConfig(path string) (*Config, error) { }} } - // Default DB path - if cfg.DBPath == "" { - cfg.DBPath = "data/meshcore.db" + if v := os.Getenv("CORESCOPE_DATABASE_URL"); v != "" { + cfg.DatabaseURL = v + } + if v := os.Getenv("CORESCOPE_WRITER_DATABASE_URL"); v != "" { + cfg.DatabaseURL = v + } + if v := os.Getenv("CORESCOPE_STATE_DIR"); v != "" { + cfg.StateDir = v } + if v := os.Getenv("CORESCOPE_APPROVED_CHANNELS_DATABASE_URL"); v != "" { + if cfg.UserManagement == nil { + cfg.UserManagement = &UserManagementConfig{} + } + cfg.UserManagement.ApprovedChannelsDatabaseURL = v + } + // Legacy DBPath is never interpreted as a connection string. + // Default DB path // Normalize: convert legacy single mqtt config to mqttSources if len(cfg.MQTTSources) == 0 && cfg.MQTT != nil && cfg.MQTT.Broker != "" { @@ -440,3 +596,101 @@ func (c *Config) ResolvedSources() []MQTTSource { } return c.MQTTSources } + +// AutoRegionKeysConfig controls the opt-in derivation of region keys from the +// names repeaters declare over RF (node_declared_regions), on top of the +// explicit hashRegions list. +// +// TOP-LEVEL BLOCK, a sibling of hashRegions — not nested inside it. Config +// loading is plain json.Unmarshal with no DisallowUnknownFields, so a +// mis-nested key is silently ignored and derivation stays off with no error, +// the same trap clientRxObservations documents. +type AutoRegionKeysConfig struct { + Enabled bool `json:"enabled"` + MaxDerived int `json:"maxDerived"` + RefreshMinutes int `json:"refreshMinutes"` +} + +// autoRegionKeysDefaultMaxDerived bounds the derived tier. Every added key +// raises the random ambiguity rate by 1/65536 per scoped packet (code1 is two +// bytes) and costs one more HMAC per transport-scoped packet — the match is +// O(keys) and cannot be indexed, because the code depends on the payload. 256 +// on top of a typical explicit set puts the ambiguity rate near 0.5%, which is +// the ceiling this design accepts. +const autoRegionKeysDefaultMaxDerived = 256 + +// autoRegionKeysDefaultRefreshMinutes is how often the derived tier is rebuilt +// from the database. Declared-region answers arrive at human pace (a drive-by +// with a companion app, or a 24h observer report), so minutes-scale staleness +// is irrelevant and a tighter interval only burns queries. +const autoRegionKeysDefaultRefreshMinutes = 15 + +// UserManagementConfig is the ingestor's view of userManagement. +type UserManagementConfig struct { + DatabaseURL string `json:"databaseURL,omitempty"` + Enabled bool `json:"enabled"` + ApprovedChannelsDatabaseURL string `json:"approvedChannelsDatabaseURL,omitempty"` + DBPath string `json:"dbPath,omitempty"` + ChannelProposals *ChannelProposalsConfig `json:"channelProposals,omitempty"` +} + +// ChannelProposalsConfig is the ingestor's view of +// userManagement.channelProposals. +type ChannelProposalsConfig struct { + Enabled bool `json:"enabled"` + MaxApproved int `json:"maxApproved,omitempty"` +} + +const defaultMaxApprovedChannels = 128 + +// ApprovedChannelsEnabled reports whether approved hashtag channels from +// users.db are decrypted: user management and channel proposals both on. +func (c *Config) ApprovedChannelsEnabled() bool { + u := c.UserManagement + return u != nil && u.Enabled && u.ChannelProposals != nil && u.ChannelProposals.Enabled +} + +// ApprovedChannelsMax caps the approved keys read from users.db; absent, +// zero or negative means the default, as on the server. +func (c *Config) ApprovedChannelsMax() int { + u := c.UserManagement + if u == nil || u.ChannelProposals == nil || u.ChannelProposals.MaxApproved <= 0 { + return defaultMaxApprovedChannels + } + return u.ChannelProposals.MaxApproved +} + +// ApprovedChannelsURL is a separate restricted account-reader credential. +func (c *Config) ApprovedChannelsURL() string { + if c.UserManagement == nil { + return "" + } + return strings.TrimSpace(c.UserManagement.ApprovedChannelsDatabaseURL) +} + +// AutoRegionKeysEnabled reports whether region keys may be derived from +// declared-region answers. Default false. +func (c *Config) AutoRegionKeysEnabled() bool { + return c.AutoRegionKeys != nil && c.AutoRegionKeys.Enabled +} + +// AutoRegionKeysMaxDerived returns the derived-tier cap, falling back to the +// default for absent, zero, or negative values — a zero is indistinguishable +// from "key omitted" after json.Unmarshal, and neither should silently mean +// "derive nothing" when the operator has switched the feature on. +func (c *Config) AutoRegionKeysMaxDerived() int { + if c.AutoRegionKeys == nil || c.AutoRegionKeys.MaxDerived <= 0 { + return autoRegionKeysDefaultMaxDerived + } + return c.AutoRegionKeys.MaxDerived +} + +// AutoRegionKeysRefreshMinutes returns the refresh interval in minutes, +// falling back to the default for absent, zero, or negative values — a zero +// here would otherwise panic time.NewTicker. +func (c *Config) AutoRegionKeysRefreshMinutes() int { + if c.AutoRegionKeys == nil || c.AutoRegionKeys.RefreshMinutes <= 0 { + return autoRegionKeysDefaultRefreshMinutes + } + return c.AutoRegionKeys.RefreshMinutes +} diff --git a/cmd/ingestor/config_test.go b/cmd/ingestor/config_test.go index 24d0a1d2c..52657c9c0 100644 --- a/cmd/ingestor/config_test.go +++ b/cmd/ingestor/config_test.go @@ -40,8 +40,8 @@ func TestLoadConfigMissingFile(t *testing.T) { if err != nil { t.Fatalf("missing config should not error (zero-config mode), got: %v", err) } - if cfg.DBPath != "data/meshcore.db" { - t.Errorf("dbPath=%s, want data/meshcore.db", cfg.DBPath) + if cfg.DBPath != "" || cfg.DatabaseURL != "" || cfg.StateDir != "" { + t.Errorf("unexpected database/state defaults") } // Should default to localhost MQTT if len(cfg.MQTTSources) != 1 { @@ -193,8 +193,8 @@ func TestLoadConfigDefaultDBPath(t *testing.T) { if err != nil { t.Fatal(err) } - if cfg.DBPath != "data/meshcore.db" { - t.Errorf("dbPath=%s, want data/meshcore.db", cfg.DBPath) + if cfg.DBPath != "" || cfg.DatabaseURL != "" || cfg.StateDir != "" { + t.Errorf("unexpected database/state defaults") } } @@ -549,3 +549,18 @@ func TestGetPathTrustNilConfig(t *testing.T) { t.Errorf("expected default %d for nil *Config, got %d", packetpath.DefaultMinHashBytesForMapping, pt.MinHashBytesForMapping) } } + +func TestClientRegionsDaysOrZero(t *testing.T) { + var c Config + if got := c.ClientRegionsDaysOrZero(); got != 0 { + t.Errorf("unset retention = %d, want 0", got) + } + c.Retention = &RetentionConfig{ClientRegionsDays: 21} + if got := c.ClientRegionsDaysOrZero(); got != 21 { + t.Errorf("retention = %d, want 21", got) + } + c.Retention = &RetentionConfig{ClientRegionsDays: 0} + if got := c.ClientRegionsDaysOrZero(); got != 0 { + t.Errorf("retention=0 = %d, want 0", got) + } +} diff --git a/cmd/ingestor/coverage_boost_test.go b/cmd/ingestor/coverage_boost_test.go index fc2daa889..33d86958c 100644 --- a/cmd/ingestor/coverage_boost_test.go +++ b/cmd/ingestor/coverage_boost_test.go @@ -635,7 +635,9 @@ func TestDecryptChannelMessageBadCiphertextHex(t *testing.T) { func TestCheckpointDoesNotPanic(t *testing.T) { store := newTestStore(t) - store.Checkpoint() + if _, err := store.db.Exec(`CHECKPOINT`); err == nil { + t.Fatal("runtime role may force server checkpoints") + } } func TestLogStatsDoesNotPanic(t *testing.T) { @@ -762,7 +764,7 @@ func TestOpenStoreInvalidPath(t *testing.T) { t.Fatalf("setup: %v", err) } f.Close() - _, err = OpenStore(filepath.Join(f.Name(), "db.sqlite")) + _, err = openPostgresTestStore(t, filepath.Join(f.Name(), "db.sqlite")) if err == nil { t.Error("should error on impossible path") } @@ -1025,28 +1027,28 @@ func TestApplySchemaMigrationsOnFreshDB(t *testing.T) { // Check that migrations were recorded var count int - if err := store.db.QueryRow("SELECT COUNT(*) FROM _migrations").Scan(&count); err != nil { + if err := store.db.QueryRow(testNativeSQL(`SELECT COUNT(*) FROM _migrations WHERE name='observers_identity_autoincrement_v1'`, "SELECT COUNT(*) FROM corescope_schema WHERE kind='telemetry' AND ready=true")).Scan(&count); err != nil { t.Fatal(err) } - if count < 3 { - t.Errorf("expected at least 3 migrations recorded, got %d", count) + if count != 1 { + t.Errorf("expected one ready telemetry schema, got %d", count) } // Check observations table exists with dedup index var tblName string - err := store.db.QueryRow("SELECT name FROM sqlite_master WHERE type='table' AND name='observations'").Scan(&tblName) + err := store.db.QueryRow(testNativeSQL(`SELECT name FROM sqlite_master WHERE type='table' AND name='observations'`, "SELECT table_name FROM information_schema.tables WHERE table_schema=current_schema() AND table_name='observations'")).Scan(&tblName) if err != nil { t.Error("observations table should exist") } // Check inactive_nodes table exists - err = store.db.QueryRow("SELECT name FROM sqlite_master WHERE type='table' AND name='inactive_nodes'").Scan(&tblName) + err = store.db.QueryRow(testNativeSQL(`SELECT name FROM sqlite_master WHERE type='table' AND name='inactive_nodes'`, "SELECT table_name FROM information_schema.tables WHERE table_schema=current_schema() AND table_name='inactive_nodes'")).Scan(&tblName) if err != nil { t.Error("inactive_nodes table should exist") } // Check packets_v view exists - err = store.db.QueryRow("SELECT name FROM sqlite_master WHERE type='view' AND name='packets_v'").Scan(&tblName) + err = store.db.QueryRow(testNativeSQL(`SELECT name FROM sqlite_master WHERE type='view' AND name='packets_v'`, "SELECT table_name FROM information_schema.views WHERE table_schema=current_schema() AND table_name='packets_v'")).Scan(&tblName) if err != nil { t.Error("packets_v view should exist") } @@ -1058,14 +1060,14 @@ func TestOpenStoreExistingDB(t *testing.T) { dbPath := dir + "/test.db" // Open and close - s1, err := OpenStore(dbPath) + s1, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatal(err) } s1.Close() // Re-open — should skip migrations (already applied) - s2, err := OpenStore(dbPath) + s2, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatal(err) } @@ -1078,7 +1080,7 @@ func TestMoveStaleNodesReplace(t *testing.T) { pk := "stale_node_replace_0000000000000000000000000001" // Insert into inactive_nodes first - if _, err := store.db.Exec("INSERT INTO inactive_nodes (public_key, name, role, last_seen, first_seen) VALUES (?, 'Old', 'companion', '2019-01-01T00:00:00Z', '2019-01-01T00:00:00Z')", pk); err != nil { + if _, err := store.db.Exec("INSERT INTO inactive_nodes (public_key, name, role, last_seen, first_seen) VALUES ($1, 'Old', 'companion', '2019-01-01T00:00:00Z', '2019-01-01T00:00:00Z')", pk); err != nil { t.Fatal(err) } @@ -1095,7 +1097,7 @@ func TestMoveStaleNodesReplace(t *testing.T) { // Should have replaced the inactive node var name string - if err := store.db.QueryRow("SELECT name FROM inactive_nodes WHERE public_key = ?", pk).Scan(&name); err != nil { + if err := store.db.QueryRow("SELECT name FROM inactive_nodes WHERE public_key = $1", pk).Scan(&name); err != nil { t.Fatal(err) } if name != "StaleNode" { @@ -1165,7 +1167,7 @@ func TestRemoveStaleObservers(t *testing.T) { } // Override last_seen to 30 days ago cutoff := time.Now().UTC().AddDate(0, 0, -30).Format(time.RFC3339) - _, err = store.db.Exec("UPDATE observers SET last_seen = ? WHERE id = ?", cutoff, "obs-old") + _, err = store.db.Exec("UPDATE observers SET last_seen = $1 WHERE id = $2", cutoff, "obs-old") if err != nil { t.Fatal(err) } @@ -1195,7 +1197,7 @@ func TestRemoveStaleObservers(t *testing.T) { // Check that the old observer is marked inactive var inactive int - if err := store.db.QueryRow("SELECT inactive FROM observers WHERE id = ?", "obs-old").Scan(&inactive); err != nil { + if err := store.db.QueryRow("SELECT inactive FROM observers WHERE id = $1", "obs-old").Scan(&inactive); err != nil { t.Fatal(err) } if inactive != 1 { @@ -1204,7 +1206,7 @@ func TestRemoveStaleObservers(t *testing.T) { // Check that the recent observer is still active var newInactive int - if err := store.db.QueryRow("SELECT inactive FROM observers WHERE id = ?", "obs-new").Scan(&newInactive); err != nil { + if err := store.db.QueryRow("SELECT inactive FROM observers WHERE id = $1", "obs-new").Scan(&newInactive); err != nil { t.Fatal(err) } if newInactive != 0 { @@ -1233,7 +1235,7 @@ func TestRemoveStaleObserversKeepForever(t *testing.T) { t.Fatal(err) } cutoff := time.Now().UTC().AddDate(0, 0, -365).Format(time.RFC3339) - _, err = store.db.Exec("UPDATE observers SET last_seen = ? WHERE id = ?", cutoff, "obs-ancient") + _, err = store.db.Exec("UPDATE observers SET last_seen = $1 WHERE id = $2", cutoff, "obs-ancient") if err != nil { t.Fatal(err) } @@ -1257,7 +1259,7 @@ func TestRemoveStaleObserversKeepForever(t *testing.T) { // Observer should NOT be marked inactive var inactive int - if err := store.db.QueryRow("SELECT inactive FROM observers WHERE id = ?", "obs-ancient").Scan(&inactive); err != nil { + if err := store.db.QueryRow("SELECT inactive FROM observers WHERE id = $1", "obs-ancient").Scan(&inactive); err != nil { t.Fatal(err) } if inactive != 0 { @@ -1274,7 +1276,7 @@ func TestRemoveStaleObserversReactivation(t *testing.T) { t.Fatal(err) } cutoff := time.Now().UTC().AddDate(0, 0, -30).Format(time.RFC3339) - _, err = store.db.Exec("UPDATE observers SET last_seen = ? WHERE id = ?", cutoff, "obs-test") + _, err = store.db.Exec("UPDATE observers SET last_seen = $1 WHERE id = $2", cutoff, "obs-test") if err != nil { t.Fatal(err) } @@ -1289,7 +1291,7 @@ func TestRemoveStaleObserversReactivation(t *testing.T) { // Verify it's inactive var inactive int - if err := store.db.QueryRow("SELECT inactive FROM observers WHERE id = ?", "obs-test").Scan(&inactive); err != nil { + if err := store.db.QueryRow("SELECT inactive FROM observers WHERE id = $1", "obs-test").Scan(&inactive); err != nil { t.Fatal(err) } if inactive != 1 { @@ -1302,7 +1304,7 @@ func TestRemoveStaleObserversReactivation(t *testing.T) { t.Fatal(err) } - if err := store.db.QueryRow("SELECT inactive FROM observers WHERE id = ?", "obs-test").Scan(&inactive); err != nil { + if err := store.db.QueryRow("SELECT inactive FROM observers WHERE id = $1", "obs-test").Scan(&inactive); err != nil { t.Fatal(err) } if inactive != 0 { diff --git a/cmd/ingestor/coverage_gate_test.go b/cmd/ingestor/coverage_gate_test.go index 0ba2f3cbb..4ef5eade2 100644 --- a/cmd/ingestor/coverage_gate_test.go +++ b/cmd/ingestor/coverage_gate_test.go @@ -91,7 +91,7 @@ func TestClientRxCoverageGateOffDoesNotFallThroughToObserverPath(t *testing.T) { t.Fatalf("feature OFF: expected 0 client_receptions rows, got %d", n) } var observerRows int - if err := store.db.QueryRow(`SELECT COUNT(*) FROM observers WHERE id = ?`, testCompanionPK).Scan(&observerRows); err != nil { + if err := store.db.QueryRow(`SELECT COUNT(*) FROM observers WHERE id = $1`, testCompanionPK).Scan(&observerRows); err != nil { t.Fatal(err) } if observerRows != 0 { diff --git a/cmd/ingestor/coverage_query_bench_test.go b/cmd/ingestor/coverage_query_bench_test.go index 078ec8643..e8a928816 100644 --- a/cmd/ingestor/coverage_query_bench_test.go +++ b/cmd/ingestor/coverage_query_bench_test.go @@ -12,9 +12,9 @@ import ( // match on the heard node. const coverageBenchSQL = `SELECT lat, lon, snr, rssi, heard_key, rx_at FROM client_receptions - WHERE lat BETWEEN ? AND ? AND lon BETWEEN ? AND ? - AND ( (heard_keylen = 32 AND heard_key = ?) - OR (heard_keylen IN (2,3) AND substr(?, 1, heard_keylen*2) = heard_key) )` + WHERE lat BETWEEN $1 AND $2 AND lon BETWEEN $3 AND $4 + AND ( (heard_keylen = 32 AND heard_key = $5) + OR (heard_keylen IN (2,3) AND substr($6, 1, heard_keylen*2) = heard_key) )` // BenchmarkCoverageQuery seeds ~1M receptions across a metro-area bbox and times // the coverage query with the indexes (#5/#18) versus a forced full table scan. @@ -24,7 +24,7 @@ func BenchmarkCoverageQuery(b *testing.B) { const prefixPool = 2000 // distinct 3-byte heard_key prefixes dir := b.TempDir() - s, err := OpenStore(dir + "/bench.db") + s, err := openPostgresTestStore(b, dir+"/bench.db") if err != nil { b.Fatal(err) } @@ -37,7 +37,7 @@ func BenchmarkCoverageQuery(b *testing.B) { } stmt, err := tx.Prepare(`INSERT INTO client_receptions (rx_pubkey,heard_key,heard_keylen,snr,lat,lon,rx_at,ingested_at,src) - VALUES (?,?,?,?,?,?,?,?,?)`) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)`) if err != nil { b.Fatal(err) } @@ -78,7 +78,7 @@ func BenchmarkCoverageQuery(b *testing.B) { // full pubkey and its 2/3-byte prefixes, so an IN-list seeks them via the // heard_key-leading composite instead of scanning the bbox. inListSQL := `SELECT lat, lon, snr, rssi, heard_key, rx_at FROM client_receptions - WHERE heard_key IN (?,?,?) AND lat BETWEEN ? AND ? AND lon BETWEEN ? AND ?` + WHERE heard_key IN ($1,$2,$3) AND lat BETWEEN $4 AND $5 AND lon BETWEEN $6 AND $7` runIN := func(b *testing.B) { for i := 0; i < b.N; i++ { rows, err := s.db.Query(inListSQL, target, target[:4], target[:6], 51.1, 51.3, 3.6, 3.8) diff --git a/cmd/ingestor/db.go b/cmd/ingestor/db.go index f1eb92f3f..5f67b63b3 100644 --- a/cmd/ingestor/db.go +++ b/cmd/ingestor/db.go @@ -14,9 +14,11 @@ import ( "sync/atomic" "time" + _ "github.com/mattn/go-sqlite3" + "github.com/meshcore-analyzer/dbconfig" "github.com/meshcore-analyzer/dbschema" "github.com/meshcore-analyzer/packetpath" - _ "modernc.org/sqlite" + "github.com/meshcore-analyzer/pgutil" ) // DBStats tracks operational metrics for the ingestor database. @@ -65,17 +67,20 @@ func (s *DBStats) SnapshotBackfills() map[string]int64 { return out } -// Store wraps the SQLite database for packet ingestion. +// Store wraps the selected native database for packet ingestion. type Store struct { - db *sql.DB - path string // filesystem path to the SQLite DB (used to resolve queue dirs) - Stats DBStats + backend dbconfig.Backend + db *sql.DB + path string // compatibility anchor for local file queues; never a database URL + stateDir string + Stats DBStats stmtGetTxByHash *sql.Stmt stmtInsertTransmission *sql.Stmt stmtUpdateTxFirstSeen *sql.Stmt stmtBumpTxLastSeen *sql.Stmt stmtInsertObservation *sql.Stmt + stmtInsertAdvertEvidence *sql.Stmt stmtUpsertNode *sql.Stmt stmtIncrementAdvertCount *sql.Stmt stmtUpsertObserver *sql.Stmt @@ -85,9 +90,13 @@ type Store struct { stmtTouchNodeLastSeen *sql.Stmt stmtUpsertMetrics *sql.Stmt + stmtGetLegacyAdvertObservation *sql.Stmt + sampleIntervalSec int backfillWg sync.WaitGroup + advertEvidenceComplete atomic.Bool // restored from durable migration status + // prefixIdx holds the prefix → pubkey index used by the // resolved_path writer (#1547). Rebuilt on startup and once per // neighbor-edges builder tick (60s). @@ -117,772 +126,174 @@ const relayTouchDebounce = 5 * time.Minute // an extra UPDATE, never a missed one. const relayTouchedMaxEntries = 50000 -// OpenStore opens or creates a SQLite DB at the given path, applying the -// v3 schema that is compatible with the Node.js server. -func OpenStore(dbPath string) (*Store, error) { - return OpenStoreWithInterval(dbPath, 300) +// OpenStore preserves native path/URL callers; runtime uses selected Storage. +func OpenStore(databaseURL string) (*Store, error) { return OpenStoreWithInterval(databaseURL, 300) } +func OpenStoreWithInterval(databaseURL string, sampleIntervalSec int) (*Store, error) { + return OpenStoreWithState(databaseURL, "", sampleIntervalSec) } - -// OpenStoreWithInterval opens or creates a SQLite DB with a configurable sample interval. -func OpenStoreWithInterval(dbPath string, sampleIntervalSec int) (*Store, error) { - dir := filepath.Dir(dbPath) - if err := os.MkdirAll(dir, 0o755); err != nil { - return nil, fmt.Errorf("creating data dir: %w", err) - } - - db, err := sql.Open("sqlite", dbPath+"?_pragma=auto_vacuum(INCREMENTAL)&_pragma=journal_mode(WAL)&_pragma=foreign_keys(ON)&_pragma=busy_timeout(5000)") - if err != nil { - return nil, fmt.Errorf("opening db: %w", err) - } - - if err := db.Ping(); err != nil { - return nil, fmt.Errorf("pinging db: %w", err) - } - - db.SetMaxOpenConns(1) - db.SetMaxIdleConns(1) - log.Printf("SQLite config: busy_timeout=5000ms, max_open_conns=1, max_idle_conns=1, journal=WAL") - - if err := applySchema(db); err != nil { - return nil, fmt.Errorf("applying schema: %w", err) - } - - // Apply the additional server-originated migrations (now owned by - // the ingestor per #1287). Adds the indexes/columns that used to live - // in cmd/server/ensure_*.go: server now ASSERTS these exist. - if err := dbschema.Apply(db, log.Printf); err != nil { - return nil, fmt.Errorf("dbschema.Apply: %w", err) +func OpenStoreWithState(target, stateDir string, sampleIntervalSec int) (*Store, error) { + storage := dbconfig.Storage{Backend: dbconfig.SQLite, DBPath: target, StateDir: stateDir} + if strings.HasPrefix(target, "postgres:") || strings.HasPrefix(target, "postgresql:") { + storage = dbconfig.Storage{Backend: dbconfig.Postgres, WriterDatabaseURL: target, StateDir: stateDir} } - - s := &Store{db: db, path: dbPath, sampleIntervalSec: sampleIntervalSec} - if err := s.prepareStatements(); err != nil { - return nil, fmt.Errorf("preparing statements: %w", err) - } - - // Schedule async migrations. These must NOT block boot. See - // async_migration.go for the convention. - // PREFLIGHT: async=true reason="composite index build on observations (1.9M+ rows in prod) — converted from sync after v3.8.3" - var idxDone int - if s.db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'obs_observer_ts_idx_v1'").Scan(&idxDone) != nil { - if err := s.RunAsyncMigration(context.Background(), "obs_observer_ts_idx_v1", - func(ctx context.Context, d *sql.DB) error { - log.Println("[migration/async] Building (observer_idx, timestamp) composite index on observations...") - if _, err := d.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_observations_observer_idx_timestamp ON observations(observer_idx, timestamp)`); err != nil { - return err - } - if _, err := d.ExecContext(ctx, `INSERT OR IGNORE INTO _migrations (name) VALUES ('obs_observer_ts_idx_v1')`); err != nil { - return err - } - log.Println("[migration/async] observations(observer_idx, timestamp) index created") - return nil - }); err != nil { - log.Printf("[migration/async] scheduling obs_observer_ts_idx_v1 failed: %v", err) - } - } - - // #1690: backfill transmissions.last_seen from MAX(observations.timestamp) - // per transmission. The column is added inline by dbschema.Apply (cheap - // metadata-only ALTER); the populate query is potentially expensive - // (full obs scan + group) so we run it async. Subsequent observation - // inserts maintain the column inline (see InsertTransmission below). - // PREFLIGHT: async=true reason="full-table backfill JOIN (1.9M+ obs × 86k+ tx in prod) — must not block ingestor boot" - if err := s.RunAsyncMigration(context.Background(), "tx_last_seen_backfill_v1", - func(ctx context.Context, d *sql.DB) error { - log.Println("[migration/async] Backfilling transmissions.last_seen from MAX(observations.timestamp)...") - res, err := d.ExecContext(ctx, ` - UPDATE transmissions - SET last_seen = COALESCE(( - SELECT MAX(timestamp) FROM observations WHERE transmission_id = transmissions.id - ), last_seen) - WHERE last_seen = 0 - `) - if err != nil { - return err - } - n, _ := res.RowsAffected() - log.Printf("[migration/async] transmissions.last_seen backfill complete: %d rows updated", n) - return nil - }); err != nil { - log.Printf("[migration/async] scheduling tx_last_seen_backfill_v1 failed: %v", err) - } - - return s, nil + return openStore(storage, nil, sampleIntervalSec, true) } - -func applySchema(db *sql.DB) error { - // auto_vacuum=INCREMENTAL is set via DSN pragma (must be before journal_mode). - // Logging of current mode is handled by CheckAutoVacuum — no duplicate log here. - - schema := ` - CREATE TABLE IF NOT EXISTS nodes ( - public_key TEXT PRIMARY KEY, - name TEXT, - role TEXT, - lat REAL, - lon REAL, - last_seen TEXT, - first_seen TEXT, - advert_count INTEGER DEFAULT 0, - battery_mv INTEGER, - temperature_c REAL, - foreign_advert INTEGER DEFAULT 0 - ); - - CREATE TABLE IF NOT EXISTS observers ( - id TEXT PRIMARY KEY, - name TEXT, - iata TEXT, - last_seen TEXT, - first_seen TEXT, - packet_count INTEGER DEFAULT 0, - model TEXT, - firmware TEXT, - client_version TEXT, - radio TEXT, - battery_mv INTEGER, - uptime_secs INTEGER, - noise_floor REAL, - inactive INTEGER DEFAULT 0, - last_packet_at TEXT DEFAULT NULL, - clock_skew_seconds INTEGER DEFAULT NULL, - clock_skew_count_24h INTEGER DEFAULT 0, - clock_last_naive_at TEXT DEFAULT NULL, - can_relay INTEGER DEFAULT 1, - can_relay_seen INTEGER DEFAULT 0 - ); - - CREATE INDEX IF NOT EXISTS idx_nodes_last_seen ON nodes(last_seen); - CREATE INDEX IF NOT EXISTS idx_observers_last_seen ON observers(last_seen); - - CREATE TABLE IF NOT EXISTS inactive_nodes ( - public_key TEXT PRIMARY KEY, - name TEXT, - role TEXT, - lat REAL, - lon REAL, - last_seen TEXT, - first_seen TEXT, - advert_count INTEGER DEFAULT 0, - battery_mv INTEGER, - temperature_c REAL, - foreign_advert INTEGER DEFAULT 0 - ); - - CREATE INDEX IF NOT EXISTS idx_inactive_nodes_last_seen ON inactive_nodes(last_seen); - - CREATE TABLE IF NOT EXISTS transmissions ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - raw_hex TEXT NOT NULL, - hash TEXT NOT NULL UNIQUE, - first_seen TEXT NOT NULL, - route_type INTEGER, - payload_type INTEGER, - payload_version INTEGER, - decoded_json TEXT, - from_pubkey TEXT, - last_seen INTEGER NOT NULL DEFAULT 0, - created_at TEXT DEFAULT (datetime('now')), - code1 TEXT, - code2 TEXT - ); - - CREATE INDEX IF NOT EXISTS idx_transmissions_hash ON transmissions(hash); - CREATE INDEX IF NOT EXISTS idx_transmissions_first_seen ON transmissions(first_seen); - CREATE INDEX IF NOT EXISTS idx_transmissions_payload_type ON transmissions(payload_type); - -- idx_tx_code1 is created by the transport_codes_v1 migration below, - -- after the ALTER runs — same reasoning as idx_transmissions_from_pubkey - -- and idx_tx_last_seen_zero above: a legacy DB (table-exists, - -- column-missing) would trip on this CREATE INDEX referencing code1 - -- before the column exists, since CREATE TABLE IF NOT EXISTS is a - -- no-op against a pre-existing table. - -- idx_transmissions_from_pubkey is created by the from_pubkey_v1 - -- migration after the column is added on legacy DBs (#1143). - -- idx_tx_last_seen_zero (partial, WHERE last_seen=0) is created by - -- dbschema.Apply after ensuring the last_seen column exists (#1690, - -- partial-index swap #1740) — keep it OUT of this base schema block - -- so legacy DBs (table-exists, column-missing) don't trip on the - -- CREATE INDEX before the ALTER runs. - - -- Mobile client RX coverage: a roaming companion = a mobile observer - -- with a moving GPS position, so it gets its own table rather than - -- observations (which assumes a fixed observer/location). - CREATE TABLE IF NOT EXISTS client_receptions ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - rx_pubkey TEXT NOT NULL, - heard_key TEXT NOT NULL, - heard_keylen INTEGER NOT NULL, - rssi INTEGER, - snr REAL, - lat REAL NOT NULL, - lon REAL NOT NULL, - pos_acc_m REAL, - rx_at TEXT NOT NULL, - ingested_at TEXT NOT NULL, - src TEXT NOT NULL, - UNIQUE(rx_pubkey, heard_key, rx_at) - ); - -- Coverage queries filter by bbox AND match the heard node either by full - -- key (heard_keylen=32 AND heard_key=?) or by 2-3 byte prefix. The composite - -- (heard_key, heard_keylen, lat, lon) serves the heard_key-equality seek and - -- carries lat/lon so the bbox range is satisfied from the index; it also - -- supersedes the old single-column heard_key index. idx_client_recept_latlon - -- lets the planner instead drive from a selective bbox. (#5, #18) - CREATE INDEX IF NOT EXISTS idx_client_recept_heard_geo ON client_receptions(heard_key, heard_keylen, lat, lon); - CREATE INDEX IF NOT EXISTS idx_client_recept_latlon ON client_receptions(lat, lon); - -- rx_at backs both the retention reaper (DELETE WHERE rx_at < ?) and the - -- leaderboard, which range-scans WHERE rx_at >= ? and aggregates per - -- rx_pubkey in Go (see rxLeaderboard's frontier-weighted scoring). Without - -- this index either would full-scan the table under the writer lock - -- (verified by an EXPLAIN test). A dedicated rx_pubkey index stays - -- redundant — the leaderboard no longer groups by rx_pubkey in SQL. - CREATE INDEX IF NOT EXISTS idx_client_recept_rxat ON client_receptions(rx_at); - DROP INDEX IF EXISTS idx_client_recept_rxpk; - - -- Self-reported name of each mobile client (companion), from the SELF_INFO - -- name the app sends as "origin". Lets the leaderboard show a name even - -- when the companion never advertised (so it isn't in the nodes table). - CREATE TABLE IF NOT EXISTS client_observers ( - pubkey TEXT PRIMARY KEY, - name TEXT, - last_seen TEXT - ); - - -- Diagnostic RF observations from mobile clients. Unlike client_receptions - -- this holds EVERY decodable packet, attributable or not, so it must never - -- be used for coverage. pkt_hash is ComputeContentHash() — identical to - -- transmissions.hash — so dark-traffic queries are a plain equality join. - CREATE TABLE IF NOT EXISTS client_rx_observations ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - rx_pubkey TEXT NOT NULL, - rx_at TEXT NOT NULL, - ingested_at TEXT NOT NULL, - pkt_hash TEXT NOT NULL, - route_type INTEGER NOT NULL, - payload_type INTEGER NOT NULL, - code1 TEXT, - code2 TEXT, - scope_name TEXT, - hash_size INTEGER NOT NULL, - hop_count INTEGER NOT NULL, - path_json TEXT, - forwarder TEXT, - snr REAL, - rssi INTEGER, - lat REAL NOT NULL, - lon REAL NOT NULL, - pos_acc_m REAL, - UNIQUE(rx_pubkey, pkt_hash, rx_at) - ); - CREATE INDEX IF NOT EXISTS idx_cro_prune ON client_rx_observations(rx_at); - CREATE INDEX IF NOT EXISTS idx_cro_hash ON client_rx_observations(pkt_hash, rx_at); - CREATE INDEX IF NOT EXISTS idx_cro_forwarder ON client_rx_observations(forwarder, rx_at); - CREATE INDEX IF NOT EXISTS idx_cro_scope ON client_rx_observations(scope_name, rx_at); - - -- RF environment samples from mobile clients: radio counters paired with - -- a GPS point. Absolutes only — deltas are computed at query time, and a - -- decrease in uptime_secs (reboot) or any counter (wrap) breaks the chain. - CREATE TABLE IF NOT EXISTS client_rf_samples ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - rx_pubkey TEXT NOT NULL, - sampled_at TEXT NOT NULL, - ingested_at TEXT NOT NULL, - lat REAL NOT NULL, - lon REAL NOT NULL, - pos_acc_m REAL, - stationary INTEGER NOT NULL DEFAULT 0, - uptime_secs INTEGER NOT NULL, - battery_mv INTEGER, - queue_len INTEGER, - errors INTEGER, - noise_floor INTEGER, - last_rssi INTEGER, - last_snr REAL, - tx_air_secs INTEGER, - rx_air_secs INTEGER, - recv INTEGER, - sent INTEGER, - flood_rx INTEGER, - direct_rx INTEGER, - flood_tx INTEGER, - direct_tx INTEGER, - recv_errors INTEGER, - UNIQUE(rx_pubkey, sampled_at) - ); - CREATE INDEX IF NOT EXISTS idx_crf_prune ON client_rf_samples(sampled_at); - CREATE INDEX IF NOT EXISTS idx_crf_track ON client_rf_samples(rx_pubkey, sampled_at); - ` - if _, err := db.Exec(schema); err != nil { - return fmt.Errorf("base schema: %w", err) - } - - // Create observations table (v3 schema) - obsExists := false - row := db.QueryRow("SELECT name FROM sqlite_master WHERE type='table' AND name='observations'") - var dummy string - if row.Scan(&dummy) == nil { - obsExists = true - } - - if !obsExists { - obs := ` - CREATE TABLE observations ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - transmission_id INTEGER NOT NULL REFERENCES transmissions(id), - observer_idx INTEGER, - direction TEXT, - snr REAL, - rssi REAL, - score INTEGER, - path_json TEXT, - timestamp INTEGER NOT NULL - ); - CREATE INDEX idx_observations_transmission_id ON observations(transmission_id); - CREATE INDEX idx_observations_observer_idx ON observations(observer_idx); - CREATE INDEX idx_observations_timestamp ON observations(timestamp); - CREATE UNIQUE INDEX IF NOT EXISTS idx_observations_dedup ON observations(transmission_id, observer_idx, COALESCE(path_json, '')); - ` - if _, err := db.Exec(obs); err != nil { - return fmt.Errorf("observations schema: %w", err) +func openStoreStorage(storage dbconfig.Storage, cfg *DBConfig, sampleIntervalSec int) (*Store, error) { + return openStore(storage, cfg, sampleIntervalSec, false) +} +func openStore(storage dbconfig.Storage, cfg *DBConfig, sampleIntervalSec int, create bool) (*Store, error) { + var db *sql.DB + var err error + stateDir := storage.StateDir + switch storage.Backend { + case dbconfig.SQLite: + if storage.DBPath == "" || strings.Contains(storage.DBPath, "://") || strings.HasPrefix(storage.DBPath, "postgres:") || strings.HasPrefix(storage.DBPath, "postgresql:") { + return nil, fmt.Errorf("SQLite telemetry requires a filesystem path") } - } - - // Create/rebuild packets_v view (v3 schema: observer_idx → observers.rowid) - // The Go server reads this view; without it fresh installs get "no such table: packets_v". - db.Exec(`DROP VIEW IF EXISTS packets_v`) - _, vErr := db.Exec(` - CREATE VIEW packets_v AS - SELECT o.id, COALESCE(o.raw_hex, t.raw_hex) AS raw_hex, - datetime(o.timestamp, 'unixepoch') AS timestamp, - obs.id AS observer_id, obs.name AS observer_name, - o.direction, o.snr, o.rssi, o.score, t.hash, t.route_type, - t.payload_type, t.payload_version, o.path_json, t.decoded_json, - t.created_at - FROM observations o - JOIN transmissions t ON t.id = o.transmission_id - LEFT JOIN observers obs ON obs.rowid = o.observer_idx AND (obs.inactive IS NULL OR obs.inactive = 0) - `) - if vErr != nil { - return fmt.Errorf("packets_v view: %w", vErr) - } - - // One-time migration: recalculate advert_count to count unique transmissions only - db.Exec(`CREATE TABLE IF NOT EXISTS _migrations (name TEXT PRIMARY KEY)`) - var migDone int - row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'advert_count_unique_v1'") - if row.Scan(&migDone) != nil { - log.Println("[migration] Recalculating advert_count (unique transmissions only)...") - // Note: this migration is gated on a one-shot _migrations row, so it - // runs at most once per DB. The historical version used a LIKE-on-JSON - // substring match (#1143). Switching to from_pubkey here is safe even - // though the column may not yet be backfilled on legacy DBs: the - // migration is already marked done on those DBs and won't re-run. - db.Exec(` - UPDATE nodes SET advert_count = ( - SELECT COUNT(*) FROM transmissions t - WHERE t.payload_type = 4 - AND t.from_pubkey = nodes.public_key - ) - `) - db.Exec(`INSERT INTO _migrations (name) VALUES ('advert_count_unique_v1')`) - log.Println("[migration] advert_count recalculated") - } - - // One-time migration: change noise_floor from INTEGER to REAL affinity. - // SQLite doesn't support ALTER COLUMN, but existing float values are stored - // as REAL regardless of column affinity. New table definition already uses REAL. - // This migration casts any integer-stored noise_floor values to real. - row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'noise_floor_real_v1'") - if row.Scan(&migDone) != nil { - log.Println("[migration] Ensuring noise_floor values are stored as REAL...") - db.Exec(`UPDATE observers SET noise_floor = CAST(noise_floor AS REAL) WHERE noise_floor IS NOT NULL AND typeof(noise_floor) = 'integer'`) - db.Exec(`INSERT INTO _migrations (name) VALUES ('noise_floor_real_v1')`) - log.Println("[migration] noise_floor migration complete") - } - - // One-time migration: add telemetry columns to nodes and inactive_nodes tables. - row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'node_telemetry_v1'") - if row.Scan(&migDone) != nil { - log.Println("[migration] Adding telemetry columns to nodes/inactive_nodes...") - - // checkAndAddColumn checks whether `column` already exists in `table` - // using PRAGMA table_info, and adds it if missing. All call sites pass - // hardcoded table/column/type literals so there is no SQL injection risk. - checkAndAddColumn := func(table, column, colType string) error { - rows, err := db.Query(fmt.Sprintf("PRAGMA table_info(%s)", table)) - if err != nil { - return fmt.Errorf("querying table info for %s: %w", table, err) - } - defer rows.Close() - - exists := false - for rows.Next() { - var cid int - var name, ctype string - var notnull, pk int - var dfltValue sql.NullString - if err := rows.Scan(&cid, &name, &ctype, ¬null, &dfltValue, &pk); err != nil { - return fmt.Errorf("scanning table info for %s: %w", table, err) - } - if name == column { - exists = true - break - } - } - if err := rows.Err(); err != nil { - return fmt.Errorf("iterating table info for %s: %w", table, err) - } - if exists { - return nil + if create { + if err = os.MkdirAll(filepath.Dir(storage.DBPath), 0700); err != nil { + return nil, err } - if _, err := db.Exec(fmt.Sprintf("ALTER TABLE %s ADD COLUMN %s %s", table, column, colType)); err != nil { - return fmt.Errorf("adding column %s to %s: %w", column, table, err) - } - return nil - } - - if err := checkAndAddColumn("nodes", "battery_mv", "INTEGER"); err != nil { - return err - } - if err := checkAndAddColumn("nodes", "temperature_c", "REAL"); err != nil { - return err } - if err := checkAndAddColumn("inactive_nodes", "battery_mv", "INTEGER"); err != nil { - return err - } - if err := checkAndAddColumn("inactive_nodes", "temperature_c", "REAL"); err != nil { - return err - } - if _, err := db.Exec(`INSERT INTO _migrations (name) VALUES ('node_telemetry_v1')`); err != nil { - return fmt.Errorf("recording node_telemetry_v1 migration: %w", err) + var dsn string + dsn, err = sqliteTelemetryWriterDSN(storage.DBPath, create) + if err == nil { + db, err = sql.Open("sqlite3", dsn) } - log.Println("[migration] node telemetry columns added") - } - - // One-time migration: add timestamp index on observations for fast stats queries. - // Older databases created before this index was added suffer from full table scans - // on COUNT(*) WHERE timestamp > ?, causing /api/stats to take 30s+. - row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'obs_timestamp_index_v1'") - if row.Scan(&migDone) != nil { - log.Println("[migration] Adding timestamp index on observations...") - db.Exec(`CREATE INDEX IF NOT EXISTS idx_observations_timestamp ON observations(timestamp)`) - db.Exec(`INSERT INTO _migrations (name) VALUES ('obs_timestamp_index_v1')`) - log.Println("[migration] observations timestamp index created") - } - - // #1481 P0-3: covering index for GetObserverPacketCounts. The query - // joins observations → observers and GROUP BYs observer_idx with a - // timestamp WHERE filter; a composite (observer_idx, timestamp) - // index lets SQLite resolve the grouping + range filter from the - // index alone instead of a 1.9M-row scan. - // - // CONVERTED TO ASYNC (preflight-async-migration-gate). Scheduling - // happens in OpenStore() once the real *Store exists so the - // backfill WaitGroup is shared with the rest of the ingestor. - // The legacy `_migrations` gate is preserved by the async fn so - // DBs that already completed the sync build stay no-op. - - // #1483: normalize nodes.public_key to lowercase. The server's - // GetNodeLocationsByKeys lookup dropped LOWER(public_key) for perf - // (#1481 P0-3) and now relies on stored keys being lowercase. The - // decoder writes lowercase today, but legacy/admin/API inserts may - // have left mixed-case rows. Idempotent: counts and lowers any - // non-lowercase rows on every boot, runs once via _migrations gate - // for the bulk fix. Re-running stays cheap because subsequent - // passes match zero rows. - if r := db.QueryRow("SELECT COUNT(*) FROM nodes WHERE public_key != lower(public_key)"); r != nil { - var n int64 - _ = r.Scan(&n) - if n > 0 { - log.Printf("[migration] Normalizing %d nodes.public_key row(s) to lowercase (#1483)...", n) - if _, err := db.Exec(`UPDATE nodes SET public_key = lower(public_key) WHERE public_key != lower(public_key)`); err != nil { - log.Printf("[migration] public_key lowercase normalize failed: %v", err) - } else { - log.Printf("[migration] public_key lowercase normalize complete (%d rows)", n) - } + if stateDir == "" { + stateDir = filepath.Dir(storage.DBPath) } - } - - // observer_metrics table for RF health dashboard - row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'observer_metrics_v1'") - if row.Scan(&migDone) != nil { - log.Println("[migration] Creating observer_metrics table...") - _, err := db.Exec(` - CREATE TABLE IF NOT EXISTS observer_metrics ( - observer_id TEXT NOT NULL, - timestamp TEXT NOT NULL, - noise_floor REAL, - tx_air_secs INTEGER, - rx_air_secs INTEGER, - recv_errors INTEGER, - battery_mv INTEGER, - PRIMARY KEY (observer_id, timestamp) - ) - `) - if err != nil { - return fmt.Errorf("observer_metrics schema: %w", err) + case dbconfig.Postgres: + db, err = pgutil.Open(storage.WriterDatabaseURL, false) + if stateDir == "" { + stateDir = "data" } - db.Exec(`INSERT INTO _migrations (name) VALUES ('observer_metrics_v1')`) - log.Println("[migration] observer_metrics table created") + default: + return nil, fmt.Errorf("telemetry requires an explicitly selected storage backend") } - - // Migration: add timestamp index for cross-observer time-range queries - row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'observer_metrics_ts_idx'") - if row.Scan(&migDone) != nil { - log.Println("[migration] Creating observer_metrics timestamp index...") - _, err := db.Exec(`CREATE INDEX IF NOT EXISTS idx_observer_metrics_timestamp ON observer_metrics(timestamp)`) - if err != nil { - return fmt.Errorf("observer_metrics timestamp index: %w", err) - } - db.Exec(`INSERT INTO _migrations (name) VALUES ('observer_metrics_ts_idx')`) - log.Println("[migration] observer_metrics timestamp index created") + if err != nil { + return nil, err } - - // Migration: add inactive column to observers for soft-delete retention - row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'observers_inactive_v1'") - if row.Scan(&migDone) != nil { - log.Println("[migration] Adding inactive column to observers...") - _, err := db.Exec(`ALTER TABLE observers ADD COLUMN inactive INTEGER DEFAULT 0`) - if err != nil { - // Column may already exist (e.g. fresh install with schema above) - log.Printf("[migration] observers.inactive: %v (may already exist)", err) - } - db.Exec(`INSERT INTO _migrations (name) VALUES ('observers_inactive_v1')`) - log.Println("[migration] observers.inactive column added") - } - - // Migration: add packets_sent and packets_recv columns to observer_metrics - row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'observer_metrics_packets_v1'") - if row.Scan(&migDone) != nil { - log.Println("[migration] Adding packets_sent/packets_recv columns to observer_metrics...") - db.Exec(`ALTER TABLE observer_metrics ADD COLUMN packets_sent INTEGER`) - db.Exec(`ALTER TABLE observer_metrics ADD COLUMN packets_recv INTEGER`) - db.Exec(`INSERT INTO _migrations (name) VALUES ('observer_metrics_packets_v1')`) - log.Println("[migration] packets_sent/packets_recv columns added") - } - - // Migration: add channel_hash column for fast channel queries (#762) - row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'channel_hash_v1'") - if row.Scan(&migDone) != nil { - log.Println("[migration] Adding channel_hash column to transmissions...") - db.Exec(`ALTER TABLE transmissions ADD COLUMN channel_hash TEXT DEFAULT NULL`) - db.Exec(`CREATE INDEX IF NOT EXISTS idx_tx_channel_hash ON transmissions(channel_hash) WHERE payload_type = 5`) - // Backfill: extract channel name for decrypted (CHAN) packets - res, err := db.Exec(`UPDATE transmissions SET channel_hash = json_extract(decoded_json, '$.channel') WHERE payload_type = 5 AND channel_hash IS NULL AND json_extract(decoded_json, '$.type') = 'CHAN'`) - if err == nil { - n, _ := res.RowsAffected() - log.Printf("[migration] Backfilled channel_hash for %d CHAN packets", n) - } - // Backfill: extract channelHashHex for encrypted (GRP_TXT) packets, prefixed with 'enc_' - res, err = db.Exec(`UPDATE transmissions SET channel_hash = 'enc_' || json_extract(decoded_json, '$.channelHashHex') WHERE payload_type = 5 AND channel_hash IS NULL AND json_extract(decoded_json, '$.type') = 'GRP_TXT'`) - if err == nil { - n, _ := res.RowsAffected() - log.Printf("[migration] Backfilled channel_hash for %d GRP_TXT packets", n) - } - db.Exec(`INSERT INTO _migrations (name) VALUES ('channel_hash_v1')`) - log.Println("[migration] channel_hash column added and backfilled") - } - - // Migration: dropped_packets table for signature validation failures (#793) - row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'dropped_packets_v1'") - if row.Scan(&migDone) != nil { - log.Println("[migration] Creating dropped_packets table...") - _, err := db.Exec(` - CREATE TABLE IF NOT EXISTS dropped_packets ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - hash TEXT, - raw_hex TEXT, - reason TEXT NOT NULL, - observer_id TEXT, - observer_name TEXT, - node_pubkey TEXT, - node_name TEXT, - dropped_at DATETIME DEFAULT CURRENT_TIMESTAMP - ); - CREATE INDEX IF NOT EXISTS idx_dropped_observer ON dropped_packets(observer_id); - CREATE INDEX IF NOT EXISTS idx_dropped_node ON dropped_packets(node_pubkey); - `) - if err != nil { - return fmt.Errorf("dropped_packets schema: %w", err) + ok := false + defer func() { + if !ok { + db.Close() } - db.Exec(`INSERT INTO _migrations (name) VALUES ('dropped_packets_v1')`) - log.Println("[migration] dropped_packets table created") - } - - // Migration: observations.raw_hex (#881) is now owned by - // internal/dbschema/dbschema.go (#1321). The server PRAGMA-detects - // this column as hasObsRawHex; keeping a single canonical Apply - // path closes the startup race where the server's detector ran - // before this ALTER finished. - - // Migration: transmissions.scope_name (#899) is now owned by - // internal/dbschema/dbschema.go (#1321). See above. - - // Migration: add last_packet_at column to observers (#last-packet-at) - row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'observers_last_packet_at_v1'") - if row.Scan(&migDone) != nil { - log.Println("[migration] Adding last_packet_at column to observers...") - _, alterErr := db.Exec(`ALTER TABLE observers ADD COLUMN last_packet_at TEXT DEFAULT NULL`) - if alterErr != nil && !strings.Contains(alterErr.Error(), "duplicate column") { - return fmt.Errorf("observers last_packet_at ALTER: %w", alterErr) + }() + db.SetMaxOpenConns(1) + db.SetMaxIdleConns(1) + if storage.Backend == dbconfig.SQLite { + if !create { + if err = validateExistingTelemetry(db); err != nil { + return nil, err + } } - // Backfill: set last_packet_at = last_seen only for observers that actually have - // observation rows (packet_count alone is unreliable — UpsertObserver sets it to 1 - // on INSERT even for status-only observers). - res, err := db.Exec(`UPDATE observers SET last_packet_at = last_seen - WHERE last_packet_at IS NULL - AND rowid IN (SELECT DISTINCT observer_idx FROM observations WHERE observer_idx IS NOT NULL)`) - if err == nil { - n, _ := res.RowsAffected() - log.Printf("[migration] Backfilled last_packet_at for %d observers with packets", n) + if err = dbschema.ApplySQLite(db, log.Printf); err != nil { + return nil, err } - db.Exec(`INSERT INTO _migrations (name) VALUES ('observers_last_packet_at_v1')`) - log.Println("[migration] observers.last_packet_at column added") - } - - // Migration: per-observer naive-clock skew tracking (#1478). - // When the ingestor clamps a packet's envelope timestamp because the - // observer emitted a zone-less local-time string off from UTC by >15min - // (resolveRxTime in main.go), we record the event here so the UI can - // surface a ⚠️ chip + banner. Decays after 24h via server-side read sweep. - row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'observers_clock_naive_v1'") - if row.Scan(&migDone) != nil { - log.Println("[migration] Adding clock-naive columns to observers (#1478)...") - // Each ALTER is independent — ignore "duplicate column" so reruns are safe. - for _, stmt := range []string{ - `ALTER TABLE observers ADD COLUMN clock_skew_seconds INTEGER DEFAULT NULL`, - `ALTER TABLE observers ADD COLUMN clock_skew_count_24h INTEGER DEFAULT 0`, - `ALTER TABLE observers ADD COLUMN clock_last_naive_at TEXT DEFAULT NULL`, - } { - if _, err := db.Exec(stmt); err != nil && !strings.Contains(err.Error(), "duplicate column") { - return fmt.Errorf("clock_naive migration: %w", err) + if !create { + if _, err = db.Exec(`PRAGMA auto_vacuum=INCREMENTAL; PRAGMA journal_mode=WAL`); err != nil { + return nil, err } } - db.Exec(`INSERT INTO _migrations (name) VALUES ('observers_clock_naive_v1')`) - log.Println("[migration] observers.clock_naive columns added") - } - - // Migration: backfill observations.path_json from raw_hex (#888) - // NOTE: This runs ASYNC via BackfillPathJSONAsync() to avoid blocking MQTT startup. - // See staging outage where ~502K rows blocked ingest for 15+ hours. - // One-time cleanup: delete legacy packets with empty hash or empty first_seen (#994) - row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'cleanup_legacy_null_hash_ts'") - if row.Scan(&migDone) != nil { - log.Println("[migration] Cleaning up legacy packets with empty hash/timestamp...") - db.Exec(`DELETE FROM observations WHERE transmission_id IN (SELECT id FROM transmissions WHERE hash = '' OR first_seen = '')`) - res, err := db.Exec(`DELETE FROM transmissions WHERE hash = '' OR first_seen = ''`) - if err == nil { - deleted, _ := res.RowsAffected() - log.Printf("[migration] deleted %d legacy packets with empty hash/timestamp", deleted) - } - db.Exec(`INSERT INTO _migrations (name) VALUES ('cleanup_legacy_null_hash_ts')`) - } - - // Migration: foreign_advert column on nodes/inactive_nodes (#730) - // Marks nodes whose ADVERT GPS lies outside the configured geofilter polygon. - // Default 0; set to 1 by the ingestor when GeoFilter is configured and - // PassesFilter() returns false. Allows operators to surface bridged/leaked - // adverts without silently dropping them. - row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'foreign_advert_v1'") - if row.Scan(&migDone) != nil { - log.Println("[migration] Adding foreign_advert column to nodes/inactive_nodes...") - if _, err := db.Exec(`ALTER TABLE nodes ADD COLUMN foreign_advert INTEGER DEFAULT 0`); err != nil { - log.Printf("[migration] nodes.foreign_advert: %v (may already exist)", err) + } else { + var synchronous string + if err = db.QueryRow(`SHOW synchronous_commit`).Scan(&synchronous); err != nil { + return nil, err } - if _, err := db.Exec(`ALTER TABLE inactive_nodes ADD COLUMN foreign_advert INTEGER DEFAULT 0`); err != nil { - log.Printf("[migration] inactive_nodes.foreign_advert: %v (may already exist)", err) + if synchronous != "on" { + return nil, fmt.Errorf("telemetry writer requires synchronous_commit=on") } - db.Exec(`CREATE INDEX IF NOT EXISTS idx_nodes_foreign_advert ON nodes(foreign_advert) WHERE foreign_advert = 1`) - db.Exec(`INSERT INTO _migrations (name) VALUES ('foreign_advert_v1')`) - log.Println("[migration] foreign_advert column added") - } - - // Migration: from_pubkey column on transmissions (#1143). - // Replaces the unsound `decoded_json LIKE '%pubkey%'` attribution path with - // an exact-match indexed column. Synchronously adds the column + index; - // row-level backfill is run by the SERVER asynchronously - // (cmd/server/from_pubkey_migration.go) so we don't block ingestor boot. - row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'from_pubkey_v1'") - if row.Scan(&migDone) != nil { - log.Println("[migration] Adding from_pubkey column + index to transmissions (#1143)...") - if _, err := db.Exec(`ALTER TABLE transmissions ADD COLUMN from_pubkey TEXT`); err != nil { - log.Printf("[migration] transmissions.from_pubkey: %v (may already exist)", err) + + if err = dbschema.AssertReady(db); err != nil { + return nil, err } - if _, err := db.Exec(`CREATE INDEX IF NOT EXISTS idx_transmissions_from_pubkey ON transmissions(from_pubkey)`); err != nil { - log.Printf("[migration] idx_transmissions_from_pubkey: %v", err) + if err = dbschema.AssertWriter(db); err != nil { + return nil, err } - db.Exec(`INSERT INTO _migrations (name) VALUES ('from_pubkey_v1')`) - log.Println("[migration] from_pubkey column + index added") - } - - // Migration: nodes.default_scope (#899 Feature 3) is now owned by - // internal/dbschema/dbschema.go (#1321). The server PRAGMA-detects - // this column as hasDefaultScope; keeping a single canonical Apply - // path closes the startup race that #1321 documented. - - // Migration: normalize known channel_hash values for existing rows. - // Before this PR, config key "public" was stored as channel_hash="public". - // After this PR, new rows use channel_hash="Public". Without backfill, - // channel grouping queries split into two buckets across the upgrade boundary. - row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'channel_hash_casing_v1'") - if row.Scan(&migDone) != nil { - log.Println("[migration] Normalizing known channel_hash values...") - res, err := db.Exec(`UPDATE transmissions SET channel_hash = 'Public' WHERE channel_hash = 'public' AND payload_type = 5`) - if err != nil { - log.Printf("[migration] ERROR: failed to normalize channel_hash: %v", err) - return fmt.Errorf("migration channel_hash_casing_v1 UPDATE failed: %w", err) + } + if stateDir == "" { + stateDir = "data" + } + if err = os.MkdirAll(stateDir, 0700); err != nil { + return nil, fmt.Errorf("create state directory: %w", err) + } + anchor := filepath.Join(stateDir, "meshcore") + if storage.Backend == dbconfig.SQLite { + anchor = filepath.Join(stateDir, filepath.Base(storage.DBPath)) + } + s := &Store{db: db, path: anchor, stateDir: stateDir, backend: storage.Backend, sampleIntervalSec: sampleIntervalSec} + if err = s.prepareStatements(); err != nil { + return nil, fmt.Errorf("prepare writer statements: %w", err) + } + if err = s.LoadScopeMatchTotals(); err != nil { + log.Printf("[regions] restoring scope-match tally: %v", err) + } + + if s.Backend() == dbconfig.SQLite { + var present int + if err := db.QueryRow(`SELECT count(*) FROM sqlite_master WHERE type='index' AND name='idx_observations_observer_idx_timestamp'`).Scan(&present); err != nil { + return nil, err } - n, _ := res.RowsAffected() - log.Printf("[migration] Normalized %d channel_hash rows from 'public' to 'Public'", n) - if _, err := db.Exec(`INSERT OR IGNORE INTO _migrations (name) VALUES ('channel_hash_casing_v1')`); err != nil { - log.Printf("[migration] WARNING: failed to record migration: %v", err) + // Completion metadata must not hide a missing physical index after recovery. + if present == 0 { + if _, err := db.Exec(`UPDATE _async_migrations SET status='pending_async' WHERE name='obs_observer_ts_idx_v1' AND status='done'`); err != nil { + return nil, err + } } - log.Println("[migration] channel_hash casing normalization complete") - } - - // Migration: transmissions.code1/code2 — the transport codes were decoded - // and discarded; storing them makes scope forwarding queryable per repeater. - row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'transport_codes_v1'") - if row.Scan(new(int)) != nil { - log.Println("[migration] Adding code1/code2 columns to transmissions...") - // Each ALTER is independent — ignore "duplicate column" so reruns (and - // the fresh-database path, where the base schema already created the - // columns) are safe. Any other failure must stop before the guard row - // is inserted, or prepareStatements' code1/code2 INSERT fails forever - // on every subsequent restart with no way to retry the migration. - for _, stmt := range []string{ - `ALTER TABLE transmissions ADD COLUMN code1 TEXT DEFAULT NULL`, - `ALTER TABLE transmissions ADD COLUMN code2 TEXT DEFAULT NULL`, - } { - if _, err := db.Exec(stmt); err != nil && !strings.Contains(err.Error(), "duplicate column") { - return fmt.Errorf("transport_codes_v1 migration: %w", err) + // PREFLIGHT: async=true reason="upstream composite observation index; keep its potentially large build off the startup goroutine" + if err := s.RunAsyncMigration(context.Background(), "obs_observer_ts_idx_v1", func(ctx context.Context, d *sql.DB) error { + if err := dbschema.EnsureSQLiteObserverTimeIndex(ctx, d); err != nil { + return err } + _, err := d.ExecContext(ctx, `INSERT INTO _migrations(name) VALUES('obs_observer_ts_idx_v1') ON CONFLICT DO NOTHING`) + return err + }); err != nil { + return nil, err } - db.Exec(`CREATE INDEX IF NOT EXISTS idx_tx_code1 ON transmissions(code1) WHERE code1 IS NOT NULL`) - db.Exec(`INSERT INTO _migrations (name) VALUES ('transport_codes_v1')`) - log.Println("[migration] code1/code2 columns added") + } + var evidenceStatus string + _ = db.QueryRow(`SELECT status FROM _async_migrations WHERE name='advert_route_evidence_v1'`).Scan(&evidenceStatus) + s.advertEvidenceComplete.Store(evidenceStatus == "done") + // The data backfill is resumable bookkeeping, never runtime DDL. It does + // not block readiness, and only rows not maintained by live ingest match. + if err = s.RunAsyncMigration(context.Background(), "tx_last_seen_backfill_v1", func(ctx context.Context, d *sql.DB) error { + _, e := d.ExecContext(ctx, `UPDATE transmissions SET last_seen=COALESCE((SELECT MAX(timestamp) FROM observations WHERE transmission_id=transmissions.id),last_seen) WHERE last_seen=0`) + return e + }); err != nil { + return nil, err } - return nil + ok = true + return s, nil } func (s *Store) prepareStatements() error { var err error + s.stmtInsertAdvertEvidence, err = s.db.Prepare(insertAdvertEvidenceSQL) + if err != nil { + return err + } + s.stmtGetLegacyAdvertObservation, err = s.db.Prepare(legacyAdvertObservationSQL) + if err != nil { + return err + } - s.stmtGetTxByHash, err = s.db.Prepare("SELECT id, first_seen FROM transmissions WHERE hash = ?") + s.stmtGetTxByHash, err = s.db.Prepare("SELECT id, first_seen FROM transmissions WHERE hash = " + s.parameter(1)) if err != nil { return err } s.stmtInsertTransmission, err = s.db.Prepare(` INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json, channel_hash, scope_name, from_pubkey, last_seen, code1, code2) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + VALUES (` + s.parameter(1) + `, ` + s.parameter(2) + `, ` + s.parameter(3) + `, ` + s.parameter(4) + `, ` + s.parameter(5) + `, ` + s.parameter(6) + `, ` + s.parameter(7) + `, ` + s.parameter(8) + `, ` + s.parameter(9) + `, ` + s.parameter(10) + `, ` + s.parameter(11) + `, ` + s.parameter(12) + `, ` + s.parameter(13) + `) RETURNING id `) if err != nil { return err } - s.stmtUpdateTxFirstSeen, err = s.db.Prepare("UPDATE transmissions SET first_seen = ? WHERE id = ?") + s.stmtUpdateTxFirstSeen, err = s.db.Prepare("UPDATE transmissions SET first_seen = " + s.parameter(1) + " WHERE id = " + s.parameter(2)) if err != nil { return err } @@ -897,74 +308,100 @@ func (s *Store) prepareStatements() error { // "tx_last_seen_backfill_v1". // PREFLIGHT: async=true reason="prepared-statement row-level UPDATE BY PRIMARY KEY (transmissions.id) — single-row touch per observation, indexed by PK, constant-time at any scale. Not a migration." s.stmtTouchNodeLastSeen, err = s.db.Prepare( - "UPDATE nodes SET last_seen = ? WHERE public_key = ? AND (last_seen IS NULL OR last_seen < ?)") + "UPDATE nodes SET last_seen = " + s.parameter(1) + " WHERE public_key = " + s.parameter(2) + " AND (last_seen IS NULL OR last_seen < " + s.parameter(3) + ")") if err != nil { return fmt.Errorf("preparing touch node last_seen: %w", err) } - s.stmtBumpTxLastSeen, err = s.db.Prepare("UPDATE transmissions SET last_seen = ? WHERE id = ? AND last_seen < ?") + s.stmtBumpTxLastSeen, err = s.db.Prepare("UPDATE transmissions SET last_seen = " + s.parameter(1) + " WHERE id = " + s.parameter(2) + " AND last_seen < " + s.parameter(3)) if err != nil { return err } s.stmtInsertObservation, err = s.db.Prepare(` INSERT INTO observations (transmission_id, observer_idx, direction, snr, rssi, score, path_json, timestamp, raw_hex, resolved_path) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + VALUES (` + s.parameter(1) + `, ` + s.parameter(2) + `, ` + s.parameter(3) + `, ` + s.parameter(4) + `, ` + s.parameter(5) + `, ` + s.parameter(6) + `, ` + s.parameter(7) + `, ` + s.parameter(8) + `, ` + s.parameter(9) + `, ` + s.parameter(10) + `) ON CONFLICT(transmission_id, observer_idx, COALESCE(path_json, '')) DO UPDATE SET - snr = COALESCE(excluded.snr, snr), - rssi = COALESCE(excluded.rssi, rssi), - score = COALESCE(excluded.score, score), - raw_hex = COALESCE(excluded.raw_hex, raw_hex), - resolved_path = COALESCE(excluded.resolved_path, resolved_path) + snr = COALESCE(excluded.snr, observations.snr), + rssi = COALESCE(excluded.rssi, observations.rssi), + score = COALESCE(excluded.score, observations.score), + raw_hex = COALESCE(excluded.raw_hex, observations.raw_hex), + resolved_path = COALESCE(excluded.resolved_path, observations.resolved_path) `) if err != nil { return err } - s.stmtUpsertNode, err = s.db.Prepare(` + s.stmtUpsertNode, err = s.db.Prepare(s.nativeSQL(` INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen) - VALUES (?, ?, ?, ?, ?, ?, ?) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7) ON CONFLICT(public_key) DO UPDATE SET - name = COALESCE(?, name), - role = COALESCE(?, role), - lat = COALESCE(?, lat), - lon = COALESCE(?, lon), - last_seen = MAX(MIN(COALESCE(last_seen, ''), ?), ?) - `) + name = COALESCE(?8, nodes.name), + role = COALESCE(?9, nodes.role), + lat = COALESCE(?10, nodes.lat), + lon = COALESCE(?11, nodes.lon), + last_seen = MAX(MIN(COALESCE(nodes.last_seen, ''), ?12), ?13) + `, ` + INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen) + VALUES (`+s.parameter(1)+`, `+s.parameter(2)+`, `+s.parameter(3)+`, `+s.parameter(4)+`, `+s.parameter(5)+`, `+s.parameter(6)+`, `+s.parameter(7)+`) + ON CONFLICT(public_key) DO UPDATE SET + name = COALESCE(`+s.parameter(8)+`, nodes.name), + role = COALESCE(`+s.parameter(9)+`, nodes.role), + lat = COALESCE(`+s.parameter(10)+`, nodes.lat), + lon = COALESCE(`+s.parameter(11)+`, nodes.lon), + last_seen = GREATEST(LEAST(COALESCE(nodes.last_seen, ''), `+s.parameter(12)+`), `+s.parameter(13)+`) + `)) if err != nil { return err } s.stmtIncrementAdvertCount, err = s.db.Prepare(` - UPDATE nodes SET advert_count = advert_count + 1 WHERE public_key = ? + UPDATE nodes SET advert_count = advert_count + 1 WHERE public_key = ` + s.parameter(1) + ` `) if err != nil { return err } - s.stmtUpsertObserver, err = s.db.Prepare(` + s.stmtUpsertObserver, err = s.db.Prepare(s.nativeSQL(` INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count, model, firmware, client_version, radio, battery_mv, uptime_secs, noise_floor, can_relay, can_relay_seen) - VALUES (?, ?, ?, ?, ?, 1, ?, ?, ?, ?, ?, ?, ?, COALESCE(?, 1), CASE WHEN ? IS NULL THEN 0 ELSE 1 END) + VALUES (?1, ?2, ?3, ?4, ?5, 1, ?6, ?7, ?8, ?9, ?10, ?11, ?12, COALESCE(?13, 1), CASE WHEN CAST(?14 AS INTEGER) IS NULL THEN 0 ELSE 1 END) ON CONFLICT(id) DO UPDATE SET - name = COALESCE(?, name), - iata = COALESCE(?, iata), - last_seen = MAX(MIN(COALESCE(last_seen, ''), ?), ?), - packet_count = packet_count + 1, - model = COALESCE(?, model), - firmware = COALESCE(?, firmware), - client_version = COALESCE(?, client_version), - radio = COALESCE(?, radio), - battery_mv = COALESCE(?, battery_mv), - uptime_secs = COALESCE(?, uptime_secs), - noise_floor = COALESCE(?, noise_floor), - can_relay = COALESCE(?, can_relay), - can_relay_seen = CASE WHEN ? IS NULL THEN can_relay_seen ELSE 1 END - `) + name = COALESCE(?15, observers.name), + iata = COALESCE(?16, observers.iata), + last_seen = MAX(MIN(COALESCE(observers.last_seen, ''), ?17), ?18), + packet_count = observers.packet_count + 1, + model = COALESCE(?19, observers.model), + firmware = COALESCE(?20, observers.firmware), + client_version = COALESCE(?21, observers.client_version), + radio = COALESCE(?22, observers.radio), + battery_mv = COALESCE(?23, observers.battery_mv), + uptime_secs = COALESCE(?24, observers.uptime_secs), + noise_floor = COALESCE(?25, observers.noise_floor), + can_relay = COALESCE(?26, observers.can_relay), + can_relay_seen = CASE WHEN CAST(?27 AS INTEGER) IS NULL THEN observers.can_relay_seen ELSE 1 END + `, ` + INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count, model, firmware, client_version, radio, battery_mv, uptime_secs, noise_floor, can_relay, can_relay_seen) + VALUES (`+s.parameter(1)+`, `+s.parameter(2)+`, `+s.parameter(3)+`, `+s.parameter(4)+`, `+s.parameter(5)+`, 1, `+s.parameter(6)+`, `+s.parameter(7)+`, `+s.parameter(8)+`, `+s.parameter(9)+`, `+s.parameter(10)+`, `+s.parameter(11)+`, `+s.parameter(12)+`, COALESCE(`+s.parameter(13)+`, 1), CASE WHEN `+s.parameter(14)+`::bigint IS NULL THEN 0 ELSE 1 END) + ON CONFLICT(id) DO UPDATE SET + name = COALESCE(`+s.parameter(15)+`, observers.name), + iata = COALESCE(`+s.parameter(16)+`, observers.iata), + last_seen = GREATEST(LEAST(COALESCE(observers.last_seen, ''), `+s.parameter(17)+`), `+s.parameter(18)+`), + packet_count = observers.packet_count + 1, + model = COALESCE(`+s.parameter(19)+`, observers.model), + firmware = COALESCE(`+s.parameter(20)+`, observers.firmware), + client_version = COALESCE(`+s.parameter(21)+`, observers.client_version), + radio = COALESCE(`+s.parameter(22)+`, observers.radio), + battery_mv = COALESCE(`+s.parameter(23)+`, observers.battery_mv), + uptime_secs = COALESCE(`+s.parameter(24)+`, observers.uptime_secs), + noise_floor = COALESCE(`+s.parameter(25)+`, observers.noise_floor), + can_relay = COALESCE(`+s.parameter(26)+`, observers.can_relay), + can_relay_seen = CASE WHEN `+s.parameter(27)+`::bigint IS NULL THEN observers.can_relay_seen ELSE 1 END + `)) if err != nil { return err } - s.stmtGetObserverRowid, err = s.db.Prepare("SELECT rowid FROM observers WHERE id = ?") + s.stmtGetObserverRowid, err = s.db.Prepare("SELECT rowid FROM observers WHERE id = " + s.parameter(1)) if err != nil { return err } @@ -972,29 +409,36 @@ func (s *Store) prepareStatements() error { // Args: ingestNow, rxTime, ingestNow, rxTime, rowid // MIN(existing, ingestNow) clamps any future value already in the DB before // taking MAX with rxTime, so the guard never locks in a past bug's stale future. - s.stmtUpdateObserverLastSeen, err = s.db.Prepare(` + s.stmtUpdateObserverLastSeen, err = s.db.Prepare(s.nativeSQL(` + UPDATE observers SET + last_seen = MAX(MIN(COALESCE(last_seen, ''), ?1), ?2), + last_packet_at = MAX(MIN(COALESCE(last_packet_at, ''), ?3), ?4) + WHERE rowid = ?5`, ` UPDATE observers SET - last_seen = MAX(MIN(COALESCE(last_seen, ''), ?), ?), - last_packet_at = MAX(MIN(COALESCE(last_packet_at, ''), ?), ?) - WHERE rowid = ?`) + last_seen = GREATEST(LEAST(COALESCE(last_seen, ''), `+s.parameter(1)+`), `+s.parameter(2)+`), + last_packet_at = GREATEST(LEAST(COALESCE(last_packet_at, ''), `+s.parameter(3)+`), `+s.parameter(4)+`) + WHERE rowid = `+s.parameter(5))) if err != nil { return err } s.stmtUpdateNodeTelemetry, err = s.db.Prepare(` UPDATE nodes SET - battery_mv = COALESCE(?, battery_mv), - temperature_c = COALESCE(?, temperature_c) - WHERE public_key = ? + battery_mv = COALESCE(` + s.parameter(1) + `, battery_mv), + temperature_c = COALESCE(` + s.parameter(2) + `, temperature_c) + WHERE public_key = ` + s.parameter(3) + ` `) if err != nil { return err } s.stmtUpsertMetrics, err = s.db.Prepare(` - INSERT OR REPLACE INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors, battery_mv, packets_sent, packets_recv) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) - `) + INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors, battery_mv, packets_sent, packets_recv) + VALUES (` + s.parameter(1) + `, ` + s.parameter(2) + `, ` + s.parameter(3) + `, ` + s.parameter(4) + `, ` + s.parameter(5) + `, ` + s.parameter(6) + `, ` + s.parameter(7) + `, ` + s.parameter(8) + `, ` + s.parameter(9) + `) + ON CONFLICT(observer_id,timestamp) DO UPDATE SET noise_floor=excluded.noise_floor, + tx_air_secs=excluded.tx_air_secs,rx_air_secs=excluded.rx_air_secs,recv_errors=excluded.recv_errors, + battery_mv=excluded.battery_mv,packets_sent=excluded.packets_sent,packets_recv=excluded.packets_recv + `) if err != nil { return err } @@ -1005,137 +449,139 @@ func (s *Store) prepareStatements() error { // InsertTransmission inserts a decoded packet into transmissions + observations. // Returns true if a new transmission was created (not a duplicate hash). func (s *Store) InsertTransmission(data *PacketData) (bool, error) { - hash := data.Hash - if hash == "" { + if data.Hash == "" { return false, nil } - - // Wait/hold instrumentation (#1340). The hot path uses prepared - // statements that auto-commit; gate the whole function under - // writerMu so concurrent mqtt_handler inserts queue behind any - // other writer (vacuum, prune, neighbor-builder) and the wait is - // Go-visible. - mqttWaitStart := time.Now() + waitStart := time.Now() writerMu.Lock() - mqttWait := time.Since(mqttWaitStart) - mqttHoldStart := time.Now() + wait := time.Since(waitStart) + holdStart := time.Now() defer func() { - mqttHold := time.Since(mqttHoldStart) writerMu.Unlock() - recordWriterTiming("mqtt_handler", mqttWait, mqttHold, "InsertTransmission") + recordWriterTiming("mqtt_handler", wait, time.Since(holdStart), "InsertTransmission") }() - - rxTime := data.Timestamp + tx, err := beginWrite(s.db, s.Backend()) + if err != nil { + return false, err + } + defer tx.Rollback() ingestNow := time.Now().UTC().Format(time.RFC3339) + rxTime := data.Timestamp if rxTime == "" { rxTime = ingestNow } - var txID int64 - isNew := false - - // Check for existing transmission - var existingID int64 - var existingFirstSeen string - err := s.stmtGetTxByHash.QueryRow(hash).Scan(&existingID, &existingFirstSeen) - if err == nil { - // Existing transmission - txID = existingID - if rxTime < existingFirstSeen { - _, _ = s.stmtUpdateTxFirstSeen.Exec(rxTime, txID) - } - } else { - // New transmission - isNew = true - result, err := s.stmtInsertTransmission.Exec( - data.RawHex, hash, rxTime, - data.RouteType, data.PayloadType, data.PayloadVersion, - data.DecodedJSON, nilIfEmpty(data.ChannelHash), - scopeNameForDB(data), - nilIfEmpty(data.FromPubkey), - epochSecondsForLastSeen(rxTime), - nilIfEmpty(data.Code1), - nilIfEmpty(data.Code2), - ) - if err != nil { - s.Stats.WriteErrors.Add(1) - return false, fmt.Errorf("insert transmission: %w", err) - } - txID, _ = result.LastInsertId() - s.Stats.TransmissionsInserted.Add(1) + var firstSeen string + err = tx.Stmt(s.stmtGetTxByHash).QueryRow(data.Hash).Scan(&txID, &firstSeen) + isNew := err == sql.ErrNoRows + if err != nil && !isNew { + return false, err } - - if !isNew { - s.Stats.DuplicateTransmissions.Add(1) + if isNew { + err = tx.Stmt(s.stmtInsertTransmission).QueryRow(data.RawHex, data.Hash, rxTime, data.RouteType, data.PayloadType, data.PayloadVersion, data.DecodedJSON, nilIfEmpty(data.ChannelHash), scopeNameForDB(data), nilIfEmpty(data.FromPubkey), epochSecondsForLastSeen(rxTime), nilIfEmpty(data.Code1), nilIfEmpty(data.Code2)).Scan(&txID) + } else if rxTime < firstSeen { + _, err = tx.Stmt(s.stmtUpdateTxFirstSeen).Exec(rxTime, txID) + } + if err != nil { + s.Stats.WriteErrors.Add(1) + return false, fmt.Errorf("write transmission: %w", err) + } + if data.PayloadType == 4 { + if bit := packetpath.AdvertRouteEvidence(data.RawHex); bit != 0 { + if err := optionalWrite(tx, func() error { _, e := tx.Stmt(s.stmtInsertAdvertEvidence).Exec(txID, bit, txID, bit); return e }); err != nil { + s.Stats.WriteErrors.Add(1) + log.Printf("[db] record advert route evidence (non-fatal): %v", err) + } + } } - - // Resolve observer_idx and update last_seen var observerIdx *int64 if data.ObserverID != "" { - var rowid int64 - err := s.stmtGetObserverRowid.QueryRow(data.ObserverID).Scan(&rowid) + var id int64 + err = tx.Stmt(s.stmtGetObserverRowid).QueryRow(data.ObserverID).Scan(&id) if err == nil { - observerIdx = &rowid - // observer.last_seen and last_packet_at answer "when did the analyzer - // last hear from this observer" — both are ingest-time questions. - // Per-packet rxTime is stored separately on observations/transmissions - // using envelope time (see InsertTransmission above). See #1465. - _, _ = s.stmtUpdateObserverLastSeen.Exec(ingestNow, ingestNow, ingestNow, ingestNow, rowid) + observerIdx = &id + _, err = tx.Stmt(s.stmtUpdateObserverLastSeen).Exec(ingestNow, ingestNow, ingestNow, ingestNow, id) + } + if err != nil && err != sql.ErrNoRows { + return false, err + } + } + if !isNew && data.PayloadType == 4 && observerIdx != nil && data.RawHex != "" { + if err := optionalWrite(tx, func() error { return s.preserveLegacyAdvertObservation(tx, txID, *observerIdx, data.PathJSON) }); err != nil { + s.Stats.WriteErrors.Add(1) + log.Printf("[db] preserve legacy advert evidence (non-fatal): %v", err) } } - - // Insert observation epochTs := time.Now().Unix() - if t, err := time.Parse(time.RFC3339, rxTime); err == nil { + if t, e := time.Parse(time.RFC3339, rxTime); e == nil { epochTs = t.Unix() } - - // Resolve hop prefixes to full pubkeys for `observations.resolved_path`. - // Per #1547: this writer was lost in the #1289 refactor and lives in - // the ingestor now. Per #1560: use the context-aware resolver so - // 1-byte prefix collisions are disambiguated via NeighborGraph - // adjacency (anchored on from_pubkey for ADVERTs, previous hop - // otherwise). Empty resolved JSON → NULL via nilIfEmpty. - resolved := resolvePathWithContext( - parsePathArray(data.PathJSON), - strings.ToLower(data.FromPubkey), - s.neighborGraph.load(), - s.prefixIdx.load(), - ) - resolvedJSON := marshalResolvedPath(resolved) - - _, err = s.stmtInsertObservation.Exec( - txID, observerIdx, data.Direction, - data.SNR, data.RSSI, data.Score, - data.PathJSON, epochTs, nilIfEmpty(data.RawHex), - nilIfEmpty(resolvedJSON), - ) + resolved := resolvePathWithContext(parsePathArray(data.PathJSON), strings.ToLower(data.FromPubkey), s.neighborGraph.load(), s.prefixIdx.load()) + _, err = tx.Stmt(s.stmtInsertObservation).Exec(txID, observerIdx, data.Direction, data.SNR, data.RSSI, data.Score, data.PathJSON, epochTs, nilIfEmpty(data.RawHex), nilIfEmpty(marshalResolvedPath(resolved))) if err != nil { s.Stats.WriteErrors.Add(1) - log.Printf("[db] observation insert (non-fatal): %v", err) + return false, fmt.Errorf("insert observation: %w", err) + } + if _, err = tx.Stmt(s.stmtBumpTxLastSeen).Exec(epochTs, txID, epochTs); err != nil { + return false, err + } + if err = tx.Commit(); err != nil { + s.Stats.WriteErrors.Add(1) + return false, fmt.Errorf("commit packet: %w", err) + } + if isNew { + s.Stats.TransmissionsInserted.Add(1) } else { - s.Stats.ObservationsInserted.Add(1) - // #1598: a resolved hop proves the node was forwarding traffic at - // rxTime. Refresh its last_seen so staleness/eviction logic sees - // relay activity, not just ADVERTs. - s.touchRelayNodesLocked(resolvedPubkeys(resolved), rxTime) - // #1690: bump transmissions.last_seen so cold-load can filter on - // effective recency. Conditional `last_seen < ?` so we never go - // backwards on out-of-order ingest. - if _, err := s.stmtBumpTxLastSeen.Exec(epochTs, txID, epochTs); err != nil { - log.Printf("[db] tx last_seen bump (non-fatal): %v", err) - } + s.Stats.DuplicateTransmissions.Add(1) } - - // Each prepared-stmt Exec auto-commits. Count one WAL commit per - // successful InsertTransmission so the perf page sees commit pressure. + s.Stats.ObservationsInserted.Add(1) s.Stats.WALCommits.Add(1) - + s.touchRelayNodesLocked(resolvedPubkeys(resolved), rxTime) return isNew, nil } +// beginWrite serializes ID allocation before any statement can consume an +// identity value. A transaction lock releases only after commit or rollback. +func beginWrite(db *sql.DB, backend ...dbconfig.Backend) (*sql.Tx, error) { + return beginWriteContext(context.Background(), db, backend...) +} +func beginWriteContext(ctx context.Context, db *sql.DB, backend ...dbconfig.Backend) (*sql.Tx, error) { + tx, err := db.BeginTx(ctx, nil) + if err != nil { + return nil, err + } + if len(backend) == 0 || backend[0] == dbconfig.Postgres { + if _, err = tx.ExecContext(ctx, `SELECT pg_advisory_xact_lock(hashtextextended(current_schema(),$1))`, dbschema.WriterLockKey); err != nil { + tx.Rollback() + return nil, err + } + } + return tx, nil +} + +// Analytics are best effort; a failed PostgreSQL statement otherwise poisons +// the packet transaction. Savepoints preserve the existing non-fatal behavior. +func optionalWrite(tx *sql.Tx, fn func() error) error { + if _, err := tx.Exec(`SAVEPOINT packet_analytics`); err != nil { + return err + } + err := fn() + if err != nil { + if _, rollbackErr := tx.Exec(`ROLLBACK TO SAVEPOINT packet_analytics`); rollbackErr != nil { + return rollbackErr + } + } + _, releaseErr := tx.Exec(`RELEASE SAVEPOINT packet_analytics`) + if err != nil { + return err + } + return releaseErr +} + // UpsertNode inserts or updates a node. func (s *Store) UpsertNode(pubKey, name, role string, lat, lon *float64, lastSeen string) error { + // Preserve the decoder's byte cap while making the name safe for SQL text. + name = strings.ToValidUTF8(name, "") ingestNow := time.Now().UTC().Format(time.RFC3339) now := lastSeen if now == "" { @@ -1167,7 +613,7 @@ func (s *Store) MarkNodeForeign(pubKey string) error { if pubKey == "" { return nil } - _, err := s.db.Exec(`UPDATE nodes SET foreign_advert = 1 WHERE public_key = ?`, pubKey) + _, err := s.db.Exec(`UPDATE nodes SET foreign_advert = 1 WHERE public_key = `+s.parameter(1), pubKey) if err != nil { s.Stats.WriteErrors.Add(1) } @@ -1245,7 +691,7 @@ func (s *Store) UpsertObserverAt(id, name, iata string, meta *ObserverMeta, last s.Stats.ObserverUpserts.Add(1) // Reactivate if this observer was previously marked inactive - s.db.Exec(`UPDATE observers SET inactive = 0 WHERE id = ? AND inactive = 1`, id) + s.db.Exec(`UPDATE observers SET inactive = 0 WHERE id = `+s.parameter(1)+` AND inactive = 1`, id) return nil } @@ -1266,20 +712,33 @@ func (s *Store) UpsertObserverRetained(id, name, iata string, meta *ObserverMeta normalizedIATA := strings.TrimSpace(strings.ToUpper(iata)) model, firmware, clientVersion, radio, batteryMv, uptimeSecs, noiseFloor, canRelay := observerMetaColumns(meta) - _, err := s.db.Exec(` + _, err := s.db.Exec(s.nativeSQL(` + UPDATE observers SET + name = COALESCE(?1, name), + iata = COALESCE(?2, iata), + model = COALESCE(?3, model), + firmware = COALESCE(?4, firmware), + client_version = COALESCE(?5, client_version), + radio = COALESCE(?6, radio), + battery_mv = COALESCE(?7, battery_mv), + uptime_secs = COALESCE(?8, uptime_secs), + noise_floor = COALESCE(?9, noise_floor), + can_relay = COALESCE(?10, can_relay), + can_relay_seen = CASE WHEN CAST(?11 AS INTEGER) IS NULL THEN can_relay_seen ELSE 1 END + WHERE id = ?12`, ` UPDATE observers SET - name = COALESCE(?, name), - iata = COALESCE(?, iata), - model = COALESCE(?, model), - firmware = COALESCE(?, firmware), - client_version = COALESCE(?, client_version), - radio = COALESCE(?, radio), - battery_mv = COALESCE(?, battery_mv), - uptime_secs = COALESCE(?, uptime_secs), - noise_floor = COALESCE(?, noise_floor), - can_relay = COALESCE(?, can_relay), - can_relay_seen = CASE WHEN ? IS NULL THEN can_relay_seen ELSE 1 END - WHERE id = ?`, + name = COALESCE(`+s.parameter(1)+`, name), + iata = COALESCE(`+s.parameter(2)+`, iata), + model = COALESCE(`+s.parameter(3)+`, model), + firmware = COALESCE(`+s.parameter(4)+`, firmware), + client_version = COALESCE(`+s.parameter(5)+`, client_version), + radio = COALESCE(`+s.parameter(6)+`, radio), + battery_mv = COALESCE(`+s.parameter(7)+`, battery_mv), + uptime_secs = COALESCE(`+s.parameter(8)+`, uptime_secs), + noise_floor = COALESCE(`+s.parameter(9)+`, noise_floor), + can_relay = COALESCE(`+s.parameter(10)+`, can_relay), + can_relay_seen = CASE WHEN `+s.parameter(11)+`::bigint IS NULL THEN can_relay_seen ELSE 1 END + WHERE id = `+s.parameter(12)), name, normalizedIATA, model, firmware, clientVersion, radio, batteryMv, uptimeSecs, noiseFloor, canRelay, canRelay, id, ) @@ -1331,10 +790,9 @@ func observerMetaColumns(meta *ObserverMeta) (model, firmware, clientVersion, ra return model, firmware, clientVersion, radio, batteryMv, uptimeSecs, noiseFloor, canRelay } -// Close checkpoints the WAL and closes the database. +// Close waits for data backfills and closes the database. func (s *Store) Close() error { s.backfillWg.Wait() - s.Checkpoint() return s.db.Close() } @@ -1401,7 +859,7 @@ func (s *Store) InsertMetrics(data *MetricsData) error { func (s *Store) PruneOldMetrics(retentionDays int) (int64, error) { cutoff := time.Now().UTC().AddDate(0, 0, -retentionDays).Format(time.RFC3339) // Tagged for /api/perf writer-lock visibility (#1340). - result, err := s.instrumentedExec("prune_metrics", `DELETE FROM observer_metrics WHERE timestamp < ?`, cutoff) + result, err := s.instrumentedExec("prune_metrics", `DELETE FROM observer_metrics WHERE timestamp < `+s.parameter(1), cutoff) if err != nil { return 0, fmt.Errorf("prune metrics: %w", err) } @@ -1412,75 +870,6 @@ func (s *Store) PruneOldMetrics(retentionDays int) (int64, error) { return n, nil } -// CheckAutoVacuum inspects the current auto_vacuum mode and logs a warning -// if not INCREMENTAL. Performs opt-in full VACUUM if db.vacuumOnStartup is set (#919). -func (s *Store) CheckAutoVacuum(cfg *Config) { - var autoVacuum int - if err := s.db.QueryRow("PRAGMA auto_vacuum").Scan(&autoVacuum); err != nil { - log.Printf("[db] warning: could not read auto_vacuum: %v", err) - return - } - - if autoVacuum == 2 { - log.Printf("[db] auto_vacuum=INCREMENTAL") - return - } - - modes := map[int]string{0: "NONE", 1: "FULL", 2: "INCREMENTAL"} - mode := modes[autoVacuum] - if mode == "" { - mode = fmt.Sprintf("UNKNOWN(%d)", autoVacuum) - } - - log.Printf("[db] auto_vacuum=%s — DB needs one-time VACUUM to enable incremental auto-vacuum. "+ - "Set db.vacuumOnStartup: true in config to migrate (will block startup for several minutes on large DBs). "+ - "See https://github.com/Kpa-clawbot/CoreScope/issues/919", mode) - - if cfg.DB != nil && cfg.DB.VacuumOnStartup { - // WARNING: Full VACUUM creates a temporary copy of the entire DB file. - // Requires ~2× the DB file size in free disk space or it will fail. - log.Printf("[db] vacuumOnStartup=true — starting one-time full VACUUM (ensure 2x DB size free disk space)...") - start := time.Now() - - if _, err := s.instrumentedExec("vacuum", "PRAGMA auto_vacuum = INCREMENTAL"); err != nil { - log.Printf("[db] VACUUM failed: could not set auto_vacuum: %v", err) - return - } - if _, err := s.instrumentedExec("vacuum", "VACUUM"); err != nil { - log.Printf("[db] VACUUM failed: %v", err) - return - } - - elapsed := time.Since(start) - log.Printf("[db] VACUUM complete in %v — auto_vacuum is now INCREMENTAL", elapsed.Round(time.Millisecond)) - } -} - -// RunIncrementalVacuum returns free pages to the OS (#919). -// Safe to call on auto_vacuum=NONE databases (noop). -func (s *Store) RunIncrementalVacuum(pages int) { - // Tagged for /api/perf writer-lock visibility (#1340). - if _, err := s.instrumentedExec("vacuum", fmt.Sprintf("PRAGMA incremental_vacuum(%d)", pages)); err != nil { - log.Printf("[vacuum] incremental_vacuum error: %v", err) - } -} - -// Checkpoint runs a WAL checkpoint (TRUNCATE mode). -// Returns the number of WAL frames checkpointed (0 if WAL was already empty). -// TRUNCATE resets the WAL file to zero bytes when all frames are checkpointed; -// if active readers hold frames, it checkpoints what it can and leaves the rest. -func (s *Store) Checkpoint() int { - var busy, walFrames, checkpointed int - if err := s.db.QueryRow("PRAGMA wal_checkpoint(TRUNCATE)").Scan(&busy, &walFrames, &checkpointed); err != nil { - log.Printf("[db] WAL checkpoint error: %v", err) - return 0 - } - if walFrames > 0 { - log.Printf("[db] WAL checkpoint: %d/%d frames checkpointed (blocked=%v)", checkpointed, walFrames, busy != 0) - } - return checkpointed -} - // BackfillPathJSONAsync launches the path_json backfill in a background goroutine. // It processes observations with NULL/empty path_json that have raw_hex available, // decoding hop paths and updating the column. Safe to run concurrently with ingest @@ -1517,7 +906,7 @@ func (s *Store) BackfillPathJSONAsync() { -- to prevent the infinite re-UPDATE loop fixed in #1119. AND (o.path_json IS NULL OR o.path_json = '') AND t.payload_type != 9 - LIMIT ?`, batchSize) + LIMIT `+s.parameter(1), batchSize) if err != nil { log.Printf("[backfill] path_json query error: %v", err) errored = true @@ -1541,7 +930,7 @@ func (s *Store) BackfillPathJSONAsync() { for _, r := range batch { hops, err := packetpath.DecodePathFromRawHex(r.rawHex) if err != nil || len(hops) == 0 { - if _, execErr := s.db.Exec(`UPDATE observations SET path_json = '[]' WHERE id = ?`, r.id); execErr != nil { + if _, execErr := s.db.Exec(`UPDATE observations SET path_json = '[]' WHERE id = `+s.parameter(1), r.id); execErr != nil { log.Printf("[backfill] write error (id=%d): %v", r.id, execErr) } else { s.Stats.IncBackfill("path_json") @@ -1549,7 +938,7 @@ func (s *Store) BackfillPathJSONAsync() { continue } b, _ := json.Marshal(hops) - if _, execErr := s.db.Exec(`UPDATE observations SET path_json = ? WHERE id = ?`, string(b), r.id); execErr != nil { + if _, execErr := s.db.Exec(`UPDATE observations SET path_json = `+s.parameter(1)+` WHERE id = `+s.parameter(2), string(b), r.id); execErr != nil { log.Printf("[backfill] write error (id=%d): %v", r.id, execErr) } else { updated++ @@ -1581,9 +970,11 @@ func (s *Store) BackfillPathJSONAsync() { // MQTT packet inserts and any concurrent backfill goroutines — serialize through // the single connection pool. busy_timeout(5000) handles transient cross-process // contention with the read-only server process. No additional locking is needed. -func (s *Store) BackfillDefaultScopeAsync(regionKeys map[string][]byte) { - // No region keys configured — all scope_name values will be NULL, nothing to backfill. - if len(regionKeys) == 0 { +func (s *Store) BackfillDefaultScopeAsync(regionSet *regionKeySet) { + // No region keys at all — every scope_name is NULL, nothing to backfill. + // The emptiness gate is read once here rather than per row: the loop below + // uses no keys, it only copies names that are already stored. + if len(regionSet.snapshot().all) == 0 { return } s.backfillWg.Add(1) @@ -1628,7 +1019,7 @@ func (s *Store) BackfillDefaultScopeAsync(regionKeys map[string][]byte) { // inserted before the transport_codes_v1 migration, by re-parsing raw_hex. // Runs in a background goroutine so it does not block MQTT startup. // -// Batched: SQLite holds a single write connection (SetMaxOpenConns(1)), so a +// Batched: the ingestor retains a single write connection, so a // multi-million-row UPDATE in one transaction would stall live ingest for the // whole run. 5k-row batches keep each transaction short. func (s *Store) BackfillTransportCodesAsync() { @@ -1678,9 +1069,9 @@ func (s *Store) backfillTransportCodes(batchSize int) (int, error) { for { rows, err := s.db.Query(` SELECT id, raw_hex FROM transmissions - WHERE id > ? AND code1 IS NULL AND route_type IN (0, 3) + WHERE id > `+s.parameter(1)+` AND code1 IS NULL AND route_type IN (0, 3) ORDER BY id - LIMIT ?`, lastID, batchSize) + LIMIT `+s.parameter(2), lastID, batchSize) if err != nil { return total, fmt.Errorf("transport_codes select: %w", err) } @@ -1711,11 +1102,11 @@ func (s *Store) backfillTransportCodes(batchSize int) (int, error) { } if len(items) > 0 { - tx, err := s.db.Begin() + tx, err := beginWrite(s.db, s.Backend()) if err != nil { return total, fmt.Errorf("transport_codes begin: %w", err) } - stmt, err := tx.Prepare(`UPDATE transmissions SET code1 = ?, code2 = ? WHERE id = ?`) + stmt, err := tx.Prepare(`UPDATE transmissions SET code1 = ` + s.parameter(1) + `, code2 = ` + s.parameter(2) + ` WHERE id = ` + s.parameter(3)) if err != nil { tx.Rollback() return total, fmt.Errorf("transport_codes prepare: %w", err) @@ -1762,7 +1153,7 @@ func (s *Store) LogStats() { // carries the requirement for future callers. // // Ownership (#1283/#1287/#1289): nodes is written by the ingestor only. -// The server opens SQLite mode=ro; its former touchRelayLastSeen has been +// The server uses a restricted PostgreSQL reader; its former touchRelayLastSeen has been // failing on every call since that refactor and is removed in this change. // cmd/server/readonly_invariant_test.go now guards against reintroduction. // @@ -1774,7 +1165,7 @@ func (s *Store) LogStats() { // Never inserts: the UPDATE matches an existing row or does nothing. // Never rewinds: the last_seen guard makes out-of-order ingest a no-op. // UpsertNode's ON CONFLICT clause is monotonic in the same direction -// (MAX(MIN(last_seen, ingestNow), rxTime) reduces to MAX(last_seen, rxTime) +// (GREATEST(LEAST(last_seen, ingestNow), rxTime) reduces to MAX(last_seen, rxTime) // for any non-future stored value), so an ADVERT cannot undo a touch. func (s *Store) touchRelayNodesLocked(pubkeys []string, rxTime string) { if len(pubkeys) == 0 || s.stmtTouchNodeLastSeen == nil { @@ -1838,17 +1229,17 @@ func (s *Store) compactRelayTouched(now time.Time) { // Returns the number of nodes moved. func (s *Store) MoveStaleNodes(nodeDays int) (int64, error) { cutoff := time.Now().UTC().AddDate(0, 0, -nodeDays).Format(time.RFC3339) - tx, err := s.db.Begin() + tx, err := beginWrite(s.db, s.Backend()) if err != nil { return 0, fmt.Errorf("begin tx: %w", err) } defer tx.Rollback() - _, err = tx.Exec(`INSERT OR REPLACE INTO inactive_nodes SELECT * FROM nodes WHERE last_seen < ?`, cutoff) + _, err = tx.Exec(`INSERT INTO inactive_nodes (public_key,name,role,lat,lon,last_seen,first_seen,advert_count,battery_mv,temperature_c,foreign_advert,default_scope,configured_scope,configured_scope_at,multibyte_sup,multibyte_evidence) SELECT public_key,name,role,lat,lon,last_seen,first_seen,advert_count,battery_mv,temperature_c,foreign_advert,default_scope,configured_scope,configured_scope_at,multibyte_sup,multibyte_evidence FROM nodes WHERE last_seen < `+s.parameter(1)+` ON CONFLICT(public_key) DO UPDATE SET name=excluded.name,role=excluded.role,lat=excluded.lat,lon=excluded.lon,last_seen=excluded.last_seen,first_seen=excluded.first_seen,advert_count=excluded.advert_count,battery_mv=excluded.battery_mv,temperature_c=excluded.temperature_c,foreign_advert=excluded.foreign_advert,default_scope=excluded.default_scope,configured_scope=excluded.configured_scope,configured_scope_at=excluded.configured_scope_at,multibyte_sup=excluded.multibyte_sup,multibyte_evidence=excluded.multibyte_evidence`, cutoff) if err != nil { return 0, fmt.Errorf("insert inactive: %w", err) } - result, err := tx.Exec(`DELETE FROM nodes WHERE last_seen < ?`, cutoff) + result, err := tx.Exec(`DELETE FROM nodes WHERE last_seen < `+s.parameter(1), cutoff) if err != nil { return 0, fmt.Errorf("delete stale: %w", err) } @@ -1873,7 +1264,7 @@ func (s *Store) RemoveStaleObservers(observerDays int) (int64, error) { } cutoff := time.Now().UTC().AddDate(0, 0, -observerDays).Format(time.RFC3339) // Tagged for /api/perf writer-lock visibility (#1340). - result, err := s.instrumentedExec("prune_observers", `UPDATE observers SET inactive = 1 WHERE last_seen < ? AND (inactive IS NULL OR inactive = 0)`, cutoff) + result, err := s.instrumentedExec("prune_observers", `UPDATE observers SET inactive = 1 WHERE last_seen < `+s.parameter(1)+` AND (inactive IS NULL OR inactive = 0)`, cutoff) if err != nil { return 0, fmt.Errorf("mark stale observers inactive: %w", err) } @@ -1908,7 +1299,7 @@ func (s *Store) PurgeStaleObservers(purgeDays int) (int64, error) { result, err := s.instrumentedExec("purge_observers", ` DELETE FROM observers WHERE inactive = 1 - AND last_seen < ? + AND last_seen < `+s.parameter(1)+` AND NOT EXISTS (SELECT 1 FROM observations o WHERE o.observer_idx = observers.rowid) AND NOT EXISTS (SELECT 1 FROM observer_metrics m WHERE m.observer_id = observers.id) AND NOT EXISTS (SELECT 1 FROM dropped_packets d WHERE d.observer_id = observers.id)`, cutoff) @@ -1935,9 +1326,13 @@ type DroppedPacket struct { // InsertDroppedPacket records a rejected packet in the dropped_packets table. func (s *Store) InsertDroppedPacket(dp *DroppedPacket) error { + // The decoder's byte cap can split a UTF-8 rune. Drop incomplete bytes + // without expanding the name past that cap; raw_hex retains the evidence. + nodeName := strings.ToValidUTF8(dp.NodeName, "") + observerName := clampObserverField(dp.ObserverName, maxObserverTextLen) _, err := s.db.Exec( - `INSERT INTO dropped_packets (hash, raw_hex, reason, observer_id, observer_name, node_pubkey, node_name) VALUES (?, ?, ?, ?, ?, ?, ?)`, - dp.Hash, dp.RawHex, dp.Reason, dp.ObserverID, dp.ObserverName, dp.NodePubKey, dp.NodeName, + `INSERT INTO dropped_packets (hash, raw_hex, reason, observer_id, observer_name, node_pubkey, node_name) VALUES (`+s.parameter(1)+`, `+s.parameter(2)+`, `+s.parameter(3)+`, `+s.parameter(4)+`, `+s.parameter(5)+`, `+s.parameter(6)+`, `+s.parameter(7)+`)`, + dp.Hash, dp.RawHex, dp.Reason, dp.ObserverID, observerName, dp.NodePubKey, nodeName, ) if err != nil { s.Stats.WriteErrors.Add(1) @@ -1953,7 +1348,7 @@ func (s *Store) PruneDroppedPackets(retentionDays int) (int64, error) { return 0, nil } cutoff := time.Now().UTC().AddDate(0, 0, -retentionDays).Format(time.RFC3339) - result, err := s.db.Exec(`DELETE FROM dropped_packets WHERE dropped_at < ?`, cutoff) + result, err := s.db.Exec(`DELETE FROM dropped_packets WHERE dropped_at < `+s.parameter(1), cutoff) if err != nil { return 0, fmt.Errorf("prune dropped packets: %w", err) } @@ -1998,7 +1393,7 @@ func nilIfEmpty(s string) interface{} { return s } -// boolToInt converts a bool to SQLite's 0/1 INTEGER representation. +// boolToInt converts a bool to the retained 0/1 integer representation. func boolToInt(b bool) int { if b { return 1 @@ -2032,15 +1427,127 @@ func (s *Store) UpdateNodeDefaultScope(pubkey, scope string) error { } // Short-circuit: skip if already stored. var cur sql.NullString - row := s.db.QueryRow(`SELECT default_scope FROM nodes WHERE public_key = ?`, pubkey) + row := s.db.QueryRow(`SELECT default_scope FROM nodes WHERE public_key = `+s.parameter(1), pubkey) if row.Scan(&cur) == nil && cur.Valid && cur.String == scope { return nil } - if _, err := s.db.Exec(`UPDATE nodes SET default_scope = ? WHERE public_key = ?`, scope, pubkey); err != nil { + if _, err := s.db.Exec(`UPDATE nodes SET default_scope = `+s.parameter(1)+` WHERE public_key = `+s.parameter(2), scope, pubkey); err != nil { return err } // Mirror to inactive_nodes (node may be there if recently moved by retention). - _, err := s.db.Exec(`UPDATE inactive_nodes SET default_scope = ? WHERE public_key = ?`, scope, pubkey) + _, err := s.db.Exec(`UPDATE inactive_nodes SET default_scope = `+s.parameter(1)+` WHERE public_key = `+s.parameter(2), scope, pubkey) + return err +} + +// normalizeReportTS parses an observer report timestamp and returns it in +// canonical UTC RFC3339 form ("2006-01-02T15:04:05Z"). It accepts both the +// firmware's fractional/offset form (e.g. "2026-07-26T09:43:48.000000+00:00") +// and plain "Z"/offset variants. An empty or unparseable input returns "" so +// the caller writes an empty configured_scope_at and skips the ordering guard; +// this keeps every stored timestamp either canonical or empty, never a mix of +// offset/precision formats that would break lexicographic last-write-wins. +func normalizeReportTS(raw string) string { + if raw == "" { + return "" + } + for _, layout := range []string{time.RFC3339Nano, time.RFC3339} { + if t, err := time.Parse(layout, raw); err == nil { + return t.UTC().Format(time.RFC3339) + } + } + return "" +} + +// normalizeScopeList canonicalises the comma-separated region-scope string an +// observer reports for a node, so configured_scope is directly comparable with +// default_scope instead of only looking similar (#1865, raised by @cwichura). +// +// The OTA scope query returns bare region names ("dk,eu,dk-aarhus"), while +// everything else in CoreScope carries the leading "#": every stored +// default_scope value on a live 1425-node instance starts with one (361 of +// 361), and the hashRegions/hashChannels config paths already prefix a missing +// "#" before deriving the channel key from it (main.go). The "#" is therefore +// part of the region's name here, not decoration, so a bare "dk" and a stored +// "#dk" are the same region and must be written the same way. +// +// Rules, in order: entries are trimmed; empty entries are dropped; "*" is a +// wildcard rather than a region name and is passed through untouched; anything +// else gains a leading "#" if it lacks one. Order is preserved and duplicates +// are NOT collapsed, because the observer's ordering carries meaning we do not +// interpret here. Case is left alone: region keys are derived as +// sha256("#name"), so folding case would silently retarget a scope. +// +// An empty or all-empty input returns "", which UpdateNodeConfiguredScope +// stores as a valid "responded, no scopes configured" statement. +func normalizeScopeList(raw string) string { + if strings.TrimSpace(raw) == "" { + return "" + } + parts := strings.Split(raw, ",") + out := make([]string, 0, len(parts)) + for _, p := range parts { + p = strings.TrimSpace(p) + if p == "" { + continue + } + if p != "*" && !strings.HasPrefix(p, "#") { + p = "#" + p + } + out = append(out, p) + } + return strings.Join(out, ",") +} + +// UpdateNodeConfiguredScope records the region scopes a node has CONFIGURED, +// as concrete evidence from an observer /neighbors report (#1865). Unlike +// UpdateNodeDefaultScope (inferred, overwritten on every observation), this is +// only called for the observer's own `self` scopes and for neighbors whose OTA +// scope query returned status="responded" — so the caller, not this method, +// gates on status. An empty scope IS a valid "responded, no scopes configured" +// statement and is stored; a timeout must simply not reach this method. +// +// reportedAt is the report envelope timestamp (ISO-8601). It is stored in +// configured_scope_at and used for last-write-wins: an out-of-order older +// report must not clobber a newer confirmed value. A blank reportedAt skips +// the ordering guard (always writes). +func (s *Store) UpdateNodeConfiguredScope(pubkey, scope, reportedAt string) error { + if pubkey == "" { + return nil + } + // Normalize to canonical UTC RFC3339 so the last-write-wins comparison is + // chronological, not lexicographic (see normalizeReportTS). Stored values + // are therefore always canonical or empty. + reportedAt = normalizeReportTS(reportedAt) + // The report's timestamp is chosen by the publisher. Last-write-wins + // below means a report stamped far in the future would be written once + // and then block every genuine later report, so drop those. + if reportTooFarInFuture(reportedAt) { + return nil + } + // Canonicalise the scope syntax for the same reason: a value that is stored + // differently from default_scope cannot be compared against it (see + // normalizeScopeList). + scope = normalizeScopeList(scope) + if len(scope) > maxConfiguredScopeLen { + return nil + } + // Last-write-wins: skip if the stored confirmation is newer-or-equal. + if reportedAt != "" { + var curAt sql.NullString + row := s.db.QueryRow(`SELECT configured_scope_at FROM nodes WHERE public_key = `+s.parameter(1), pubkey) + if row.Scan(&curAt) == nil && curAt.Valid && curAt.String != "" && curAt.String >= reportedAt { + return nil + } + } + if _, err := s.db.Exec( + `UPDATE nodes SET configured_scope = `+s.parameter(1)+`, configured_scope_at = `+s.parameter(2)+` WHERE public_key = `+s.parameter(3), + scope, reportedAt, pubkey); err != nil { + return err + } + // Mirror to inactive_nodes (node may be there if recently moved by retention). + _, err := s.db.Exec( + `UPDATE inactive_nodes SET configured_scope = `+s.parameter(1)+`, configured_scope_at = `+s.parameter(2)+` WHERE public_key = `+s.parameter(3), + scope, reportedAt, pubkey) return err } @@ -2064,14 +1571,14 @@ func (s *Store) RecordNaiveSkew(observerID string, deltaSec int64, now time.Time // increments it. _, err := s.db.Exec(` INSERT INTO observers (id, clock_skew_seconds, clock_skew_count_24h, clock_last_naive_at) - VALUES (?, ?, 1, ?) + VALUES (`+s.parameter(1)+`, `+s.parameter(2)+`, 1, `+s.parameter(3)+`) ON CONFLICT(id) DO UPDATE SET clock_skew_seconds = excluded.clock_skew_seconds, clock_last_naive_at = excluded.clock_last_naive_at, clock_skew_count_24h = CASE - WHEN clock_last_naive_at IS NULL OR clock_last_naive_at < ? + WHEN observers.clock_last_naive_at IS NULL OR observers.clock_last_naive_at < `+s.parameter(4)+` THEN 1 - ELSE COALESCE(clock_skew_count_24h, 0) + 1 + ELSE COALESCE(observers.clock_skew_count_24h, 0) + 1 END `, observerID, deltaSec, nowStr, cutoff) return err @@ -2104,7 +1611,7 @@ type MQTTPacketMessage struct { // into the past. Packet ordering is owned by the server clock; client // clocks are untrusted. msg.Timestamp still flows into observer.last_seen // via UpsertObserverAt — that's #1233's MAX/MIN guarded path and is fine. -func BuildPacketData(msg *MQTTPacketMessage, decoded *DecodedPacket, observerID, region string, regionKeys map[string][]byte) *PacketData { +func BuildPacketData(msg *MQTTPacketMessage, decoded *DecodedPacket, observerID, region string, regionSet *regionKeySet) *PacketData { pathJSON := "[]" // For TRACE packets, path_json must be the payload-decoded route hops // (decoded.Path.Hops), NOT the raw_hex header bytes which are SNR values. @@ -2157,7 +1664,7 @@ func BuildPacketData(msg *MQTTPacketMessage, decoded *DecodedPacket, observerID, pd.Code2 = decoded.TransportCodes.Code2 if decoded.TransportCodes.Code1 != "0000" { pd.IsTransportScoped = true - pd.ScopeName = matchScope(regionKeys, byte(decoded.Header.PayloadType), decoded.payloadRaw, decoded.TransportCodes.Code1) + pd.ScopeName = regionSet.matchScopeName(byte(decoded.Header.PayloadType), decoded.payloadRaw, decoded.TransportCodes.Code1) } } @@ -2173,7 +1680,7 @@ func BuildPacketData(msg *MQTTPacketMessage, decoded *DecodedPacket, observerID, // ─── Writer-lock instrumentation (issue #1340) ──────────────────────────── // -// Make SQLite writer-lock starvation visible to operators. Per-component +// Make serialized writer-lock starvation visible to operators. Per-component // wait_ms / hold_ms / contention_total histograms, surfaced via // /api/perf/write-sources under the "writer_perf" key. Component tags: // neighbor_builder, mqtt_handler, prune_packets, prune_observers, @@ -2185,7 +1692,7 @@ func BuildPacketData(msg *MQTTPacketMessage, decoded *DecodedPacket, observerID, // every wrapped call site through the same package-level mutex. // WriterStatsSnapshot is a per-component wait/hold latency snapshot -// surfaced via /api/perf to make SQLite writer-lock starvation visible +// surfaced via /api/perf to make serialized writer-lock starvation visible // to operators (issue #1340). Times are in milliseconds. type WriterStatsSnapshot struct { Count int64 `json:"count"` @@ -2401,7 +1908,7 @@ func recordWriterTiming(component string, wait, hold time.Duration, queryForLog // writerMu serialises every wrapped writer call so the wait the next // caller sees is the wait the perf snapshot can attribute. The -// SQLite driver also enforces serial writes (SetMaxOpenConns(1)), +// connection pool also serializes writes (SetMaxOpenConns(1)), // but the wait inside the driver is invisible to Go — writerMu makes // it Go-visible. var writerMu sync.Mutex @@ -2427,7 +1934,7 @@ func (s *Store) WriterTx(component string, fn func(*sql.Tx) error) error { writerMu.Lock() wait := time.Since(waitStart) holdStart := time.Now() - tx, err := s.db.Begin() + tx, err := beginWrite(s.db, s.Backend()) if err != nil { hold := time.Since(holdStart) writerMu.Unlock() diff --git a/cmd/ingestor/db_test.go b/cmd/ingestor/db_test.go index 1cea8a57e..5d0ef086b 100644 --- a/cmd/ingestor/db_test.go +++ b/cmd/ingestor/db_test.go @@ -13,6 +13,7 @@ import ( "time" "github.com/meshcore-analyzer/packetpath" + "github.com/meshcore-analyzer/pgutil" ) func tempDBPath(t *testing.T) string { @@ -33,14 +34,14 @@ func tempDBPath(t *testing.T) string { } func TestOpenStore(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } defer s.Close() // Verify tables exist - rows, err := s.db.Query("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name") + rows, err := s.db.Query(testNativeSQL(`SELECT name FROM sqlite_master WHERE type='table' ORDER BY name`, "SELECT table_name FROM information_schema.tables WHERE table_schema=current_schema() ORDER BY table_name")) if err != nil { t.Fatal(err) } @@ -69,7 +70,7 @@ func TestOpenStore(t *testing.T) { // Verify packets_v view exists var viewCount int - err = s.db.QueryRow("SELECT COUNT(*) FROM sqlite_master WHERE type='view' AND name='packets_v'").Scan(&viewCount) + err = s.db.QueryRow(testNativeSQL(`SELECT COUNT(*) FROM sqlite_master WHERE type='view' AND name='packets_v'`, "SELECT COUNT(*) FROM information_schema.views WHERE table_schema=current_schema() AND table_name='packets_v'")).Scan(&viewCount) if err != nil { t.Fatal(err) } @@ -79,7 +80,7 @@ func TestOpenStore(t *testing.T) { } func TestInsertTransmission(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -129,7 +130,7 @@ func TestInsertTransmission(t *testing.T) { } func TestPacketsViewQueryable(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -177,7 +178,7 @@ func TestPacketsViewQueryable(t *testing.T) { } func TestUpsertNode(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -216,7 +217,7 @@ func TestUpsertNode(t *testing.T) { } func TestUpsertObserver(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -237,7 +238,7 @@ func TestUpsertObserver(t *testing.T) { } func TestUpsertObserverWithMeta(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -298,21 +299,21 @@ func TestUpsertObserverWithMeta(t *testing.T) { // Verify typeof returns correct SQLite types var typBattery, typUptime, typNoise string - s.db.QueryRow("SELECT typeof(battery_mv), typeof(uptime_secs), typeof(noise_floor) FROM observers WHERE id = 'obs1'"). + s.db.QueryRow(testNativeSQL(`SELECT typeof(battery_mv), typeof(uptime_secs), typeof(noise_floor) FROM observers WHERE id = 'obs1'`, "SELECT pg_typeof(battery_mv)::text, pg_typeof(uptime_secs)::text, pg_typeof(noise_floor)::text FROM observers WHERE id = 'obs1'")). Scan(&typBattery, &typUptime, &typNoise) - if typBattery != "integer" { + if typBattery != testNativeSQL("integer", "bigint") { t.Errorf("typeof(battery_mv)=%s, want integer", typBattery) } - if typUptime != "integer" { + if typUptime != testNativeSQL("integer", "bigint") { t.Errorf("typeof(uptime_secs)=%s, want integer", typUptime) } - if typNoise != "real" { + if typNoise != testNativeSQL("real", "double precision") { t.Errorf("typeof(noise_floor)=%s, want real", typNoise) } } func TestUpsertObserverMetaPreservesExisting(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -453,14 +454,14 @@ func TestExtractObserverMeta(t *testing.T) { } func TestSchemaNoiseFloorIsReal(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } defer s.Close() // Check column type affinity via PRAGMA - rows, err := s.db.Query("PRAGMA table_info(observers)") + rows, err := s.db.Query(testNativeSQL(`SELECT cid,name,type,"notnull",dflt_value,pk FROM pragma_table_info('observers') ORDER BY cid`, `SELECT ordinal_position::int,column_name,CASE data_type WHEN 'bigint' THEN 'INTEGER' WHEN 'double precision' THEN 'REAL' ELSE upper(data_type) END,CASE is_nullable WHEN 'NO' THEN 1 ELSE 0 END,column_default,0 FROM information_schema.columns WHERE table_schema=current_schema() AND table_name='observers' ORDER BY ordinal_position`)) if err != nil { t.Fatal(err) } @@ -486,7 +487,7 @@ func TestSchemaNoiseFloorIsReal(t *testing.T) { } func TestInsertTransmissionWithObserver(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -522,7 +523,7 @@ func TestInsertTransmissionWithObserver(t *testing.T) { // #463: Verify that inserting a packet updates the observer's last_seen, // so low-traffic observers don't incorrectly appear offline. func TestInsertTransmissionUpdatesObserverLastSeen(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -534,11 +535,11 @@ func TestInsertTransmissionUpdatesObserverLastSeen(t *testing.T) { } // Backdate last_seen to 2 hours ago oldTime := "2026-03-24T22:00:00Z" - s.db.Exec("UPDATE observers SET last_seen = ? WHERE id = ?", oldTime, "obs1") + s.db.Exec("UPDATE observers SET last_seen = $1 WHERE id = $2", oldTime, "obs1") // Verify it was backdated var lastSeenBefore string - s.db.QueryRow("SELECT last_seen FROM observers WHERE id = ?", "obs1").Scan(&lastSeenBefore) + s.db.QueryRow("SELECT last_seen FROM observers WHERE id = $1", "obs1").Scan(&lastSeenBefore) if lastSeenBefore != oldTime { t.Fatalf("expected last_seen=%s, got %s", oldTime, lastSeenBefore) } @@ -562,7 +563,7 @@ func TestInsertTransmissionUpdatesObserverLastSeen(t *testing.T) { // Verify last_seen was updated to INGEST time, not envelope time (#1465). var lastSeenAfter string - s.db.QueryRow("SELECT last_seen FROM observers WHERE id = ?", "obs1").Scan(&lastSeenAfter) + s.db.QueryRow("SELECT last_seen FROM observers WHERE id = $1", "obs1").Scan(&lastSeenAfter) if lastSeenAfter == oldTime { t.Error("observer last_seen was NOT updated after packet insertion — low-traffic observers will appear offline") } @@ -578,7 +579,7 @@ func TestInsertTransmissionUpdatesObserverLastSeen(t *testing.T) { } func TestLastPacketAtUpdatedOnPacketOnly(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -590,7 +591,7 @@ func TestLastPacketAtUpdatedOnPacketOnly(t *testing.T) { } var lastPacketAt sql.NullString - s.db.QueryRow("SELECT last_packet_at FROM observers WHERE id = ?", "obs1").Scan(&lastPacketAt) + s.db.QueryRow("SELECT last_packet_at FROM observers WHERE id = $1", "obs1").Scan(&lastPacketAt) if lastPacketAt.Valid { t.Fatalf("expected last_packet_at to be NULL after UpsertObserver, got %s", lastPacketAt.String) } @@ -612,7 +613,7 @@ func TestLastPacketAtUpdatedOnPacketOnly(t *testing.T) { } after := time.Now().Unix() - s.db.QueryRow("SELECT last_packet_at FROM observers WHERE id = ?", "obs1").Scan(&lastPacketAt) + s.db.QueryRow("SELECT last_packet_at FROM observers WHERE id = $1", "obs1").Scan(&lastPacketAt) if !lastPacketAt.Valid { t.Fatal("expected last_packet_at to be non-NULL after InsertTransmission") } @@ -634,14 +635,14 @@ func TestLastPacketAtUpdatedOnPacketOnly(t *testing.T) { } var lastPacketAtAfterStatus sql.NullString - s.db.QueryRow("SELECT last_packet_at FROM observers WHERE id = ?", "obs1").Scan(&lastPacketAtAfterStatus) + s.db.QueryRow("SELECT last_packet_at FROM observers WHERE id = $1", "obs1").Scan(&lastPacketAtAfterStatus) if !lastPacketAtAfterStatus.Valid || lastPacketAtAfterStatus.String != lastPacketAt.String { t.Errorf("UpsertObserver should not change last_packet_at; expected %s, got %v", lastPacketAt.String, lastPacketAtAfterStatus) } } func TestEndToEndIngest(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -677,7 +678,7 @@ func TestEndToEndIngest(t *testing.T) { // Verify node was created var nodeName string - err = s.db.QueryRow("SELECT name FROM nodes WHERE public_key = ?", decoded.Payload.PubKey).Scan(&nodeName) + err = s.db.QueryRow("SELECT name FROM nodes WHERE public_key = $1", decoded.Payload.PubKey).Scan(&nodeName) if err != nil { t.Fatal(err) } @@ -687,7 +688,7 @@ func TestEndToEndIngest(t *testing.T) { } func TestInsertTransmissionEmptyHash(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -711,7 +712,7 @@ func TestInsertTransmissionEmptyHash(t *testing.T) { } func TestInsertTransmissionEmptyTimestamp(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -729,14 +730,14 @@ func TestInsertTransmissionEmptyTimestamp(t *testing.T) { } var firstSeen string - s.db.QueryRow("SELECT first_seen FROM transmissions WHERE hash = ?", data.Hash).Scan(&firstSeen) + s.db.QueryRow("SELECT first_seen FROM transmissions WHERE hash = $1", data.Hash).Scan(&firstSeen) if firstSeen == "" { t.Error("first_seen should be set even with empty timestamp") } } func TestInsertTransmissionEarlierFirstSeen(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -765,14 +766,14 @@ func TestInsertTransmissionEarlierFirstSeen(t *testing.T) { } var firstSeen string - s.db.QueryRow("SELECT first_seen FROM transmissions WHERE hash = ?", data.Hash).Scan(&firstSeen) + s.db.QueryRow("SELECT first_seen FROM transmissions WHERE hash = $1", data.Hash).Scan(&firstSeen) if firstSeen != "2026-03-25T06:00:00Z" { t.Errorf("first_seen=%s, want 2026-03-25T06:00:00Z (earlier timestamp)", firstSeen) } } func TestInsertTransmissionLaterFirstSeenNotUpdated(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -800,14 +801,14 @@ func TestInsertTransmissionLaterFirstSeenNotUpdated(t *testing.T) { } var firstSeen string - s.db.QueryRow("SELECT first_seen FROM transmissions WHERE hash = ?", data.Hash).Scan(&firstSeen) + s.db.QueryRow("SELECT first_seen FROM transmissions WHERE hash = $1", data.Hash).Scan(&firstSeen) if firstSeen != "2026-03-25T06:00:00Z" { t.Errorf("first_seen=%s should not change to later time", firstSeen) } } func TestInsertTransmissionNilSNRRSSI(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -926,7 +927,7 @@ func TestBuildPacketDataNilSNRRSSI(t *testing.T) { } func TestUpsertNodeEmptyLastSeen(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -949,14 +950,14 @@ func TestUpsertNodeEmptyLastSeen(t *testing.T) { func TestOpenStoreTwice(t *testing.T) { // Opening same DB twice tests the "observations already exists" path in applySchema path := tempDBPath(t) - s1, err := OpenStore(path) + s1, err := openPostgresTestStore(t, path) if err != nil { t.Fatal(err) } s1.Close() // Second open — observations table already exists - s2, err := OpenStore(path) + s2, err := openPostgresTestStore(t, path) if err != nil { t.Fatal(err) } @@ -971,7 +972,7 @@ func TestOpenStoreTwice(t *testing.T) { } func TestInsertTransmissionDedupObservation(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -1005,7 +1006,7 @@ func TestInsertTransmissionDedupObservation(t *testing.T) { } func TestSchemaCompatibility(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -1013,7 +1014,7 @@ func TestSchemaCompatibility(t *testing.T) { // Verify column names match what Node.js expects expectedTxCols := []string{"id", "raw_hex", "hash", "first_seen", "route_type", "payload_type", "payload_version", "decoded_json", "created_at"} - rows, _ := s.db.Query("PRAGMA table_info(transmissions)") + rows, _ := s.db.Query(testNativeSQL(`SELECT cid,name,type,"notnull",dflt_value,pk FROM pragma_table_info('transmissions') ORDER BY cid`, `SELECT ordinal_position::int,column_name,CASE data_type WHEN 'bigint' THEN 'INTEGER' WHEN 'double precision' THEN 'REAL' ELSE upper(data_type) END,CASE is_nullable WHEN 'NO' THEN 1 ELSE 0 END,column_default,0 FROM information_schema.columns WHERE table_schema=current_schema() AND table_name='transmissions' ORDER BY ordinal_position`)) var txCols []string for rows.Next() { var cid int @@ -1041,7 +1042,7 @@ func TestSchemaCompatibility(t *testing.T) { // Verify observations columns expectedObsCols := []string{"id", "transmission_id", "observer_idx", "direction", "snr", "rssi", "score", "path_json", "timestamp"} - rows, _ = s.db.Query("PRAGMA table_info(observations)") + rows, _ = s.db.Query(testNativeSQL(`SELECT cid,name,type,"notnull",dflt_value,pk FROM pragma_table_info('observations') ORDER BY cid`, `SELECT ordinal_position::int,column_name,CASE data_type WHEN 'bigint' THEN 'INTEGER' WHEN 'double precision' THEN 'REAL' ELSE upper(data_type) END,CASE is_nullable WHEN 'NO' THEN 1 ELSE 0 END,column_default,0 FROM information_schema.columns WHERE table_schema=current_schema() AND table_name='observations' ORDER BY ordinal_position`)) var obsCols []string for rows.Next() { var cid int @@ -1069,7 +1070,7 @@ func TestSchemaCompatibility(t *testing.T) { } func TestConcurrentWrites(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -1167,7 +1168,7 @@ func TestConcurrentWrites(t *testing.T) { } func TestDBStats(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -1231,7 +1232,7 @@ func TestDBStats(t *testing.T) { } func TestLoadTestThroughput(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -1372,7 +1373,7 @@ func TestLoadTestThroughput(t *testing.T) { } func TestUpdateNodeTelemetry(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -1420,7 +1421,7 @@ func TestUpdateNodeTelemetry(t *testing.T) { } func TestTelemetryMigrationAddsColumns(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -1437,9 +1438,9 @@ func TestTelemetryMigrationAddsColumns(t *testing.T) { } var count int - s.db.QueryRow("SELECT COUNT(*) FROM _migrations WHERE name = 'node_telemetry_v1'").Scan(&count) + s.db.QueryRow(testNativeSQL(`SELECT COUNT(*) FROM _migrations WHERE name='observers_identity_autoincrement_v1'`, "SELECT COUNT(*) FROM corescope_schema WHERE kind='telemetry' AND ready=true")).Scan(&count) if count != 1 { - t.Errorf("migration node_telemetry_v1 should be recorded, count=%d", count) + t.Errorf("canonical telemetry schema should be ready, count=%d", count) } } @@ -1591,7 +1592,7 @@ func TestObsTimestampIndexMigration(t *testing.T) { // Case 1: new DB — OpenStore should create idx_observations_timestamp as part // of the observations table schema. t.Run("NewDB", func(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -1599,7 +1600,7 @@ func TestObsTimestampIndexMigration(t *testing.T) { var count int err = s.db.QueryRow( - "SELECT COUNT(*) FROM sqlite_master WHERE type='index' AND name='idx_observations_timestamp'", + testNativeSQL(`SELECT COUNT(*) FROM sqlite_master WHERE type='index' AND name='idx_observations_timestamp'`, "SELECT COUNT(*) FROM pg_indexes WHERE schemaname=current_schema() AND indexname='idx_observations_timestamp'"), ).Scan(&count) if err != nil { t.Fatal(err) @@ -1622,83 +1623,35 @@ func TestObsTimestampIndexMigration(t *testing.T) { // Case 2: existing DB that has the observations table but lacks the index // and lacks the _migrations entry — simulates an older installation. - t.Run("MigrationPath", func(t *testing.T) { - path := tempDBPath(t) - - // Build a bare-bones DB that mimics an old installation: - // observations table exists but idx_observations_timestamp does NOT. - db, err := sql.Open("sqlite", path) - if err != nil { - t.Fatal(err) - } - _, err = db.Exec(` - CREATE TABLE IF NOT EXISTS _migrations (name TEXT PRIMARY KEY); - CREATE TABLE IF NOT EXISTS transmissions ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - raw_hex TEXT NOT NULL, - hash TEXT NOT NULL UNIQUE, - first_seen TEXT NOT NULL, - route_type INTEGER, - payload_type INTEGER, - payload_version INTEGER, - decoded_json TEXT, - created_at TEXT DEFAULT (datetime('now')) - ); - CREATE TABLE IF NOT EXISTS observations ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - transmission_id INTEGER NOT NULL REFERENCES transmissions(id), - observer_idx INTEGER, - direction TEXT, - snr REAL, - rssi REAL, - score INTEGER, - path_json TEXT, - timestamp INTEGER NOT NULL - ); - `) - if err != nil { - db.Close() - t.Fatal(err) - } - // Confirm the index is absent before OpenStore runs. - var preCount int - db.QueryRow( - "SELECT COUNT(*) FROM sqlite_master WHERE type='index' AND name='idx_observations_timestamp'", - ).Scan(&preCount) - db.Close() - if preCount != 0 { - t.Fatalf("pre-condition failed: idx_observations_timestamp should not exist yet, got count=%d", preCount) - } - - // Now open via OpenStore — the migration should add the index. - s, err := OpenStore(path) + t.Run("MissingIndexIsRefused", func(t *testing.T) { + key := tempDBPath(t) + store, err := openPostgresTestStore(t, key) if err != nil { t.Fatal(err) } - defer s.Close() - - var idxCount int - err = s.db.QueryRow( - "SELECT COUNT(*) FROM sqlite_master WHERE type='index' AND name='idx_observations_timestamp'", - ).Scan(&idxCount) - if err != nil { + owner := testAdmin(t, store) + if _, err := owner.Exec(`DROP INDEX idx_observations_timestamp`); err != nil { t.Fatal(err) } - if idxCount != 1 { - t.Error("idx_observations_timestamp should exist after migration on old DB") - } - - var migCount int - err = s.db.QueryRow( - "SELECT COUNT(*) FROM _migrations WHERE name='obs_timestamp_index_v1'", - ).Scan(&migCount) - if err != nil { - t.Fatal(err) + store.Close() + if store.Backend() == "sqlite" { + next, err := openPostgresTestStore(t, key) + if err != nil { + t.Fatal(err) + } + defer next.Close() + var count int + if err = next.db.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE type='index' AND name='idx_observations_timestamp'`).Scan(&count); err != nil || count != 1 { + t.Fatalf("SQLite writer did not restore its index: %d %v", count, err) + } + return } - if migCount != 1 { - t.Errorf("migration obs_timestamp_index_v1 should be recorded, got count=%d", migCount) + if next, err := openPostgresTestStore(t, key); err == nil { + next.Close() + t.Fatal("runtime accepted missing canonical index") } }) + } func TestBuildPacketDataScoreAndDirection(t *testing.T) { @@ -1739,7 +1692,7 @@ func TestBuildPacketDataNilScoreDirection(t *testing.T) { } func TestInsertTransmissionWithScoreAndDirection(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -1805,7 +1758,7 @@ func TestRoundToInterval(t *testing.T) { } func TestInsertMetrics(t *testing.T) { - store, err := OpenStore(tempDBPath(t)) + store, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -1849,7 +1802,7 @@ func TestInsertMetrics(t *testing.T) { } func TestInsertMetricsIdempotent(t *testing.T) { - store, err := OpenStoreWithInterval(tempDBPath(t), 300) + store, err := openPostgresTestStoreInterval(t, tempDBPath(t), 300) if err != nil { t.Fatal(err) } @@ -1879,7 +1832,7 @@ func TestInsertMetricsIdempotent(t *testing.T) { } func TestInsertMetricsNullFields(t *testing.T) { - store, err := OpenStore(tempDBPath(t)) + store, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -1908,7 +1861,7 @@ func TestInsertMetricsNullFields(t *testing.T) { } func TestPruneOldMetrics(t *testing.T) { - store, err := OpenStore(tempDBPath(t)) + store, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -1917,8 +1870,8 @@ func TestPruneOldMetrics(t *testing.T) { // Insert old and new metrics directly oldTs := time.Now().UTC().AddDate(0, 0, -40).Format(time.RFC3339) newTs := time.Now().UTC().Format(time.RFC3339) - store.db.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor) VALUES (?, ?, ?)", "obs1", oldTs, -110.0) - store.db.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor) VALUES (?, ?, ?)", "obs1", newTs, -112.0) + store.db.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor) VALUES ($1, $2, $3)", "obs1", oldTs, -110.0) + store.db.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor) VALUES ($1, $2, $3)", "obs1", newTs, -112.0) n, err := store.PruneOldMetrics(30) if err != nil { @@ -1975,7 +1928,7 @@ func TestExtractObserverMetaNewFields(t *testing.T) { // idx_observations_dedup CREATE UNIQUE INDEX statements elsewhere in this file.) func TestInsertObservationSNRFillIn(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -2058,7 +2011,7 @@ func TestInsertObservationSNRFillIn(t *testing.T) { // TestPerObservationRawHex verifies that two MQTT packets for the same hash // from different observers store distinct raw_hex per observation (#881). func TestPerObservationRawHex(t *testing.T) { - store, err := OpenStore(tempDBPath(t)) + store, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -2210,14 +2163,14 @@ func TestBuildPacketData_NonTracePathJSON(t *testing.T) { func TestScopeNameMigration(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "test.db") - store, err := OpenStore(dbPath) + store, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore: %v", err) } defer store.Close() // Verify column exists - rows, err := store.db.Query("PRAGMA table_info(transmissions)") + rows, err := store.db.Query(testNativeSQL(`SELECT cid,name,type,"notnull",dflt_value,pk FROM pragma_table_info('transmissions') ORDER BY cid`, `SELECT ordinal_position::int,column_name,CASE data_type WHEN 'bigint' THEN 'INTEGER' WHEN 'double precision' THEN 'REAL' ELSE upper(data_type) END,CASE is_nullable WHEN 'NO' THEN 1 ELSE 0 END,column_default,0 FROM information_schema.columns WHERE table_schema=current_schema() AND table_name='transmissions' ORDER BY ordinal_position`)) if err != nil { t.Fatalf("PRAGMA: %v", err) } @@ -2266,7 +2219,7 @@ func TestScopeNameMigration(t *testing.T) { } func TestTransportCodesStored(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -2307,7 +2260,7 @@ func TestTransportCodesStored(t *testing.T) { } var c1, c2 *string - if err := s.db.QueryRow(`SELECT code1, code2 FROM transmissions WHERE hash = ?`, + if err := s.db.QueryRow(`SELECT code1, code2 FROM transmissions WHERE hash = $1`, ComputeContentHash(rawScoped)).Scan(&c1, &c2); err != nil { t.Fatalf("read scoped: %v", err) } @@ -2315,7 +2268,7 @@ func TestTransportCodesStored(t *testing.T) { t.Errorf("scoped codes = %v/%v, want AABB/CCDD", c1, c2) } - if err := s.db.QueryRow(`SELECT code1, code2 FROM transmissions WHERE hash = ?`, + if err := s.db.QueryRow(`SELECT code1, code2 FROM transmissions WHERE hash = $1`, ComputeContentHash(rawUnscoped)).Scan(&c1, &c2); err != nil { t.Fatalf("read unscoped: %v", err) } @@ -2323,7 +2276,7 @@ func TestTransportCodesStored(t *testing.T) { t.Errorf("unscoped codes = %v/%v, want NULL/NULL", c1, c2) } - if err := s.db.QueryRow(`SELECT code1, code2 FROM transmissions WHERE hash = ?`, + if err := s.db.QueryRow(`SELECT code1, code2 FROM transmissions WHERE hash = $1`, ComputeContentHash(rawZeroCodes)).Scan(&c1, &c2); err != nil { t.Fatalf("read zero-codes: %v", err) } @@ -2349,7 +2302,7 @@ func TestBackfillTransportCodes(t *testing.T) { }{{rawScoped, 0}, {rawFlood, 1}} { if _, err := s.db.Exec( `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type) - VALUES (?,?,?,?,?)`, + VALUES ($1,$2,$3,$4,$5)`, tc.raw, ComputeContentHash(tc.raw), "2026-08-01T00:00:00Z", tc.routeType, 5); err != nil { t.Fatalf("seed: %v", err) } @@ -2359,7 +2312,7 @@ func TestBackfillTransportCodes(t *testing.T) { s.backfillWg.Wait() var c1 *string - if err := s.db.QueryRow(`SELECT code1 FROM transmissions WHERE hash = ?`, + if err := s.db.QueryRow(`SELECT code1 FROM transmissions WHERE hash = $1`, ComputeContentHash(rawScoped)).Scan(&c1); err != nil { t.Fatalf("read: %v", err) } @@ -2367,7 +2320,7 @@ func TestBackfillTransportCodes(t *testing.T) { t.Errorf("backfilled code1 = %v, want AABB", c1) } - if err := s.db.QueryRow(`SELECT code1 FROM transmissions WHERE hash = ?`, + if err := s.db.QueryRow(`SELECT code1 FROM transmissions WHERE hash = $1`, ComputeContentHash(rawFlood)).Scan(&c1); err != nil { t.Fatalf("read flood: %v", err) } @@ -2415,14 +2368,14 @@ func TestBackfillTransportCodes_MultiBatchWithUndecodableRow(t *testing.T) { // good2}, batch 2 = {good3, good4, good5}, batch 3 = {good6}. if _, err := s.db.Exec( `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type) - VALUES (?,?,?,?,?)`, + VALUES ($1,$2,$3,$4,$5)`, badRaw, ComputeContentHash(badRaw), "2026-08-01T00:00:00Z", 0, 5); err != nil { t.Fatalf("seed bad row: %v", err) } for _, raw := range goodRaws { if _, err := s.db.Exec( `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type) - VALUES (?,?,?,?,?)`, + VALUES ($1,$2,$3,$4,$5)`, raw, ComputeContentHash(raw), "2026-08-01T00:00:00Z", 0, 5); err != nil { t.Fatalf("seed good row: %v", err) } @@ -2439,7 +2392,7 @@ func TestBackfillTransportCodes_MultiBatchWithUndecodableRow(t *testing.T) { for i, raw := range goodRaws { wantCode1 := fmt.Sprintf("%04X", i+1) var c1 *string - if err := s.db.QueryRow(`SELECT code1 FROM transmissions WHERE hash = ?`, + if err := s.db.QueryRow(`SELECT code1 FROM transmissions WHERE hash = $1`, ComputeContentHash(raw)).Scan(&c1); err != nil { t.Fatalf("read good row %d: %v", i, err) } @@ -2449,7 +2402,7 @@ func TestBackfillTransportCodes_MultiBatchWithUndecodableRow(t *testing.T) { } var badC1 *string - if err := s.db.QueryRow(`SELECT code1 FROM transmissions WHERE hash = ?`, + if err := s.db.QueryRow(`SELECT code1 FROM transmissions WHERE hash = $1`, ComputeContentHash(badRaw)).Scan(&badC1); err != nil { t.Fatalf("read bad row: %v", err) } @@ -2470,7 +2423,7 @@ func TestBackfillTransportCodes_ErrorWithholdsGuardRow(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "test.db") - store1, err := OpenStore(dbPath) + store1, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore: %v", err) } @@ -2478,7 +2431,7 @@ func TestBackfillTransportCodes_ErrorWithholdsGuardRow(t *testing.T) { raw := "14" + "AABB" + "CCDD" + "00" + strings.Repeat("11", 32) if _, err := store1.db.Exec( `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type) - VALUES (?,?,?,?,?)`, + VALUES ($1,$2,$3,$4,$5)`, raw, ComputeContentHash(raw), "2026-08-01T00:00:00Z", 0, 5); err != nil { t.Fatalf("seed: %v", err) } @@ -2490,7 +2443,7 @@ func TestBackfillTransportCodes_ErrorWithholdsGuardRow(t *testing.T) { t.Fatal("backfillTransportCodes on a closed store returned nil error, want non-nil") } - store2, err := OpenStore(dbPath) + store2, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("reopen: %v", err) } @@ -2508,7 +2461,7 @@ func TestBackfillTransportCodes_ErrorWithholdsGuardRow(t *testing.T) { store2.backfillWg.Wait() var c1 *string - if err := store2.db.QueryRow(`SELECT code1 FROM transmissions WHERE hash = ?`, + if err := store2.db.QueryRow(`SELECT code1 FROM transmissions WHERE hash = $1`, ComputeContentHash(raw)).Scan(&c1); err != nil { t.Fatalf("read: %v", err) } @@ -2526,7 +2479,7 @@ func TestBackfillTransportCodes_ErrorWithholdsGuardRow(t *testing.T) { func TestUpdateNodeDefaultScope(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "test.db") - store, err := OpenStore(dbPath) + store, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore: %v", err) } @@ -2592,7 +2545,7 @@ func TestUpdateNodeDefaultScope(t *testing.T) { func TestBackfillPathJsonFromRawHex(t *testing.T) { dbPath := tempDBPath(t) - s, err := OpenStore(dbPath) + s, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatal(err) } @@ -2601,26 +2554,26 @@ func TestBackfillPathJsonFromRawHex(t *testing.T) { // raw_hex: header 0x05 (route FLOOD, payload 0x01), path byte 0x42 (hash_size=2, count=2), // hops: AABB, CCDD, then some payload bytes rawHex := "0542AABBCCDD0000000000000000000000000000" - s.db.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, payload_type) VALUES (?, 'h1', '2025-01-01T00:00:00Z', 1)`, rawHex) + s.db.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, payload_type) VALUES ($1, 'h1', '2025-01-01T00:00:00Z', 1)`, rawHex) // Insert observation with raw_hex but empty path_json - s.db.Exec(`INSERT INTO observations (transmission_id, timestamp, raw_hex, path_json) VALUES (1, 1000, ?, '[]')`, rawHex) + s.db.Exec(`INSERT INTO observations (transmission_id, timestamp, raw_hex, path_json) VALUES (1, 1000, $1, '[]')`, rawHex) // Insert observation with raw_hex and NULL path_json - s.db.Exec(`INSERT INTO observations (transmission_id, timestamp, raw_hex, path_json) VALUES (1, 1001, ?, NULL)`, rawHex) + s.db.Exec(`INSERT INTO observations (transmission_id, timestamp, raw_hex, path_json) VALUES (1, 1001, $1, NULL)`, rawHex) // Insert observation with existing path_json (should NOT be overwritten) - s.db.Exec(`INSERT INTO observations (transmission_id, timestamp, raw_hex, path_json) VALUES (1, 1002, ?, '["XX","YY"]')`, rawHex) + s.db.Exec(`INSERT INTO observations (transmission_id, timestamp, raw_hex, path_json) VALUES (1, 1002, $1, '["XX","YY"]')`, rawHex) // Insert a TRACE transmission (payload_type = 0x09) — should be skipped traceRaw := "2604302D0D2359FEE7B100000000006733D63367" - s.db.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, payload_type) VALUES (?, 'h2', '2025-01-01T00:00:00Z', 9)`, traceRaw) - s.db.Exec(`INSERT INTO observations (transmission_id, timestamp, raw_hex, path_json) VALUES (2, 1003, ?, '[]')`, traceRaw) + s.db.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, payload_type) VALUES ($1, 'h2', '2025-01-01T00:00:00Z', 9)`, traceRaw) + s.db.Exec(`INSERT INTO observations (transmission_id, timestamp, raw_hex, path_json) VALUES (2, 1003, $1, '[]')`, traceRaw) // Remove the migration marker so it runs again on reopen s.db.Exec(`DELETE FROM _migrations WHERE name = 'backfill_path_json_from_raw_hex_v1'`) s.Close() // Reopen — backfill is now async, must trigger explicitly - s2, err := OpenStore(dbPath) + s2, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatal(err) } @@ -2672,107 +2625,6 @@ func TestBackfillPathJsonFromRawHex(t *testing.T) { } } -func TestCleanupLegacyNullHashTimestamp(t *testing.T) { - path := tempDBPath(t) - - // Create a bare-bones DB with legacy bad data - db, err := sql.Open("sqlite", path+"?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)") - if err != nil { - t.Fatal(err) - } - db.Exec(`CREATE TABLE IF NOT EXISTS transmissions ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - raw_hex TEXT NOT NULL, - hash TEXT NOT NULL, - first_seen TEXT NOT NULL, - route_type INTEGER, - payload_type INTEGER, - payload_version INTEGER, - decoded_json TEXT, - created_at TEXT DEFAULT (datetime('now')), - channel_hash TEXT DEFAULT NULL - )`) - db.Exec(`CREATE TABLE IF NOT EXISTS observations ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - transmission_id INTEGER NOT NULL REFERENCES transmissions(id), - observer_idx INTEGER, - direction TEXT, - snr REAL, - rssi REAL, - score INTEGER, - path_json TEXT, - timestamp INTEGER NOT NULL - )`) - db.Exec(`CREATE TABLE IF NOT EXISTS _migrations (name TEXT PRIMARY KEY)`) - db.Exec(`CREATE TABLE IF NOT EXISTS nodes (public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, advert_count INTEGER DEFAULT 0, battery_mv INTEGER, temperature_c REAL)`) - db.Exec(`CREATE TABLE IF NOT EXISTS observers (id TEXT PRIMARY KEY, name TEXT, iata TEXT, last_seen TEXT, first_seen TEXT, packet_count INTEGER DEFAULT 0, model TEXT, firmware TEXT, client_version TEXT, radio TEXT, battery_mv INTEGER, uptime_secs INTEGER, noise_floor REAL, inactive INTEGER DEFAULT 0, last_packet_at TEXT DEFAULT NULL)`) - - // Insert good transmission - db.Exec(`INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (1, 'aabb', 'abc123', '2024-01-01T00:00:00Z')`) - db.Exec(`INSERT INTO observations (transmission_id, observer_idx, timestamp) VALUES (1, 1, 1704067200)`) - - // Insert bad: empty hash - db.Exec(`INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (2, 'ccdd', '', '2024-01-01T00:00:00Z')`) - db.Exec(`INSERT INTO observations (transmission_id, observer_idx, timestamp) VALUES (2, 1, 1704067200)`) - - // Insert bad: empty first_seen - db.Exec(`INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (3, 'eeff', 'def456', '')`) - db.Exec(`INSERT INTO observations (transmission_id, observer_idx, timestamp) VALUES (3, 2, 1704067200)`) - - db.Close() - - // Now open via OpenStore which should run the migration - s, err := OpenStore(path) - if err != nil { - t.Fatal(err) - } - defer s.Close() - - // Good transmission should remain - var count int - s.db.QueryRow("SELECT COUNT(*) FROM transmissions WHERE id = 1").Scan(&count) - if count != 1 { - t.Error("good transmission should not be deleted") - } - - // Bad transmissions should be gone - s.db.QueryRow("SELECT COUNT(*) FROM transmissions WHERE id = 2").Scan(&count) - if count != 0 { - t.Errorf("transmission with empty hash should be deleted, got count=%d", count) - } - s.db.QueryRow("SELECT COUNT(*) FROM transmissions WHERE id = 3").Scan(&count) - if count != 0 { - t.Errorf("transmission with empty first_seen should be deleted, got count=%d", count) - } - - // Observations for bad transmissions should be gone - s.db.QueryRow("SELECT COUNT(*) FROM observations WHERE transmission_id IN (2, 3)").Scan(&count) - if count != 0 { - t.Errorf("observations for bad transmissions should be deleted, got count=%d", count) - } - - // Observation for good transmission should remain - s.db.QueryRow("SELECT COUNT(*) FROM observations WHERE transmission_id = 1").Scan(&count) - if count != 1 { - t.Error("observation for good transmission should remain") - } - - // Migration marker should exist - var migCount int - s.db.QueryRow("SELECT COUNT(*) FROM _migrations WHERE name = 'cleanup_legacy_null_hash_ts'").Scan(&migCount) - if migCount != 1 { - t.Error("migration marker cleanup_legacy_null_hash_ts should be recorded") - } - - // Idempotent: opening again should not error - s.Close() - s2, err := OpenStore(path) - if err != nil { - t.Fatal("second open should not fail:", err) - } - s2.Close() -} - func TestBuildPacketDataRegionFromPayload(t *testing.T) { msg := &MQTTPacketMessage{Raw: "0102030405060708", Region: "PDX"} decoded := &DecodedPacket{ @@ -2806,88 +2658,10 @@ func TestBackfillPathJSONAsync(t *testing.T) { dbPath := filepath.Join(dir, "async_test.db") // Bootstrap schema manually so we can insert test data BEFORE OpenStore - db, err := sql.Open("sqlite", dbPath+"?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)") - if err != nil { - t.Fatal(err) - } - // Create tables manually (minimal schema for this test) - _, err = db.Exec(` - CREATE TABLE _migrations (name TEXT PRIMARY KEY); - CREATE TABLE transmissions ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - raw_hex TEXT NOT NULL, - hash TEXT NOT NULL UNIQUE, - first_seen TEXT NOT NULL, - route_type INTEGER, - payload_type INTEGER, - payload_version INTEGER, - decoded_json TEXT, - created_at TEXT DEFAULT (datetime('now')), - channel_hash TEXT - ); - CREATE TABLE observers ( - id TEXT PRIMARY KEY, - name TEXT, - iata TEXT, - last_seen TEXT, - first_seen TEXT, - packet_count INTEGER DEFAULT 0, - model TEXT, - firmware TEXT, - client_version TEXT, - radio TEXT, - battery_mv INTEGER, - uptime_secs INTEGER, - noise_floor REAL, - inactive INTEGER DEFAULT 0, - last_packet_at TEXT - ); - CREATE TABLE nodes ( - public_key TEXT PRIMARY KEY, - name TEXT, role TEXT, lat REAL, lon REAL, - last_seen TEXT, first_seen TEXT, advert_count INTEGER DEFAULT 0, - battery_mv INTEGER, temperature_c REAL - ); - CREATE TABLE inactive_nodes ( - public_key TEXT PRIMARY KEY, - name TEXT, role TEXT, lat REAL, lon REAL, - last_seen TEXT, first_seen TEXT, advert_count INTEGER DEFAULT 0, - battery_mv INTEGER, temperature_c REAL - ); - CREATE TABLE observations ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - transmission_id INTEGER NOT NULL REFERENCES transmissions(id), - observer_idx INTEGER, - direction TEXT, - snr REAL, rssi REAL, score INTEGER, - path_json TEXT, - timestamp INTEGER NOT NULL, - raw_hex TEXT - ); - CREATE UNIQUE INDEX idx_observations_dedup ON observations(transmission_id, observer_idx, COALESCE(path_json, '')); - CREATE INDEX idx_observations_transmission_id ON observations(transmission_id); - CREATE INDEX idx_observations_observer_idx ON observations(observer_idx); - CREATE INDEX idx_observations_timestamp ON observations(timestamp); - CREATE TABLE observer_metrics ( - observer_id TEXT NOT NULL, - timestamp TEXT NOT NULL, - noise_floor REAL, tx_air_secs INTEGER, rx_air_secs INTEGER, - recv_errors INTEGER, battery_mv INTEGER, - packets_sent INTEGER, packets_recv INTEGER, - PRIMARY KEY (observer_id, timestamp) - ); - CREATE TABLE dropped_packets ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - hash TEXT, raw_hex TEXT, reason TEXT NOT NULL, - observer_id TEXT, observer_name TEXT, - node_pubkey TEXT, node_name TEXT, - dropped_at DATETIME DEFAULT CURRENT_TIMESTAMP - ); - `) + db, err := pgutil.Open(testPostgresURL(t, dbPath), false) if err != nil { - t.Fatal("bootstrap schema:", err) + t.Fatal(err) } - // Mark all migrations as done EXCEPT the path_json backfill for _, m := range []string{ "advert_count_unique_v1", "noise_floor_real_v1", "node_telemetry_v1", @@ -2896,19 +2670,19 @@ func TestBackfillPathJSONAsync(t *testing.T) { "dropped_packets_v1", "observations_raw_hex_v1", "observers_last_packet_at_v1", "cleanup_legacy_null_hash_ts", } { - db.Exec(`INSERT INTO _migrations (name) VALUES (?)`, m) + db.Exec(`INSERT INTO _migrations (name) VALUES ($1)`, m) } // Insert a transmission + observations with NULL path_json and valid raw_hex // raw_hex "0102AABBCCDD0000" has 2-hop path decodable by packetpath rawHex := "41020304AABBCCDD05060708" - _, err = db.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, payload_type) VALUES (?, 'hash1', '2025-01-01T00:00:00Z', 4)`, rawHex) + _, err = db.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, payload_type) VALUES ($1, 'hash1', '2025-01-01T00:00:00Z', 4)`, rawHex) if err != nil { t.Fatal("insert tx:", err) } // Insert 100 observations needing backfill for i := 0; i < 100; i++ { - _, err = db.Exec(`INSERT INTO observations (transmission_id, observer_idx, timestamp, raw_hex, path_json) VALUES (1, ?, ?, ?, NULL)`, + _, err = db.Exec(`INSERT INTO observations (transmission_id, observer_idx, timestamp, raw_hex, path_json) VALUES (1, $1, $2, $3, NULL)`, i+1, 1700000000+i, rawHex) if err != nil { // dedup index might fire — use unique observer_idx @@ -2919,7 +2693,7 @@ func TestBackfillPathJSONAsync(t *testing.T) { // Now open store via OpenStore — this must return QUICKLY (non-blocking) start := time.Now() - store, err := OpenStoreWithInterval(dbPath, 300) + store, err := openPostgresTestStoreInterval(t, dbPath, 300) elapsed := time.Since(start) if err != nil { t.Fatal("OpenStore:", err) @@ -2967,7 +2741,7 @@ func TestBackfillPathJSONAsync(t *testing.T) { func TestBackfillPathJSONAsyncMethodExists(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "method_test.db") - store, err := OpenStoreWithInterval(dbPath, 300) + store, err := openPostgresTestStoreInterval(t, dbPath, 300) if err != nil { t.Fatal(err) } @@ -2995,73 +2769,10 @@ func TestBackfillPathJSONAsync_BracketRowsTerminate(t *testing.T) { // Bootstrap a minimal schema directly so we can seed pre-existing '[]' rows // before OpenStore runs. - db, err := sql.Open("sqlite", dbPath+"?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)") - if err != nil { - t.Fatal(err) - } - _, err = db.Exec(` - CREATE TABLE _migrations (name TEXT PRIMARY KEY); - CREATE TABLE transmissions ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - raw_hex TEXT NOT NULL, - hash TEXT NOT NULL UNIQUE, - first_seen TEXT NOT NULL, - route_type INTEGER, - payload_type INTEGER, - payload_version INTEGER, - decoded_json TEXT, - created_at TEXT DEFAULT (datetime('now')), - channel_hash TEXT - ); - CREATE TABLE observers ( - id TEXT PRIMARY KEY, name TEXT, iata TEXT, - last_seen TEXT, first_seen TEXT, packet_count INTEGER DEFAULT 0, - model TEXT, firmware TEXT, client_version TEXT, radio TEXT, - battery_mv INTEGER, uptime_secs INTEGER, noise_floor REAL, - inactive INTEGER DEFAULT 0, last_packet_at TEXT - ); - CREATE TABLE nodes ( - public_key TEXT PRIMARY KEY, name TEXT, role TEXT, - lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, - advert_count INTEGER DEFAULT 0, battery_mv INTEGER, temperature_c REAL - ); - CREATE TABLE inactive_nodes ( - public_key TEXT PRIMARY KEY, name TEXT, role TEXT, - lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, - advert_count INTEGER DEFAULT 0, battery_mv INTEGER, temperature_c REAL - ); - CREATE TABLE observations ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - transmission_id INTEGER NOT NULL REFERENCES transmissions(id), - observer_idx INTEGER, direction TEXT, - snr REAL, rssi REAL, score INTEGER, - path_json TEXT, - timestamp INTEGER NOT NULL, - raw_hex TEXT - ); - CREATE UNIQUE INDEX idx_observations_dedup ON observations(transmission_id, observer_idx, COALESCE(path_json, '')); - CREATE INDEX idx_observations_transmission_id ON observations(transmission_id); - CREATE INDEX idx_observations_observer_idx ON observations(observer_idx); - CREATE INDEX idx_observations_timestamp ON observations(timestamp); - CREATE TABLE observer_metrics ( - observer_id TEXT NOT NULL, timestamp TEXT NOT NULL, - noise_floor REAL, tx_air_secs INTEGER, rx_air_secs INTEGER, - recv_errors INTEGER, battery_mv INTEGER, - packets_sent INTEGER, packets_recv INTEGER, - PRIMARY KEY (observer_id, timestamp) - ); - CREATE TABLE dropped_packets ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - hash TEXT, raw_hex TEXT, reason TEXT NOT NULL, - observer_id TEXT, observer_name TEXT, - node_pubkey TEXT, node_name TEXT, - dropped_at DATETIME DEFAULT CURRENT_TIMESTAMP - ); - `) + db, err := pgutil.Open(testPostgresURL(t, dbPath), false) if err != nil { - t.Fatal("bootstrap schema:", err) + t.Fatal(err) } - // Mark all migrations done EXCEPT backfill_path_json_from_raw_hex_v1. for _, m := range []string{ "advert_count_unique_v1", "noise_floor_real_v1", "node_telemetry_v1", @@ -3070,20 +2781,20 @@ func TestBackfillPathJSONAsync_BracketRowsTerminate(t *testing.T) { "dropped_packets_v1", "observations_raw_hex_v1", "observers_last_packet_at_v1", "cleanup_legacy_null_hash_ts", } { - db.Exec(`INSERT INTO _migrations (name) VALUES (?)`, m) + db.Exec(`INSERT INTO _migrations (name) VALUES ($1)`, m) } // raw_hex producing ZERO hops via DecodePathFromRawHex: // DIRECT route (type=2), payload_type=2, version=0 → header 0x0A; path byte 0x00. // (See internal/packetpath/path_test.go: TestDecodePathFromRawHex_ZeroHops.) rawHex := "0A00DEADBEEF" - _, err = db.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, payload_type) VALUES (?, 'h_brackets', '2025-01-01T00:00:00Z', 2)`, rawHex) + _, err = db.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, payload_type) VALUES ($1, 'h_brackets', '2025-01-01T00:00:00Z', 2)`, rawHex) if err != nil { t.Fatal("insert tx:", err) } const seedCount = 100 for i := 0; i < seedCount; i++ { - _, err = db.Exec(`INSERT INTO observations (transmission_id, observer_idx, timestamp, raw_hex, path_json) VALUES (1, ?, ?, ?, '[]')`, + _, err = db.Exec(`INSERT INTO observations (transmission_id, observer_idx, timestamp, raw_hex, path_json) VALUES (1, $1, $2, $3, '[]')`, i+1, 1700000000+i, rawHex) if err != nil { t.Fatalf("insert obs %d: %v", i, err) @@ -3091,7 +2802,7 @@ func TestBackfillPathJSONAsync_BracketRowsTerminate(t *testing.T) { } db.Close() - store, err := OpenStoreWithInterval(dbPath, 300) + store, err := openPostgresTestStoreInterval(t, dbPath, 300) if err != nil { t.Fatal("OpenStore:", err) } @@ -3133,14 +2844,14 @@ func TestSchemaMultibyteSupColumns(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "test.db") - store, err := OpenStore(dbPath) + store, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore: %v", err) } defer store.Close() for _, table := range []string{"nodes", "inactive_nodes"} { - rows, err := store.db.Query("PRAGMA table_info(" + table + ")") + rows, err := store.db.Query(testNativeSQL(`SELECT cid,name,type,"notnull",dflt_value,pk FROM pragma_table_info($1) ORDER BY cid`, `SELECT ordinal_position::int,column_name,data_type,CASE is_nullable WHEN 'NO' THEN 1 ELSE 0 END,column_default,0 FROM information_schema.columns WHERE table_schema=current_schema() AND table_name=$1`), table) if err != nil { t.Fatalf("PRAGMA table_info(%s): %v", table, err) } @@ -3173,7 +2884,7 @@ func TestSchemaMultibyteSupColumns(t *testing.T) { // in the legacy _migrations marker table — so we just re-assert the // columns exist and the second OpenStore is a no-op. store.Close() - store2, err := OpenStore(dbPath) + store2, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore (second open): %v", err) } @@ -3189,7 +2900,7 @@ func TestSchemaMultibyteSupColumns(t *testing.T) { func TestUpdateNodeDefaultScope_EmptyScopeIsNoop(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "test.db") - store, err := OpenStore(dbPath) + store, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore: %v", err) } @@ -3243,7 +2954,7 @@ func TestInsertClientRxObservation(t *testing.T) { // a swap is detectable. var gotLat, gotLon, gotSNR, gotPosAccM float64 var gotRSSI int - if err := s.db.QueryRow(`SELECT lat, lon, snr, rssi, pos_acc_m FROM client_rx_observations WHERE pkt_hash = ?`, o.PktHash). + if err := s.db.QueryRow(`SELECT lat, lon, snr, rssi, pos_acc_m FROM client_rx_observations WHERE pkt_hash = $1`, o.PktHash). Scan(&gotLat, &gotLon, &gotSNR, &gotRSSI, &gotPosAccM); err != nil { t.Fatalf("read back geometry: %v", err) } @@ -3273,7 +2984,7 @@ func TestInsertClientRxObservation(t *testing.T) { t.Fatalf("second copy: ins=%v err=%v", ins, err) } var n int - s.db.QueryRow(`SELECT COUNT(*) FROM client_rx_observations WHERE pkt_hash = ?`, o.PktHash).Scan(&n) + s.db.QueryRow(`SELECT COUNT(*) FROM client_rx_observations WHERE pkt_hash = $1`, o.PktHash).Scan(&n) if n != 2 { t.Errorf("rows for pkt_hash = %d, want 2", n) } @@ -3331,7 +3042,7 @@ func TestInsertClientRfSample(t *testing.T) { SELECT lat, lon, pos_acc_m, stationary, uptime_secs, battery_mv, queue_len, errors, noise_floor, last_rssi, last_snr, tx_air_secs, rx_air_secs, recv, sent, flood_rx, direct_rx, flood_tx, direct_tx, recv_errors - FROM client_rf_samples WHERE sampled_at = ?`, o.SampledAt).Scan( + FROM client_rf_samples WHERE sampled_at = $1`, o.SampledAt).Scan( &gotLat, &gotLon, &gotPosAccM, &gotStationary, &gotUptimeSecs, &gotBatteryMV, &gotQueueLen, &gotErrors, &gotNoiseFloor, &gotLastRSSI, &gotLastSNR, &gotTxAirSecs, &gotRxAirSecs, &gotRecv, &gotSent, &gotFloodRx, &gotDirectRx, &gotFloodTx, &gotDirectTx, &gotRecvErrors); err != nil { @@ -3406,7 +3117,7 @@ func TestInsertClientRfSample(t *testing.T) { t.Fatalf("insert second sample: ins=%v err=%v", ins, err) } var gotRecvErrors2 *int64 - if err := s.db.QueryRow(`SELECT recv_errors FROM client_rf_samples WHERE sampled_at = ?`, o2.SampledAt). + if err := s.db.QueryRow(`SELECT recv_errors FROM client_rf_samples WHERE sampled_at = $1`, o2.SampledAt). Scan(&gotRecvErrors2); err != nil { t.Fatalf("read second sample: %v", err) } @@ -3423,16 +3134,20 @@ func TestInsertClientRfSample(t *testing.T) { // this is a regression guard. func TestPruneClientRfSamplesUsesIndex(t *testing.T) { s := newTestStore(t) - rows, err := s.db.Query(`EXPLAIN QUERY PLAN DELETE FROM client_rf_samples WHERE sampled_at < ?`, "2026-01-01T00:00:00Z") + if testBackendValue() == "postgres" { + if _, err := s.db.Exec(`SET enable_seqscan=off`); err != nil { + t.Fatal(err) + } + } + rows, err := s.db.Query(testNativeSQL(`EXPLAIN QUERY PLAN DELETE FROM client_rf_samples WHERE sampled_at < $1`, `EXPLAIN (COSTS OFF) DELETE FROM client_rf_samples WHERE sampled_at < $1`), "2026-01-01T00:00:00Z") if err != nil { t.Fatal(err) } defer rows.Close() plan := "" for rows.Next() { - var id, parent, notused int var detail string - if err := rows.Scan(&id, &parent, ¬used, &detail); err != nil { + if err := scanTestPlan(rows, &detail); err != nil { t.Fatal(err) } plan += detail + "\n" @@ -3492,7 +3207,7 @@ func TestPruneOldClientRfSamples(t *testing.T) { t.Fatalf("expected 2 samples remaining (recent + boundary), got %d", remaining) } var boundarySurvived int - s.db.QueryRow(`SELECT COUNT(*) FROM client_rf_samples WHERE sampled_at = ?`, boundary).Scan(&boundarySurvived) + s.db.QueryRow(`SELECT COUNT(*) FROM client_rf_samples WHERE sampled_at = $1`, boundary).Scan(&boundarySurvived) if boundarySurvived != 1 { t.Fatalf("boundary row (500ms after cutoff instant) must survive; an RFC3339 (no-ms) cutoff would wrongly delete it because '.' < 'Z' lexicographically — got %d", boundarySurvived) } diff --git a/cmd/ingestor/db_writer_perf_test.go b/cmd/ingestor/db_writer_perf_test.go index 740663f92..4c1312a18 100644 --- a/cmd/ingestor/db_writer_perf_test.go +++ b/cmd/ingestor/db_writer_perf_test.go @@ -30,7 +30,7 @@ func TestWriterStarvationVisibleInPerf(t *testing.T) { // follower samples can't move p99 above 50s. ResetWriterStatsForTest() - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -46,7 +46,7 @@ func TestWriterStarvationVisibleInPerf(t *testing.T) { go func() { defer close(blockerDone) err := s.WriterTx("neighbor_builder", func(tx *sql.Tx) error { - if _, err := tx.Exec(`UPDATE nodes SET name = name WHERE 0`); err != nil { + if _, err := tx.Exec(`UPDATE nodes SET name = name WHERE false`); err != nil { return err } close(blockStarted) @@ -59,7 +59,11 @@ func TestWriterStarvationVisibleInPerf(t *testing.T) { }() // Wait for the blocker to be inside its transaction. - <-blockStarted + select { + case <-blockStarted: + case <-blockerDone: + t.Fatal("blocker failed before acquiring its transaction") + } // Small safety margin so the blocker is firmly holding the conn. time.Sleep(100 * time.Millisecond) @@ -74,7 +78,7 @@ func TestWriterStarvationVisibleInPerf(t *testing.T) { defer wg.Done() _, err := s.WriterExec( "mqtt_handler", - `INSERT OR IGNORE INTO _migrations (name) VALUES (?)`, + `INSERT INTO _migrations (name) VALUES ($1) ON CONFLICT DO NOTHING`, fmt.Sprintf("writer_starvation_test_%d", i), ) if err != nil { diff --git a/cmd/ingestor/decoder.go b/cmd/ingestor/decoder.go index 41aa3afc4..ae0c2f9a4 100644 --- a/cmd/ingestor/decoder.go +++ b/cmd/ingestor/decoder.go @@ -1068,55 +1068,7 @@ func DecodePacket(hexString string, channelKeys map[string]string, validateSigna // route-independent identifier for the same logical packet. For TRACE packets, // path_len is included in the hash to match firmware behavior. func ComputeContentHash(rawHex string) string { - buf, err := hex.DecodeString(rawHex) - if err != nil || len(buf) < 2 { - if len(rawHex) >= 16 { - return rawHex[:16] - } - return rawHex - } - - headerByte := buf[0] - offset := 1 - if isTransportRoute(int(headerByte & 0x03)) { - offset += 4 - } - if offset >= len(buf) { - if len(rawHex) >= 16 { - return rawHex[:16] - } - return rawHex - } - pathByte := buf[offset] - offset++ - hashSize := int((pathByte>>6)&0x3) + 1 - hashCount := int(pathByte & 0x3F) - pathBytes := hashSize * hashCount - - payloadStart := offset + pathBytes - if payloadStart > len(buf) { - if len(rawHex) >= 16 { - return rawHex[:16] - } - return rawHex - } - - payload := buf[payloadStart:] - - // Hash payload-type byte only (bits 2-5 of header), not the full header. - // Firmware: SHA256(payload_type + [path_len for TRACE] + payload) - // Using the full header caused different hashes for the same logical packet - // when route type or version bits differed. See issue #786. - payloadType := (headerByte >> 2) & 0x0F - toHash := []byte{payloadType} - if int(payloadType) == PayloadTRACE { - // Firmware uses uint16_t path_len (2 bytes, little-endian) - toHash = append(toHash, pathByte, 0x00) - } - toHash = append(toHash, payload...) - - h := sha256.Sum256(toHash) - return hex.EncodeToString(h[:])[:16] + return packetpath.ContentHash(rawHex) } // PayloadJSON serializes the payload to JSON for DB storage. diff --git a/cmd/ingestor/from_pubkey_test.go b/cmd/ingestor/from_pubkey_test.go index 5fb614961..c0a47b862 100644 --- a/cmd/ingestor/from_pubkey_test.go +++ b/cmd/ingestor/from_pubkey_test.go @@ -10,7 +10,7 @@ import ( ) func TestInsertTransmission_FromPubkeyPopulatedForAdvert(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -34,14 +34,14 @@ func TestInsertTransmission_FromPubkeyPopulatedForAdvert(t *testing.T) { } var got sql.NullString - s.db.QueryRow("SELECT from_pubkey FROM transmissions WHERE hash = ?", data.Hash).Scan(&got) + s.db.QueryRow("SELECT from_pubkey FROM transmissions WHERE hash = $1", data.Hash).Scan(&got) if !got.Valid || got.String != pk { t.Fatalf("from_pubkey = %v (valid=%v), want %q", got.String, got.Valid, pk) } } func TestInsertTransmission_FromPubkeyNullForNonAdvert(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -64,7 +64,7 @@ func TestInsertTransmission_FromPubkeyNullForNonAdvert(t *testing.T) { } var got sql.NullString - s.db.QueryRow("SELECT from_pubkey FROM transmissions WHERE hash = ?", data.Hash).Scan(&got) + s.db.QueryRow("SELECT from_pubkey FROM transmissions WHERE hash = $1", data.Hash).Scan(&got) if got.Valid { t.Fatalf("from_pubkey for non-ADVERT must be NULL, got %q", got.String) } diff --git a/cmd/ingestor/go.mod b/cmd/ingestor/go.mod index 729c8e9d7..7f26c9703 100644 --- a/cmd/ingestor/go.mod +++ b/cmd/ingestor/go.mod @@ -1,12 +1,11 @@ module github.com/corescope/ingestor -go 1.22 +go 1.25.0 require ( github.com/eclipse/paho.mqtt.golang v1.5.0 github.com/meshcore-analyzer/geofilter v0.0.0 github.com/meshcore-analyzer/sigvalidate v0.0.0 - modernc.org/sqlite v1.34.5 ) replace github.com/meshcore-analyzer/geofilter => ../../internal/geofilter @@ -30,18 +29,15 @@ require github.com/meshcore-analyzer/dbschema v0.0.0 replace github.com/meshcore-analyzer/dbschema => ../../internal/dbschema require ( - github.com/dustin/go-humanize v1.0.1 // indirect - github.com/google/uuid v1.6.0 // indirect github.com/gorilla/websocket v1.5.3 // indirect - github.com/mattn/go-isatty v0.0.20 // indirect - github.com/ncruces/go-strftime v0.1.9 // indirect - github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/pgx/v5 v5.7.6 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + golang.org/x/crypto v0.37.0 // indirect golang.org/x/net v0.27.0 // indirect - golang.org/x/sync v0.7.0 // indirect - golang.org/x/sys v0.22.0 // indirect - modernc.org/libc v1.55.3 // indirect - modernc.org/mathutil v1.6.0 // indirect - modernc.org/memory v1.8.0 // indirect + golang.org/x/sync v0.13.0 // indirect + golang.org/x/text v0.24.0 // indirect ) require github.com/meshcore-analyzer/prunequeue v0.0.0 @@ -51,3 +47,13 @@ replace github.com/meshcore-analyzer/prunequeue => ../../internal/prunequeue require github.com/meshcore-analyzer/mbcapqueue v0.0.0 replace github.com/meshcore-analyzer/mbcapqueue => ../../internal/mbcapqueue + +require ( + github.com/mattn/go-sqlite3 v1.14.52 + github.com/meshcore-analyzer/channel v0.0.0 + github.com/meshcore-analyzer/pgutil v0.0.0 +) + +replace github.com/meshcore-analyzer/channel => ../../internal/channel + +replace github.com/meshcore-analyzer/pgutil => ../../internal/pgutil diff --git a/cmd/ingestor/go.sum b/cmd/ingestor/go.sum index cc13ef47f..b17071293 100644 --- a/cmd/ingestor/go.sum +++ b/cmd/ingestor/go.sum @@ -1,51 +1,36 @@ -github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/eclipse/paho.mqtt.golang v1.5.0 h1:EH+bUVJNgttidWFkLLVKaQPGmkTUfQQqjOsyvMGvD6o= github.com/eclipse/paho.mqtt.golang v1.5.0/go.mod h1:du/2qNQVqJf/Sqs4MEL77kR8QTqANF7XU7Fk0aOTAgk= -github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo= -github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= -github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= -github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= -golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.7.6 h1:rWQc5FwZSPX58r1OQmkuaNicxdmExaEz5A2DO2hUuTk= +github.com/jackc/pgx/v5 v5.7.6/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/mattn/go-sqlite3 v1.14.52 h1:wVbm2Qnf4OXkqhBTSPuCRZDRnxfbVrrmiCEroVdog8U= +github.com/mattn/go-sqlite3 v1.14.52/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= +golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= -golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw= -golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc= -modernc.org/cc/v4 v4.21.4 h1:3Be/Rdo1fpr8GrQ7IVw9OHtplU4gWbb+wNgeoBMmGLQ= -modernc.org/cc/v4 v4.21.4/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ= -modernc.org/ccgo/v4 v4.19.2 h1:lwQZgvboKD0jBwdaeVCTouxhxAyN6iawF3STraAal8Y= -modernc.org/ccgo/v4 v4.19.2/go.mod h1:ysS3mxiMV38XGRTTcgo0DQTeTmAO4oCmJl1nX9VFI3s= -modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE= -modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ= -modernc.org/gc/v2 v2.4.1 h1:9cNzOqPyMJBvrUipmynX0ZohMhcxPtMccYgGOJdOiBw= -modernc.org/gc/v2 v2.4.1/go.mod h1:wzN5dK1AzVGoH6XOzc3YZ+ey/jPgYHLuVckd62P0GYU= -modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U= -modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w= -modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= -modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= -modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E= -modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU= -modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= -modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= -modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc= -modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss= -modernc.org/sqlite v1.34.5 h1:Bb6SR13/fjp15jt70CL4f18JIN7p7dnMExd+UFnF15g= -modernc.org/sqlite v1.34.5/go.mod h1:YLuNmX9NKs8wRNK2ko1LW1NGYcc9FkBO69JOt1AR9JE= -modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= -modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= -modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= -modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= +golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= +golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/cmd/ingestor/iata_drop_warn_test.go b/cmd/ingestor/iata_drop_warn_test.go new file mode 100644 index 000000000..fa0b0bb5d --- /dev/null +++ b/cmd/ingestor/iata_drop_warn_test.go @@ -0,0 +1,196 @@ +package main + +import ( + "fmt" + "testing" + "time" +) + +func TestIngestorIATAWhitelistGate(t *testing.T) { + // Empty whitelist: the facility stays inert, everything passes. + open := &Config{} + for _, code := range []string{"SJC", "PHL", "", "zzz"} { + if !open.IsObserverIATAAllowed(code) { + t.Errorf("empty whitelist should allow %q", code) + } + } + + cfg := &Config{ObserverIATAWhitelist: []string{"SJC", "oak", " MRY "}} + tests := []struct { + iata string + want bool + }{ + {"SJC", true}, + {"sjc", true}, + {"OAK", true}, + {"MRY", true}, + {" mry ", true}, + {"PHL", false}, + {"MCO", false}, + {"", false}, + } + for _, tt := range tests { + if got := cfg.IsObserverIATAAllowed(tt.iata); got != tt.want { + t.Errorf("IsObserverIATAAllowed(%q) = %v, want %v", tt.iata, got, tt.want) + } + } +} + +func TestIngestorIATAWarnInterval(t *testing.T) { + if got := (&Config{}).IATAWarnInterval(); got != 6*time.Hour { + t.Errorf("default interval = %v, want 6h", got) + } + if got := (&Config{IATAWarnIntervalSec: 90}).IATAWarnInterval(); got != 90*time.Second { + t.Errorf("configured interval = %v, want 90s", got) + } + // Negative/zero fall back to the default rather than logging every message. + if got := (&Config{IATAWarnIntervalSec: -5}).IATAWarnInterval(); got != 6*time.Hour { + t.Errorf("negative interval = %v, want 6h", got) + } +} + +func TestIngestorShouldWarnIATADropThrottles(t *testing.T) { + cfg := &Config{ObserverIATAWhitelist: []string{"SJC"}, IATAWarnIntervalSec: 3600} + + if !cfg.ShouldWarnIATADrop("PHL") { + t.Fatal("first drop for a region should warn") + } + if cfg.ShouldWarnIATADrop("PHL") { + t.Error("second drop inside the interval should be suppressed") + } + if cfg.ShouldWarnIATADrop("phl") { + t.Error("case variant should hit the same throttle bucket") + } + // A different region is tracked independently. + if !cfg.ShouldWarnIATADrop("MCO") { + t.Error("a distinct region should warn on its first drop") + } + + // Once the interval elapses the region re-logs, so an ongoing drop stays + // visible to a scraper instead of decaying into silence. + cfg.iataWarnMu.Lock() + cfg.iataWarnLast["PHL"] = time.Now().Add(-2 * time.Hour) + cfg.iataWarnMu.Unlock() + if !cfg.ShouldWarnIATADrop("PHL") { + t.Error("region should re-log after the interval elapses") + } +} + +func TestIngestorShouldWarnIATADropEdges(t *testing.T) { + var nilCfg *Config + if nilCfg.ShouldWarnIATADrop("PHL") { + t.Error("nil config should not warn") + } + cfg := &Config{} + if cfg.ShouldWarnIATADrop(" ") { + t.Error("blank region should not warn") + } +} + +// TestShouldWarnIATADropBoundsItsMap pins the fix for the review blocker on +// #2008: the throttle key is a topic segment the publisher controls, so an +// unbounded map is a remote memory sink. Measured on the original branch, +// 200,000 distinct codes retained 200,000 entries and 15.1 MB of heap. +func TestShouldWarnIATADropBoundsItsMap(t *testing.T) { + c := &Config{} + + // Far more distinct codes than the cap, as a hostile or broken feed would. + for i := 0; i < iataWarnMaxTracked*4; i++ { + c.ShouldWarnIATADrop(fmt.Sprintf("X%05d", i)) + } + + c.iataWarnMu.Lock() + tracked := len(c.iataWarnLast) + c.iataWarnMu.Unlock() + + if tracked > iataWarnMaxTracked { + t.Errorf("tracked %d codes, cap is %d — the map grows with publisher-supplied input", + tracked, iataWarnMaxTracked) + } +} + +// TestShouldWarnIATADropKeepsWarningPastTheCap is the other half: bounding the +// map must not silence the warning, which would reintroduce the silent-drop bug +// this feature exists to fix, one level up. Past the cap the warning is +// throttled on a shared timestamp rather than dropped. +func TestShouldWarnIATADropKeepsWarningPastTheCap(t *testing.T) { + c := &Config{} + for i := 0; i < iataWarnMaxTracked; i++ { + c.ShouldWarnIATADrop(fmt.Sprintf("F%05d", i)) + } + + // A brand new code, with the map already full. + if !c.ShouldWarnIATADrop("ZZZ") { + t.Fatal("a drop past the cap must still warn once, not be swallowed") + } + // ...and the next one is throttled rather than flooding. + if c.ShouldWarnIATADrop("YYY") { + t.Error("a second overflow drop inside the interval must be throttled") + } + + // After the interval elapses, it speaks again. Rewind the shared timestamp + // rather than sleeping. + c.iataWarnMu.Lock() + c.iataWarnOverflowLast = time.Now().Add(-2 * c.IATAWarnInterval()) + c.iataWarnMu.Unlock() + if !c.ShouldWarnIATADrop("WWW") { + t.Error("once the interval has passed the overflow warning must return") + } +} + +// TestShouldWarnIATADropStillThrottlesKnownCodes guards the normal path: the +// cap must not change behaviour for a code already being tracked. +func TestShouldWarnIATADropStillThrottlesKnownCodes(t *testing.T) { + c := &Config{} + if !c.ShouldWarnIATADrop("BRU") { + t.Fatal("first sighting must warn") + } + if c.ShouldWarnIATADrop("BRU") { + t.Error("second sighting inside the interval must be throttled") + } +} + +// TestShouldWarnIATADropReclaimsExpiredSlots covers the flaw the cap had on its +// own: the map never shrank, so the first iataWarnMaxTracked codes ever seen +// owned it forever and a legitimate region arriving later was permanently +// demoted to the shared overflow throttle behind whatever transient junk filled +// it first. +// +// Sweeping expired entries at the cap is semantically free — an entry past the +// interval would be re-logged on its next sighting regardless — so the map ends +// up holding only regions that are actively arriving. +func TestShouldWarnIATADropReclaimsExpiredSlots(t *testing.T) { + c := &Config{} + + // Fill the map with junk, then age all of it past the interval. + for i := 0; i < iataWarnMaxTracked; i++ { + c.ShouldWarnIATADrop(fmt.Sprintf("J%05d", i)) + } + stale := time.Now().Add(-2 * c.IATAWarnInterval()) + c.iataWarnMu.Lock() + for k := range c.iataWarnLast { + c.iataWarnLast[k] = stale + } + c.iataWarnMu.Unlock() + + // A legitimate region shows up now. It must get its own slot, not the + // shared overflow throttle. + if !c.ShouldWarnIATADrop("BRU") { + t.Fatal("a new region must warn once") + } + c.iataWarnMu.Lock() + _, tracked := c.iataWarnLast["BRU"] + size := len(c.iataWarnLast) + c.iataWarnMu.Unlock() + + if !tracked { + t.Error("BRU must occupy its own slot once the stale entries were reclaimed") + } + if size > iataWarnMaxTracked { + t.Errorf("map grew to %d past the cap of %d", size, iataWarnMaxTracked) + } + // And its own throttle applies, rather than the shared one. + if c.ShouldWarnIATADrop("BRU") { + t.Error("a tracked region must be throttled on its own timestamp") + } +} diff --git a/cmd/ingestor/ingest_buffer.go b/cmd/ingestor/ingest_buffer.go index 641801a3d..a762e6159 100644 --- a/cmd/ingestor/ingest_buffer.go +++ b/cmd/ingestor/ingest_buffer.go @@ -10,7 +10,7 @@ import ( // IngestBuffer decouples MQTT message receipt from DB writes (#1608). // // On boot the ingestor must subscribe to MQTT immediately, but the single -// SQLite writer (#1283) can be held for minutes by a startup migration +// serialized writer (#1283) can be held for minutes by a startup migration // (e.g. a large CREATE INDEX) or prune. Without buffering, every QoS-0 packet // received in that window is lost. IngestBuffer holds received work in a // bounded FIFO and a single consumer goroutine drains it once Ready() is diff --git a/cmd/ingestor/issue1283_test.go b/cmd/ingestor/issue1283_test.go index f19936bae..96a04afac 100644 --- a/cmd/ingestor/issue1283_test.go +++ b/cmd/ingestor/issue1283_test.go @@ -1,12 +1,9 @@ package main import ( - "database/sql" "path/filepath" "testing" "time" - - _ "modernc.org/sqlite" ) // TestIngestorPruneOldPackets enforces #1283: the writer for @@ -16,7 +13,7 @@ import ( func TestIngestorPruneOldPackets(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "prune.db") - store, err := OpenStore(path) + store, err := openPostgresTestStore(t, path) if err != nil { t.Fatalf("OpenStore: %v", err) } @@ -27,7 +24,7 @@ func TestIngestorPruneOldPackets(t *testing.T) { for i, ts := range []string{old, old, new} { _, err := store.db.Exec( `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) - VALUES (?, ?, ?, 0, 1, 1, '{}')`, + VALUES ($1, $2, $3, 0, 1, 1, '{}')`, "AA", "h"+string(rune('a'+i)), ts, ) if err != nil { @@ -52,47 +49,9 @@ func TestIngestorPruneOldPackets(t *testing.T) { } } -// TestIngestorVacuumOnStartupMigratesNONEtoINCREMENTAL exercises the -// scenario that originally broke in #1283: a fresh DB with -// auto_vacuum=NONE, vacuumOnStartup=true, no contention from a server -// process. The ingestor must complete the VACUUM and flip auto_vacuum to -// INCREMENTAL. Before the fix, the migration ran inside cmd/server and -// hit SQLITE_BUSY because the ingestor (sharing the container) was -// already writing. -func TestIngestorVacuumOnStartupMigratesNONEtoINCREMENTAL(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "vac.db") - - // Create a NONE-auto_vacuum DB (simulates an older deployment). - seed, err := sql.Open("sqlite", path+"?_pragma=journal_mode(WAL)") - if err != nil { - t.Fatal(err) - } - seed.SetMaxOpenConns(1) - if _, err := seed.Exec(`CREATE TABLE dummy(id INTEGER PRIMARY KEY)`); err != nil { - t.Fatal(err) - } - var before int - seed.QueryRow("PRAGMA auto_vacuum").Scan(&before) - if before != 0 { - t.Fatalf("precondition: auto_vacuum=%d, want 0", before) - } - seed.Close() - - store, err := OpenStore(path) - if err != nil { - t.Fatalf("OpenStore: %v", err) - } - defer store.Close() - - cfg := &Config{DB: &DBConfig{VacuumOnStartup: true}} - store.CheckAutoVacuum(cfg) - - var after int - if err := store.db.QueryRow("PRAGMA auto_vacuum").Scan(&after); err != nil { - t.Fatal(err) - } - if after != 2 { - t.Fatalf("expected auto_vacuum=2 after ingestor VACUUM, got %d", after) +func TestPostgresRuntimeCannotChangeTableVacuumPolicy(t *testing.T) { + s := newTestStore(t) + if _, err := s.db.Exec(`ALTER TABLE transmissions SET (autovacuum_enabled=false)`); err == nil { + t.Fatal("runtime writer changed owner-only vacuum policy") } } diff --git a/cmd/ingestor/issue1690_last_seen_test.go b/cmd/ingestor/issue1690_last_seen_test.go index 5b31917ec..6578f1f14 100644 --- a/cmd/ingestor/issue1690_last_seen_test.go +++ b/cmd/ingestor/issue1690_last_seen_test.go @@ -13,7 +13,7 @@ import ( ) func TestIssue1690_LastSeenUpdatedOnObservation(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -42,7 +42,7 @@ func TestIssue1690_LastSeenUpdatedOnObservation(t *testing.T) { // Sanity: confirm the seed last_seen is the 7d-ago time. var seededLastSeen int64 - if err := s.db.QueryRow(`SELECT COALESCE(last_seen, 0) FROM transmissions WHERE hash = ?`, hash).Scan(&seededLastSeen); err != nil { + if err := s.db.QueryRow(`SELECT COALESCE(last_seen, 0) FROM transmissions WHERE hash = $1`, hash).Scan(&seededLastSeen); err != nil { t.Fatalf("seed select last_seen: %v (column missing? post-fix must add it)", err) } weekAgoUnix, _ := time.Parse(time.RFC3339, weekAgo) @@ -71,7 +71,7 @@ func TestIssue1690_LastSeenUpdatedOnObservation(t *testing.T) { } var ls int64 - if err := s.db.QueryRow(`SELECT last_seen FROM transmissions WHERE hash = ?`, hash).Scan(&ls); err != nil { + if err := s.db.QueryRow(`SELECT last_seen FROM transmissions WHERE hash = $1`, hash).Scan(&ls); err != nil { t.Fatalf("post-insert select last_seen: %v", err) } // The post-fix writer must bump last_seen to at least the new observation's diff --git a/cmd/ingestor/issue1865_test.go b/cmd/ingestor/issue1865_test.go new file mode 100644 index 000000000..fff3ab600 --- /dev/null +++ b/cmd/ingestor/issue1865_test.go @@ -0,0 +1,244 @@ +package main + +// Tests for #1865: ingest the observer /neighbors report as concrete evidence +// of configured region scopes. Covers handleNeighborsReport dispatch semantics +// and the UpdateNodeConfiguredScope store method (status gating, case folding, +// last-write-wins, and the "absence/timeout is not a signal" contract). + +import ( + "database/sql" + "path/filepath" + "testing" +) + +// seedNode inserts a node into both nodes and inactive_nodes (lowercase key). +func seedNode(t *testing.T, store *Store, pubkey string) { + t.Helper() + if _, err := store.db.Exec(`INSERT INTO nodes (public_key, name) VALUES ($1, $2)`, pubkey, "n_"+pubkey[:4]); err != nil { + t.Fatalf("seed node %s: %v", pubkey, err) + } + if _, err := store.db.Exec(`INSERT INTO inactive_nodes (public_key, name) VALUES ($1, $2)`, pubkey, "n_"+pubkey[:4]); err != nil { + t.Fatalf("seed inactive node %s: %v", pubkey, err) + } +} + +func configuredScope(t *testing.T, store *Store, pubkey string) (sql.NullString, sql.NullString) { + t.Helper() + var sc, at sql.NullString + if err := store.db.QueryRow( + `SELECT configured_scope, configured_scope_at FROM nodes WHERE public_key = $1`, pubkey, + ).Scan(&sc, &at); err != nil { + t.Fatalf("read configured_scope for %s: %v", pubkey, err) + } + return sc, at +} + +func openNeighborsStore(t *testing.T) *Store { + t.Helper() + store, err := openPostgresTestStore(t, filepath.Join(t.TempDir(), "test.db")) + if err != nil { + t.Fatalf("OpenStore: %v", err) + } + t.Cleanup(func() { store.Close() }) + return store +} + +func TestHandleNeighborsReportWritesSelfAndResponded(t *testing.T) { + store := openNeighborsStore(t) + + // Report keys are UPPERCASE; nodes.public_key is lowercase hex. + const originUpper = "FEEDCA4AD4E2AE615AAAB3CB73FAEC6EF0C7AF4D410F5C58A70FC0F724B7C933" + const respUpper = "B0D17C59FCF580592F8FB78B67D2F0CE9E9187EF3483A765BDFF1D7947A5109C" + const timeoutUpper = "0CE5EA7CFA3AB01D11810EF56B73DD899CD6C58644D6A6832A5C1AE89AFC5E25" + originLower := "feedca4ad4e2ae615aaab3cb73faec6ef0c7af4d410f5c58a70fc0f724b7c933" + respLower := "b0d17c59fcf580592f8fb78b67d2f0ce9e9187ef3483a765bdff1d7947a5109c" + timeoutLower := "0ce5ea7cfa3ab01d11810ef56b73dd899cd6c58644d6a6832a5c1ae89afc5e25" + + seedNode(t, store, originLower) + seedNode(t, store, respLower) + seedNode(t, store, timeoutLower) + // Pre-existing confirmed scope on the timeout node — a timeout must NOT clobber it. + if err := store.UpdateNodeConfiguredScope(timeoutLower, "eu", "2026-07-24T00:00:00Z"); err != nil { + t.Fatal(err) + } + + report := map[string]interface{}{ + "timestamp": "2026-07-25T10:46:14.000000+00:00", + "origin_id": originUpper, + "self": map[string]interface{}{"scopes": "*"}, + "neighbors": []interface{}{ + map[string]interface{}{"pubkey": timeoutUpper, "scopes": "", "status": "timeout"}, + map[string]interface{}{"pubkey": respUpper, "scopes": "de,eu", "status": "responded"}, + }, + } + handleNeighborsReport(store, "test", "obs-topic-id", report) + + // self scopes written under the lowercased origin_id. + if sc, _ := configuredScope(t, store, originLower); !sc.Valid || sc.String != "*" { + t.Errorf("self configured_scope = %v, want '*'", sc) + } + // responded neighbor written, lowercased. + if sc, _ := configuredScope(t, store, respLower); !sc.Valid || sc.String != "#de,#eu" { + t.Errorf("responded configured_scope = %v, want '#de,#eu'", sc) + } + // timeout neighbor untouched — prior confirmed value survives. + if sc, _ := configuredScope(t, store, timeoutLower); !sc.Valid || sc.String != "#eu" { + t.Errorf("timeout node configured_scope = %v, want prior '#eu' (must not be cleared)", sc) + } +} + +func TestHandleNeighborsReportUnknownNeighborIsNoop(t *testing.T) { + store := openNeighborsStore(t) + // No node seeded for this pubkey — UPDATE must match no row (no insert, no error). + report := map[string]interface{}{ + "timestamp": "2026-07-25T10:46:14Z", + "neighbors": []interface{}{ + map[string]interface{}{"pubkey": "aa" + "00000000000000000000000000000000000000000000000000000000000000"[2:], "scopes": "de", "status": "responded"}, + }, + } + handleNeighborsReport(store, "test", "obs", report) + var n int + if err := store.db.QueryRow(`SELECT COUNT(*) FROM nodes`).Scan(&n); err != nil { + t.Fatal(err) + } + if n != 0 { + t.Errorf("nodes count = %d, want 0 (report must not create nodes)", n) + } +} + +func TestHandleNeighborsReportRespondedEmptyScopeIsStored(t *testing.T) { + store := openNeighborsStore(t) + pk := "cc00000000000000000000000000000000000000000000000000000000000001" + seedNode(t, store, pk) + // A responded query with empty scopes is a valid "no scopes configured". + report := map[string]interface{}{ + "timestamp": "2026-07-25T10:46:14Z", + "neighbors": []interface{}{ + map[string]interface{}{"pubkey": pk, "scopes": "", "status": "responded"}, + }, + } + handleNeighborsReport(store, "test", "obs", report) + sc, at := configuredScope(t, store, pk) + if !sc.Valid || sc.String != "" { + t.Errorf("responded-empty configured_scope = %v, want stored empty string", sc) + } + if !at.Valid || at.String == "" { + t.Errorf("configured_scope_at = %v, want the report timestamp", at) + } +} + +func TestUpdateNodeConfiguredScopeLastWriteWins(t *testing.T) { + store := openNeighborsStore(t) + pk := "dd00000000000000000000000000000000000000000000000000000000000001" + seedNode(t, store, pk) + + if err := store.UpdateNodeConfiguredScope(pk, "eu", "2026-07-25T12:00:00Z"); err != nil { + t.Fatal(err) + } + // Older report must NOT clobber the newer confirmed value. + if err := store.UpdateNodeConfiguredScope(pk, "stale", "2026-07-24T00:00:00Z"); err != nil { + t.Fatal(err) + } + if sc, _ := configuredScope(t, store, pk); sc.String != "#eu" { + t.Errorf("configured_scope = %q, want '#eu' (older report must not overwrite)", sc.String) + } + // Newer report updates. + if err := store.UpdateNodeConfiguredScope(pk, "de", "2026-07-26T00:00:00Z"); err != nil { + t.Fatal(err) + } + if sc, _ := configuredScope(t, store, pk); sc.String != "#de" { + t.Errorf("configured_scope = %q, want '#de' (newer report should update)", sc.String) + } + // inactive_nodes mirrored. + var inactive sql.NullString + if err := store.db.QueryRow(`SELECT configured_scope FROM inactive_nodes WHERE public_key = $1`, pk).Scan(&inactive); err != nil { + t.Fatal(err) + } + if inactive.String != "#de" { + t.Errorf("inactive_nodes.configured_scope = %q, want '#de'", inactive.String) + } +} + +// TestUpdateNodeConfiguredScopeNormalizesAndOrdersByInstant proves the +// last-write-wins guard orders by chronological instant, not by raw string. +// A "+02:00" report that is lexicographically "greater" but chronologically +// EARLIER than the stored UTC value must not win, and stored timestamps are +// canonicalized to UTC "Z" form regardless of the incoming offset/precision. +func TestUpdateNodeConfiguredScopeNormalizesAndOrdersByInstant(t *testing.T) { + store := openNeighborsStore(t) + pk := "ee00000000000000000000000000000000000000000000000000000000000001" + seedNode(t, store, pk) + + // Baseline: noon UTC. + if err := store.UpdateNodeConfiguredScope(pk, "eu", "2026-07-25T12:00:00Z"); err != nil { + t.Fatal(err) + } + if _, at := configuredScope(t, store, pk); at.String != "2026-07-25T12:00:00Z" { + t.Fatalf("stored at = %q, want canonical 'Z' form", at.String) + } + + // "2026-07-25T13:30:00+02:00" == 11:30Z, chronologically EARLIER than 12:00Z, + // but lexicographically GREATER ("13:30..." > "12:00...Z"). Must be skipped. + if err := store.UpdateNodeConfiguredScope(pk, "stale", "2026-07-25T13:30:00+02:00"); err != nil { + t.Fatal(err) + } + if sc, _ := configuredScope(t, store, pk); sc.String != "#eu" { + t.Errorf("configured_scope = %q, want '#eu' (earlier +02:00 report must not win)", sc.String) + } + + // "2026-07-25T15:00:00+02:00" == 13:00Z, chronologically LATER. Must update, + // and be stored canonicalized to UTC. + if err := store.UpdateNodeConfiguredScope(pk, "de", "2026-07-25T15:00:00+02:00"); err != nil { + t.Fatal(err) + } + sc, at := configuredScope(t, store, pk) + if sc.String != "#de" { + t.Errorf("configured_scope = %q, want '#de' (later +02:00 report should win)", sc.String) + } + if at.String != "2026-07-25T13:00:00Z" { + t.Errorf("stored at = %q, want canonical '2026-07-25T13:00:00Z'", at.String) + } + + // Firmware's real format (microseconds + "+00:00") canonicalizes to "Z". + if err := store.UpdateNodeConfiguredScope(pk, "dk", "2026-07-26T09:43:48.000000+00:00"); err != nil { + t.Fatal(err) + } + if _, at := configuredScope(t, store, pk); at.String != "2026-07-26T09:43:48Z" { + t.Errorf("stored at = %q, want canonical '2026-07-26T09:43:48Z'", at.String) + } +} + +// #1865: the OTA scope query returns bare region names while every other +// stored scope in CoreScope carries the leading "#". normalizeScopeList makes +// configured_scope comparable with default_scope; these cases pin the parts +// that are easy to get wrong when someone later "simplifies" it. +func TestNormalizeScopeList(t *testing.T) { + cases := []struct{ name, in, want string }{ + {"empty stays empty (responded, no scopes)", "", ""}, + {"whitespace only is empty, not \"#\"", " ", ""}, + {"bare region gains the hash", "dk", "#dk"}, + {"already hashed is untouched", "#dk", "#dk"}, + {"wildcard is not a region name", "*", "*"}, + {"real observer report, mixed", "*,dk,eu,europe,dk-aarhus", "*,#dk,#eu,#europe,#dk-aarhus"}, + {"already-hashed report is idempotent", "*,#de,#de-sl", "*,#de,#de-sl"}, + {"mixed hashed and bare in one report", "#de,sl", "#de,#sl"}, + {"surrounding spaces are trimmed", " dk , eu ", "#dk,#eu"}, + {"empty entries are dropped, not turned into \"#\"", "dk,,eu,", "#dk,#eu"}, + {"order is preserved", "eu,dk", "#eu,#dk"}, + {"duplicates are kept, not collapsed", "dk,dk", "#dk,#dk"}, + {"case is left alone (keys are sha256 of the name)", "DK", "#DK"}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if got := normalizeScopeList(c.in); got != c.want { + t.Errorf("normalizeScopeList(%q) = %q, want %q", c.in, got, c.want) + } + }) + } + // Applying it twice must not add a second "#": the ingest path can see the + // same report again after a reconnect. + once := normalizeScopeList("*,dk,#eu") + if twice := normalizeScopeList(once); twice != once { + t.Errorf("not idempotent: %q then %q", once, twice) + } +} diff --git a/cmd/ingestor/main.go b/cmd/ingestor/main.go index 6149afebd..e163f639c 100644 --- a/cmd/ingestor/main.go +++ b/cmd/ingestor/main.go @@ -1,11 +1,14 @@ package main import ( + "context" "crypto/hmac" + "crypto/rand" "crypto/sha256" "crypto/tls" "encoding/hex" "encoding/json" + "errors" "flag" "fmt" "log" @@ -23,6 +26,7 @@ import ( "time" mqtt "github.com/eclipse/paho.mqtt.golang" + "github.com/meshcore-analyzer/dbconfig" ) func main() { @@ -41,6 +45,10 @@ func main() { } configPath := flag.String("config", "config.json", "path to config file") + databaseURL := flag.String("database-url", "", "PostgreSQL telemetry writer URL") + stateDir := flag.String("state-dir", "", "local queue and statistics directory") + legacyDB := flag.String("db", "", "SQLite telemetry path (bootstrap override)") + backend := flag.String("backend", "", "Storage backend: sqlite or postgres (bootstrap choice)") flag.Parse() log.SetFlags(log.LstdFlags | log.Lmsgprefix) @@ -72,12 +80,32 @@ func main() { sources := cfg.ResolvedSources() - store, err := OpenStoreWithInterval(cfg.DBPath, cfg.MetricsSampleInterval()) + base, err := os.Getwd() + if err != nil { + log.Fatal("resolve working directory") + } + rawStorage, err := cfg.storageInputs(base, storageFlags{Backend: dbconfig.Backend(*backend), DBPath: *legacyDB, DatabaseURL: *databaseURL, StateDir: *stateDir}, os.Getenv) + if err != nil { + log.Fatalf("storage: %v", err) + } + storage, selectionLease, err := resolveRuntimeStorage(rawStorage) + if errors.Is(err, dbconfig.ErrSelectionMissing) { + if err = adoptLegacyStorage(rawStorage); err == nil { + storage, selectionLease, err = resolveRuntimeStorage(rawStorage) + } + } + if err != nil { + log.Fatalf("storage: %v", err) + } + defer selectionLease.Close() + cfg.StateDir = storage.StateDir + store, err := openStoreStorage(storage, cfg.DB, cfg.MetricsSampleInterval()) + if err != nil { log.Fatalf("db: %v", err) } defer store.Close() - log.Printf("SQLite opened: %s", cfg.DBPath) + log.Printf("%s telemetry writer connected", storage.Backend) // Async backfill: path_json from raw_hex (#888) — must not block MQTT startup store.BackfillPathJSONAsync() @@ -101,13 +129,33 @@ func main() { log.Printf("No channel keys loaded — GRP_TXT packets will not be decrypted") } - regionKeys := loadRegionKeys(cfg) - store.BackfillDefaultScopeAsync(regionKeys) + keySet := newChannelKeySetStorage(channelKeys, storage, cfg.ApprovedChannelsMax()) + if cfg.ApprovedChannelsEnabled() { + log.Printf("[channels] approved channel storage: %s", storage.Backend) + keySet.refresh() + go func() { + t := time.NewTicker(approvedChannelsRefresh) + defer t.Stop() + for range t.C { + keySet.refresh() + } + }() + } + + regionSet := newRegionKeySet(cfg) + if cfg.AutoRegionKeysEnabled() { + // Fill the derived tier before the first packet is matched, so a + // restart does not spend a refresh interval naming nothing. + regionSet.refreshFromStore(store) + } else { + log.Printf("[regions] autoRegionKeys disabled — only the %d configured hashRegions key(s) are in force", len(regionSet.snapshot().all)) + } + store.BackfillDefaultScopeAsync(regionSet) store.BackfillTransportCodesAsync() // Subscribe-early + buffer (#1608): the MQTT subscription is brought up // before startup maintenance so no packets are missed while the single - // SQLite writer is blocked (e.g. a large CREATE INDEX migration). Received + // serialized writer is busy with a bounded data backfill. Received // messages are buffered here and drained once Ready() is called below. ingestBuffer := NewIngestBuffer(cfg.IngestBufferSizeOrDefault()) ingestBuffer.Start() @@ -139,7 +187,7 @@ func main() { status := RegisterSourceStatus(tag, source.Broker) opts.SetOnConnectHandler(func(c mqtt.Client) { - log.Printf("MQTT [%s] connected to %s", tag, source.Broker) + log.Printf("MQTT [%s] connected to %s as client %s", tag, source.Broker, opts.ClientID) status.MarkConnect(time.Now()) // PR #1216 r1 item 2: clear the stale LastMessageUnix from // before the outage so the watchdog doesn't immediately scream @@ -181,7 +229,7 @@ func main() { markReceiptForTag(tag, time.Now()) status.MarkPacket(time.Now()) ingestBuffer.Submit(func() { - handleMessage(store, tag, src, m, channelKeys, regionKeys, cfg) + handleMessage(store, tag, src, m, keySet.Snapshot(), regionSet, cfg) }) }) @@ -313,15 +361,33 @@ func main() { } } + // Declared-region retention: bounds the opt-in node_declared_regions + // table (Task 6), independent of clientRxDays/clientRxObsDays/clientRfDays. + // 0 = disabled. + clientRegionsDays := cfg.ClientRegionsDaysOrZero() + if clientRegionsDays > 0 { + if n, err := store.PruneOldClientDeclaredRegions(clientRegionsDays); err != nil { + log.Printf("[prune] node_declared_regions: %v", err) + } else if n > 0 { + log.Printf("[prune] startup pruned %d node_declared_regions older than %d days", n, clientRegionsDays) + } + } + vacuumPages := cfg.IncrementalVacuumPages() store.RunIncrementalVacuum(vacuumPages) // Gate open: the synchronous startup writes above cannot return until the - // single SQLite writer is free, which means any blocking async migration - // (e.g. the CREATE INDEX) has finished. WaitForAsyncMigrations() makes that + // single serialized writer is free. WaitForAsyncMigrations() makes that // explicit. Now drain everything the subscription buffered during startup. store.WaitForAsyncMigrations() ingestBuffer.Ready() + // History recovery must not join the startup readiness wait above. + // Its known evidence is a lower bound: old UPSERTs erased some frames. + evidenceCtx, stopEvidenceBackfill := context.WithCancel(context.Background()) + defer stopEvidenceBackfill() + if err := store.RunAsyncMigration(evidenceCtx, "advert_route_evidence_v1", store.backfillAdvertEvidence); err != nil { + log.Printf("[migration] scheduling advert evidence backfill: %v", err) + } if d := ingestBuffer.Dropped(); d > 0 { log.Printf("[ingest-buffer] write path ready; draining backlog (dropped %d during startup — consider raising ingestBufferSize)", d) } else { @@ -344,10 +410,12 @@ func main() { store.RemoveStaleObservers(observerDays) store.PurgeStaleObservers(observerPurgeDays) store.RunIncrementalVacuum(vacuumPages) + for range observerRetentionTicker.C { store.RemoveStaleObservers(observerDays) store.PurgeStaleObservers(observerPurgeDays) store.RunIncrementalVacuum(vacuumPages) + } }() @@ -378,11 +446,11 @@ func main() { } // Daily ticker for client-RX coverage retention (#1727), reused for the - // diagnostic client_rx_observations and client_rf_samples retention - // (Task 6) rather than starting a second ticker — the three flags are - // independent (0 disables each separately), so the ticker itself must - // run when any is set. - if clientRxDays > 0 || clientRxObsDays > 0 || clientRfDays > 0 { + // diagnostic client_rx_observations, client_rf_samples, and + // node_declared_regions retention (Task 6) rather than starting a second + // ticker — the four flags are independent (0 disables each separately), + // so the ticker itself must run when any is set. + if clientRxDays > 0 || clientRxObsDays > 0 || clientRfDays > 0 || clientRegionsDays > 0 { clientRxRetentionTicker := time.NewTicker(24 * time.Hour) go func() { for range clientRxRetentionTicker.C { @@ -407,6 +475,13 @@ func main() { store.RunIncrementalVacuum(vacuumPages) } } + if clientRegionsDays > 0 { + if n, err := store.PruneOldClientDeclaredRegions(clientRegionsDays); err != nil { + log.Printf("[prune] node_declared_regions: %v", err) + } else if n > 0 { + store.RunIncrementalVacuum(vacuumPages) + } + } } }() if clientRxDays > 0 { @@ -418,22 +493,81 @@ func main() { if clientRfDays > 0 { log.Printf("[prune] auto-prune enabled: client_rf_samples older than %d days will be removed daily", clientRfDays) } + if clientRegionsDays > 0 { + log.Printf("[prune] auto-prune enabled: node_declared_regions older than %d days will be removed daily", clientRegionsDays) + } } - // Hourly WAL checkpoint to prevent unbounded WAL growth. - // TRUNCATE resets the WAL file to zero bytes when all frames are flushed; - // if the server's read connection holds frames, remaining pages stay in the - // WAL until the next tick. Staggered 30s after startup to avoid competing - // with the initial burst of ingest writes. - walCheckpointTicker := time.NewTicker(1 * time.Hour) - go func() { - time.Sleep(30 * time.Second) - store.Checkpoint() - for range walCheckpointTicker.C { + // Derived region keys refresh on their own ticker rather than the daily + // retention one: declared-region answers arrive continuously (an observer + // report lands, a node is asked again), and waiting up to 24h to name a + // newly-discovered region would defeat the point of deriving them. + if cfg.AutoRegionKeysEnabled() { + interval := time.Duration(cfg.AutoRegionKeysRefreshMinutes()) * time.Minute + regionRefreshTicker := time.NewTicker(interval) + go func() { + for range regionRefreshTicker.C { + regionSet.refreshFromStore(store) + logScopeMatchCounters() + } + }() + log.Printf("[regions] auto-derived region keys enabled: refreshing every %v, cap %d", interval, cfg.AutoRegionKeysMaxDerived()) + } + + // PostgreSQL maintenance remains owned by its database service. + if storage.Backend == dbconfig.SQLite { + // Hourly WAL checkpoint to prevent unbounded WAL growth. + // TRUNCATE resets the WAL file to zero bytes when all frames are flushed; + // if the server's read connection holds frames, remaining pages stay in the + // WAL until the next tick. Staggered 30s after startup to avoid competing + // with the initial burst of ingest writes. + walCheckpointTicker := time.NewTicker(1 * time.Hour) + go func() { + time.Sleep(30 * time.Second) store.Checkpoint() + for range walCheckpointTicker.C { + store.Checkpoint() + } + }() + log.Printf("[db] WAL checkpoint scheduled every 1h") + + // Daily planner statistics refresh (#2058), in two parts. + // + // The routine refresh is staggered 2 minutes past startup for the same reason + // as the checkpoint above: it takes the write lock, and by then the initial + // ingest burst has passed, so it also sees the rows that burst added. + // + // The build in front of it deliberately does compete with that burst, because + // a database with no statistics at all has nothing better to offer the queries + // arriving in those 2 minutes. It only runs once per database; see + // Store.EnsurePlannerStats, which also carries what that costs. + // + // Bounded by analysis_limit either way, so neither grows with the file the way + // an unbounded ANALYZE does: 2.0s against 242.9s on a 9.4 GB database, both + // timed warm. Cold, on a first start, it is 3m43.9s. + { + analysisLimit := cfg.AnalysisLimit() + if analysisLimit < 0 { + log.Printf("[analyze] planner statistics refresh disabled (db.analysisLimit=%d)", analysisLimit) + } else { + analyzeTicker := time.NewTicker(24 * time.Hour) + go func() { + // Before the stagger, and only on a database that has never been + // analyzed: the stagger is a 2 minute window in which the first + // query would otherwise run on no statistics at all. A restart + // finds sqlite_stat1 already in the file and skips this. + store.EnsurePlannerStats(analysisLimit) + time.Sleep(2 * time.Minute) + store.RefreshPlannerStats(analysisLimit) + for range analyzeTicker.C { + store.RefreshPlannerStats(analysisLimit) + } + }() + log.Printf("[analyze] planner statistics refresh scheduled every 24h (analysis_limit=%d)", analysisLimit) + } } - }() - log.Printf("[db] WAL checkpoint scheduled every 1h") + + } // Daily neighbor_edges retention (#1287 — moved from cmd/server). { @@ -462,6 +596,13 @@ func main() { go func() { for range statsTicker.C { store.LogStats() + // Persist the scope-match tally on the stats cadence rather + // than the region-refresh one: the counters are recorded for + // every transport-scoped packet, including on instances that + // never enable autoRegionKeys and so never run that ticker. + if err := store.SaveScopeMatchTotals(); err != nil { + log.Printf("[regions] saving scope-match tally: %v", err) + } if d := ingestBuffer.Dropped(); d > 0 || ingestBuffer.Pending() > 0 { log.Printf("[ingest-buffer] pending=%d dropped_total=%d", ingestBuffer.Pending(), d) } @@ -527,6 +668,7 @@ func main() { <-sig log.Println("Shutting down...") + stopEvidenceBackfill() retentionTicker.Stop() metricsRetentionTicker.Stop() if packetRetentionTicker != nil { @@ -534,8 +676,13 @@ func main() { } statsTicker.Stop() pruneQueueTicker.Stop() - walCheckpointTicker.Stop() stopWatchdog() + // A deploy is a SIGTERM, which is exactly the case that used to lose + // the tally: save before the process goes away rather than leaving up + // to 5 minutes of counting to the next tick that will not come. + if err := store.SaveScopeMatchTotals(); err != nil { + log.Printf("[regions] saving scope-match tally: %v", err) + } store.LogStats() // final stats on shutdown for _, c := range clients { c.Disconnect(5000) // 5s to allow in-flight messages to drain @@ -573,7 +720,10 @@ func buildMQTTOpts(source MQTTSource) *mqtt.ClientOptions { // (paho default 30s actually — making this explicit so it can't // drift, and so operators reading the code know it's intentional // per the #1335 RCA). - SetKeepAlive(30 * time.Second) + SetKeepAlive(30 * time.Second). + // #2013: an explicit ClientID, fixed here once per source so every + // reconnect of this client presents the same identity. + SetClientID(mqttClientID(source)) opts.SetConnectionAttemptHandler(func(broker *url.URL, tlsCfg *tls.Config) *tls.Config { // Look up the per-source liveness state (registered in main) so we @@ -606,6 +756,38 @@ func buildMQTTOpts(source MQTTSource) *mqtt.ClientOptions { return opts } +// mqttClientID returns the configured clientId, or corescope-- +// with the source name (or broker host) reduced to [0-9A-Za-z-]. The random +// suffix keeps two instances with the same config from taking over each +// other's session on a broker that does not assign unique IDs itself. +func mqttClientID(source MQTTSource) string { + if source.ClientID != "" { + return source.ClientID + } + name := source.Name + if name == "" { + if u, err := url.Parse(source.Broker); err == nil { + name = u.Hostname() + } + } + var b strings.Builder + b.WriteString("corescope-") + for _, r := range name { + if r >= '0' && r <= '9' || r >= 'A' && r <= 'Z' || r >= 'a' && r <= 'z' { + b.WriteRune(r) + } else { + b.WriteByte('-') + } + } + if name != "" { + b.WriteByte('-') + } + suffix := make([]byte, 3) + rand.Read(suffix) + b.WriteString(hex.EncodeToString(suffix)) + return b.String() +} + // buildForceReconnectFn builds the watchdog's forced-reconnect action for a // source (#1335, hardened against a race found while investigating a 100+ // minute reconnect failure). @@ -654,7 +836,7 @@ func buildForceReconnectFn(client mqtt.Client, tag string) func() { } } -func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message, channelKeys map[string]string, regionKeys map[string][]byte, cfg *Config) { +func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message, channelKeys map[string]string, regionSet *regionKeySet, cfg *Config) { // Liveness watchdog (#1212): record receipt before any processing so a // slow handler still counts as "source is alive". Cheap atomic store. markLivenessForTag(tag, time.Now()) @@ -672,11 +854,13 @@ func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message, return } - // Mobile client topics: meshcore/client/{PUBLIC_KEY}/packets (RX coverage) - // and meshcore/client/{PUBLIC_KEY}/rf (RF environment samples). A roaming - // companion reports where it directly heard a node, or its own radio's - // counters; both are handled in isolation from the observer/observations - // path. EMQX ACL binds parts[2] to the client's own key. + // Mobile client topics: meshcore/client/{PUBLIC_KEY}/packets (RX coverage), + // meshcore/client/{PUBLIC_KEY}/rf (RF environment samples), and + // meshcore/client/{PUBLIC_KEY}/regions (declared-region answers). A + // roaming companion reports where it directly heard a node, its own + // radio's counters, or a repeater's declared region list; all three are + // handled in isolation from the observer/observations path. EMQX ACL + // binds parts[2] to the client's own key. // // The topic match and the enable-gate MUST be separate: matching on // parts[1]=="client" always returns from this branch, whatever the config @@ -698,12 +882,16 @@ func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message, switch parts[3] { case "packets": if cfg.ClientRxCoverageEnabled() { - handleClientPacket(store, cfg, tag, parts[2], msg, channelKeys, regionKeys) + handleClientPacket(store, cfg, tag, parts[2], msg, channelKeys, regionSet) } case "rf": if cfg.ClientRfSamplesEnabled() { handleClientRfSample(store, tag, parts[2], msg) } + case "regions": + if cfg.ClientRegionsEnabled() { + handleClientRegions(store, cfg, tag, parts[2], msg) + } } return } @@ -723,6 +911,25 @@ func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message, // Global observer IATA whitelist: if configured, drop messages from observers // in non-whitelisted IATA regions. Applies to ALL message types (status + packets). if len(parts) > 1 && !cfg.IsObserverIATAAllowed(parts[1]) { + // Throttled to one line per region per cfg.IATAWarnInterval — see + // ShouldWarnIATADrop. Format matches the fleet's Python region filter so + // one scraper regex covers both. + if cfg.ShouldWarnIATADrop(parts[1]) { + code := strings.ToUpper(strings.TrimSpace(parts[1])) + log.Printf("MQTT [%s] [region-filter] dropping unknown region '%s' (not in observerIATAWhitelist) -- further messages from %s suppressed for %.0fh", + tag, code, code, cfg.IATAWarnInterval().Hours()) + } + return + } + + // Neighbors report topic: meshcore///neighbors (#1865). + // The ESP32 observer firmware emits a periodic neighbor report carrying its + // own configured region scopes (`self`) plus, for each zero-hop neighbor, + // the scopes fetched via an OTA scope query. Like /status this is observer + // metadata (region-independent), so the per-source packet IATA filter below + // does not apply. + if len(parts) >= 4 && parts[3] == "neighbors" { + handleNeighborsReport(store, tag, parts[2], msg) return } @@ -731,9 +938,10 @@ func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message, // is region-independent and should be accepted from all observers regardless of // which IATA regions are configured for packet ingestion. if len(parts) >= 4 && parts[3] == "status" { - observerID := parts[2] + observerID := clampObserverField(parts[2], maxObserverIDLen) name, _ := msg["origin"].(string) - iata := parts[1] + name = clampObserverField(name, maxObserverTextLen) + iata := clampObserverField(parts[1], maxObserverIATALen) meta := extractObserverMeta(msg) // A replayed status message is the broker handing us the observer's // last published snapshot — it is not evidence the observer is alive @@ -828,10 +1036,10 @@ func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message, observerID := "" region := "" if len(parts) > 2 { - observerID = parts[2] + observerID = clampObserverField(parts[2], maxObserverIDLen) } if len(parts) > 1 { - region = parts[1] + region = clampObserverField(parts[1], maxObserverIATALen) } // Fallback to source-level region config when topic has no region (#788) if region == "" && source.Region != "" { @@ -935,7 +1143,7 @@ func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message, log.Printf("MQTT [%s] foreign advert: node=%s name=%s lat=%.4f lon=%.4f observer=%s", tag, truncPK, sanitizeLogString(decoded.Payload.Name), lat, lon, sanitizeLogString(firstNonEmpty(mqttMsg.Origin, observerID))) } - pktData := BuildPacketData(mqttMsg, decoded, observerID, region, regionKeys) + pktData := BuildPacketData(mqttMsg, decoded, observerID, region, regionSet) pktData.Foreign = foreign isNew, err := store.InsertTransmission(pktData) if err != nil { @@ -970,7 +1178,7 @@ func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message, } else { // Non-ADVERT packets: store normally (routing/channel messages from // in-area observers are relevant regardless of relay hop origin). - pktData := BuildPacketData(mqttMsg, decoded, observerID, region, regionKeys) + pktData := BuildPacketData(mqttMsg, decoded, observerID, region, regionSet) if _, err := store.InsertTransmission(pktData); err != nil { log.Printf("MQTT [%s] db insert error: %v", tag, err) } @@ -979,6 +1187,7 @@ func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message, // Upsert observer if observerID != "" { origin, _ := msg["origin"].(string) + origin = clampObserverField(origin, maxObserverTextLen) // Use effective region: payload > topic > source config (#788) effectiveRegion := region if mqttMsg.Region != "" { @@ -1237,26 +1446,32 @@ func extractObserverMeta(msg map[string]interface{}) *ObserverMeta { hasData := false if v, ok := msg["model"].(string); ok && v != "" { + v = clampObserverField(v, maxObserverTextLen) meta.Model = &v hasData = true } if v, ok := msg["firmware"].(string); ok && v != "" { + v = clampObserverField(v, maxObserverTextLen) meta.Firmware = &v hasData = true } if v, ok := msg["firmware_version"].(string); ok && v != "" { + v = clampObserverField(v, maxObserverTextLen) meta.Firmware = &v hasData = true } if v, ok := msg["client_version"].(string); ok && v != "" { + v = clampObserverField(v, maxObserverTextLen) meta.ClientVersion = &v hasData = true } if v, ok := msg["clientVersion"].(string); ok && v != "" { + v = clampObserverField(v, maxObserverTextLen) meta.ClientVersion = &v hasData = true } if v, ok := msg["radio"].(string); ok && v != "" { + v = clampObserverField(v, maxObserverTextLen) meta.Radio = &v hasData = true } @@ -1624,12 +1839,25 @@ func loadRegionKeys(cfg *Config) map[string][]byte { return keys } -// matchScope performs one HMAC-SHA256 per configured region. Expected -// len(regionKeys) ≤ 50; beyond that, consider a pre-indexed lookup table. -func matchScope(regionKeys map[string][]byte, payloadType byte, payloadRaw []byte, code1 string) string { +// matchingRegions returns every configured region whose derived code equals +// the packet's code1, rather than the first one found. +// +// The distinction matters because code1 is two bytes: two configured regions +// collide on a given payload with probability 1/65536, and at 159 keys on a +// live instance that is roughly 0.25% of transport-scoped packets, hundreds a +// week rather than a curiosity. Returning the first match made the stored +// region name depend on Go's randomised map iteration order, so the same +// packet could be named differently on two runs and neither answer was +// evidence of anything. +// +// The cost is unchanged: this is the same single pass over the same keys, it +// just does not stop early. There is no indexable shortcut, because code1 is +// an HMAC over the payload and nothing here is payload-independent. +func matchingRegions(regionKeys map[string][]byte, payloadType byte, payloadRaw []byte, code1 string) []string { if code1 == "0000" || len(regionKeys) == 0 || len(payloadRaw) == 0 { - return "" + return nil } + var matched []string for name, key := range regionKeys { mac := hmac.New(sha256.New, key) mac.Write([]byte{payloadType}) @@ -1643,10 +1871,10 @@ func matchScope(regionKeys map[string][]byte, payloadType byte, payloadRaw []byt } codeBytes := [2]byte{byte(code & 0xFF), byte(code >> 8)} if strings.ToUpper(hex.EncodeToString(codeBytes[:])) == code1 { - return name + matched = append(matched, name) } } - return "" + return matched } // Version info (set via ldflags) @@ -1659,6 +1887,63 @@ func init() { } } +// handleNeighborsReport ingests an observer /neighbors report (#1865) and +// records CONFIRMED region scopes into nodes.configured_scope: +// - the observer's own scopes from `self`, keyed by origin_id (the observer +// node pubkey), which need no OTA query and are always trusted; and +// - each neighbor whose OTA scope query returned status=="responded". +// +// Per the report contract: neighbors with any other status (e.g. "timeout") +// are skipped — a failed query is NOT evidence the scopes were cleared — and a +// missing neighbor is never a signal (the report is 10 KB-capped and truncates +// by ordering, so absent != gone). Report pubkeys are uppercase; nodes.public_key +// is lowercase hex, so keys are lowercased before the UPDATE. Unknown neighbors +// are a no-op (the UPDATE matches no row) until a later advert creates the node. +func handleNeighborsReport(store *Store, tag string, observerID string, msg map[string]interface{}) { + reportedAt, _ := msg["timestamp"].(string) + + // self: the observer's own configured scopes. + originID, _ := msg["origin_id"].(string) + if originID == "" { + originID = observerID + } + originID = strings.ToLower(originID) + // The report is unauthenticated: any publisher can name any key. Only a + // 64-hex node key can match a nodes row, so drop anything else here + // instead of running UPDATEs that can never match. + if !targetPubkeyRe.MatchString(originID) { + originID = "" + } + if self, ok := msg["self"].(map[string]interface{}); ok && originID != "" { + if sc, ok := self["scopes"].(string); ok { + if err := store.UpdateNodeConfiguredScope(originID, sc, reportedAt); err != nil { + log.Printf("MQTT [%s] neighbors self scope error: %v", tag, err) + } + } + } + + // neighbors[]: only status=="responded" carries usable scope evidence. + neighbors, _ := msg["neighbors"].([]interface{}) + for _, raw := range neighbors { + n, ok := raw.(map[string]interface{}) + if !ok { + continue + } + if status, _ := n["status"].(string); status != "responded" { + continue + } + pubkey, _ := n["pubkey"].(string) + pubkey = strings.ToLower(pubkey) + if !targetPubkeyRe.MatchString(pubkey) { + continue + } + scopes, _ := n["scopes"].(string) + if err := store.UpdateNodeConfiguredScope(pubkey, scopes, reportedAt); err != nil { + log.Printf("MQTT [%s] neighbors scope error for %.8s: %v", tag, pubkey, err) + } + } +} + // shouldUpdateDefaultScope returns true when the packet carries a transport // scope whose region key matched (#1534). Without the ScopeName non-empty // guard, transport-scoped adverts from non-matching regions would overwrite diff --git a/cmd/ingestor/main_test.go b/cmd/ingestor/main_test.go index 9d241f46b..dfed98948 100644 --- a/cmd/ingestor/main_test.go +++ b/cmd/ingestor/main_test.go @@ -2,6 +2,8 @@ package main import ( "bytes" + "crypto/hmac" + "crypto/sha256" "database/sql" "encoding/hex" "encoding/json" @@ -10,6 +12,7 @@ import ( "os" "path/filepath" "runtime" + "strings" "testing" "time" @@ -122,10 +125,27 @@ func newTestStore(t *testing.T) *Store { t.Helper() dir := t.TempDir() dbPath := dir + "/test.db" - s, err := OpenStore(dbPath) + s, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatal(err) } + // OpenStore schedules two async migrations (db.go: obs_observer_ts_idx_v1 + // and tx_last_seen_backfill_v1), and their goroutines log while they run. + // Twenty-one places in these tests capture output by pointing the standard + // logger at a bytes.Buffer, so a test that does so while its OWN store is + // still migrating has two goroutines on one buffer with no synchronisation. + // That is the data race the race detector caught on master (#2065): the + // writer was RunAsyncMigration's goroutine, the reader + // TestHandleMessageDecodeErrorLog_PII_Issue1211 reading buf.String(). + // + // Close() already waits, so the goroutines never outlive the test; the race + // is inside one test, not across them. Waiting here rather than only at + // cleanup means no test body can observe a migration in flight, which + // closes the whole class rather than the one test that happened to fail. + // + // Cost is a few milliseconds against an empty temp DB. Pinned by + // TestNewTestStoreWaitsForBootMigrations. + s.WaitForAsyncMigrations() t.Cleanup(func() { s.Close() }) return s } @@ -858,7 +878,7 @@ func TestMatchScope(t *testing.T) { // Key = SHA256("#test")[:16] = 9cd8fcf22a47333b591d96a2b848b73f testKey, _ := hex.DecodeString("9cd8fcf22a47333b591d96a2b848b73f") testKeys := map[string][]byte{"#test": testKey} - if got := matchScope(testKeys, 5, []byte("hello"), "2AB5"); got != "#test" { + if got := regionKeySetFromKeys(testKeys).matchScopeName(5, []byte("hello"), "2AB5"); got != "#test" { t.Errorf("#test vector: matchScope = %q, want #test", got) } @@ -866,17 +886,17 @@ func TestMatchScope(t *testing.T) { // Key = SHA256("#belgium")[:16] = 7085b78ed010599094f8c8e7d1aa0e27 belgiumKey, _ := hex.DecodeString("7085b78ed010599094f8c8e7d1aa0e27") belgiumKeys := map[string][]byte{"#belgium": belgiumKey} - if got := matchScope(belgiumKeys, 5, []byte("hello"), "4A75"); got != "#belgium" { + if got := regionKeySetFromKeys(belgiumKeys).matchScopeName(5, []byte("hello"), "4A75"); got != "#belgium" { t.Errorf("#belgium vector: matchScope = %q, want #belgium", got) } // Code1=0000 (unscoped transport) → no region matched - if got := matchScope(belgiumKeys, 5, []byte("hello"), "0000"); got != "" { + if got := regionKeySetFromKeys(belgiumKeys).matchScopeName(5, []byte("hello"), "0000"); got != "" { t.Errorf("unscoped: matchScope = %q, want empty", got) } // Code1 present but matches no configured region → empty string - if got := matchScope(belgiumKeys, 5, []byte("hello"), "BEEF"); got != "" { + if got := regionKeySetFromKeys(belgiumKeys).matchScopeName(5, []byte("hello"), "BEEF"); got != "" { t.Errorf("no match: matchScope = %q, want empty", got) } } @@ -896,7 +916,7 @@ func TestBuildPacketDataScopeMatching(t *testing.T) { } msg := &MQTTPacketMessage{Raw: rawHex} - pktData := BuildPacketData(msg, decoded, "obs1", "region1", regionKeys) + pktData := BuildPacketData(msg, decoded, "obs1", "region1", regionKeySetFromKeys(regionKeys)) if pktData.ScopeName != "#test" { t.Errorf("ScopeName = %q, want #test", pktData.ScopeName) } @@ -1075,7 +1095,7 @@ func TestBuildPacketDataScopeMatchingNoMatch(t *testing.T) { t.Fatalf("DecodePacket: %v", err) } msg := &MQTTPacketMessage{Raw: rawHex} - pktData := BuildPacketData(msg, decoded, "obs1", "region1", regionKeys) + pktData := BuildPacketData(msg, decoded, "obs1", "region1", regionKeySetFromKeys(regionKeys)) if !pktData.IsTransportScoped { t.Fatalf("precondition: IsTransportScoped should be true (Code1 != 0000)") @@ -1115,7 +1135,7 @@ func TestHandleMessageAdvert_EmptyScopeSkipsDefaultScopeUpdate(t *testing.T) { // Pre-seed the node with a non-empty default_scope so we can detect an // erroneous overwrite with "". - if _, err := store.db.Exec(`INSERT INTO nodes (public_key, name, default_scope) VALUES (?, 'Node1', '#belgium')`, pubkey); err != nil { + if _, err := store.db.Exec(`INSERT INTO nodes (public_key, name, default_scope) VALUES ($1, 'Node1', '#belgium')`, pubkey); err != nil { t.Fatalf("seed node: %v", err) } @@ -1124,10 +1144,10 @@ func TestHandleMessageAdvert_EmptyScopeSkipsDefaultScopeUpdate(t *testing.T) { topic: "meshcore/SJC/obs1/packets", payload: []byte(`{"raw":"` + rawHex + `"}`), } - handleMessage(store, "test", source, msg, nil, map[string][]byte{}, &Config{}) + handleMessage(store, "test", source, msg, nil, nil, &Config{}) var got sql.NullString - if err := store.db.QueryRow(`SELECT default_scope FROM nodes WHERE public_key = ?`, pubkey).Scan(&got); err != nil { + if err := store.db.QueryRow(`SELECT default_scope FROM nodes WHERE public_key = $1`, pubkey).Scan(&got); err != nil { t.Fatalf("read default_scope: %v", err) } if !got.Valid || got.String != "#belgium" { @@ -1168,7 +1188,7 @@ func TestHandleMessageAdvert_MatchedScopeUpdatesDefaultScope(t *testing.T) { code1 := fmt.Sprintf("%02X%02X", byte(code&0xFF), byte(code>>8)) rawHex := "10" + code1 + "000000" + advertBytes - if _, err := store.db.Exec(`INSERT INTO nodes (public_key, name, default_scope) VALUES (?, 'Node1', '#old')`, pubkey); err != nil { + if _, err := store.db.Exec(`INSERT INTO nodes (public_key, name, default_scope) VALUES ($1, 'Node1', '#old')`, pubkey); err != nil { t.Fatalf("seed node: %v", err) } @@ -1176,13 +1196,98 @@ func TestHandleMessageAdvert_MatchedScopeUpdatesDefaultScope(t *testing.T) { topic: "meshcore/SJC/obs1/packets", payload: []byte(`{"raw":"` + rawHex + `"}`), } - handleMessage(store, "test", source, msg, nil, map[string][]byte{"#de": regionKey}, &Config{}) + handleMessage(store, "test", source, msg, nil, regionKeySetFromKeys(map[string][]byte{"#de": regionKey}), &Config{}) var got sql.NullString - if err := store.db.QueryRow(`SELECT default_scope FROM nodes WHERE public_key = ?`, pubkey).Scan(&got); err != nil { + if err := store.db.QueryRow(`SELECT default_scope FROM nodes WHERE public_key = $1`, pubkey).Scan(&got); err != nil { t.Fatalf("read default_scope: %v", err) } if !got.Valid || got.String != "#de" { t.Errorf("default_scope after matched-scope advert = %q (valid=%v), want #de", got.String, got.Valid) } } + +// codeForRegion derives the on-wire code1 a sender in this region would emit +// for this payload: the forward direction of what matchScope inverts. +func codeForRegion(name string, payloadType byte, payload []byte) string { + if !strings.HasPrefix(name, "#") { + name = "#" + name + } + sum := sha256.Sum256([]byte(name)) + mac := hmac.New(sha256.New, sum[:16]) + mac.Write([]byte{payloadType}) + mac.Write(payload) + h := mac.Sum(nil) + code := uint16(h[0]) | uint16(h[1])<<8 + if code == 0 { + code = 1 + } else if code == 0xFFFF { + code = 0xFFFE + } + return strings.ToUpper(hex.EncodeToString([]byte{byte(code & 0xFF), byte(code >> 8)})) +} + +// findRegionCollision searches for a payload whose code1 is identical under two +// region names. code1 is two bytes, so one turns up after ~65k tries and the +// search costs a fraction of a second. +// +// A hand-built fixture cannot stand in here: the whole point of the case below +// is what happens when the matcher genuinely finds two names for one packet, +// and a fabricated code1 would only prove the test agrees with itself. +func findRegionCollision(t *testing.T, nameA, nameB string, payloadType byte) ([]byte, string) { + t.Helper() + payload := make([]byte, 4) + for i := 0; i < 1<<22; i++ { + payload[0], payload[1] = byte(i), byte(i>>8) + payload[2], payload[3] = byte(i>>16), byte(i>>24) + if a, b := codeForRegion(nameA, payloadType, payload), codeForRegion(nameB, payloadType, payload); a == b { + return append([]byte(nil), payload...), a + } + } + t.Fatalf("no code1 collision between %s and %s in 2^22 payloads", nameA, nameB) + return nil, "" +} + +// TestMatchScopeNamesAnUnambiguousPacket is the ordinary case: one configured +// region derives the packet's code1, so the packet carries that region's name. +func TestMatchScopeNamesAnUnambiguousPacket(t *testing.T) { + keys := loadRegionKeys(&Config{HashRegions: []string{"#be", "#nl"}}) + payload := []byte{0x01, 0x02, 0x03, 0x04} + code1 := codeForRegion("#be", 5, payload) + + if got := regionKeySetFromKeys(keys).matchScopeName(5, payload, code1); got != "#be" { + t.Errorf("matchScope = %q, want %q", got, "#be") + } +} + +// TestMatchScopeStoresAmbiguousAsUnmatched pins the reason this changed. Two +// configured regions derive the same code1 for one payload; naming the packet +// after either is a coin flip, and before this the flip was Go's map iteration +// order, so the same packet could be stored under different regions on two +// runs of the same binary. +func TestMatchScopeStoresAmbiguousAsUnmatched(t *testing.T) { + payload, code1 := findRegionCollision(t, "#be", "#zz", 5) + keys := loadRegionKeys(&Config{HashRegions: []string{"#be", "#zz"}}) + + if n := len(matchingRegions(keys, 5, payload, code1)); n != 2 { + t.Fatalf("matchingRegions returned %d names, want 2 — the collision fixture is wrong", n) + } + if got := regionKeySetFromKeys(keys).matchScopeName(5, payload, code1); got != "" { + t.Errorf("matchScope = %q, want the unmatched state: two equally-sourced candidates have no principled winner", got) + } +} + +// TestMatchScopeIsOrderIndependent runs the ambiguous case repeatedly. Map +// iteration order is randomised per range in Go, so a first-match matcher +// returns different names across iterations of this loop; the answer must not +// move. +func TestMatchScopeIsOrderIndependent(t *testing.T) { + payload, code1 := findRegionCollision(t, "#be", "#zz", 5) + keys := loadRegionKeys(&Config{HashRegions: []string{"#be", "#zz"}}) + + for i := 0; i < 50; i++ { + if got := regionKeySetFromKeys(keys).matchScopeName(5, payload, code1); got != "" { + t.Fatalf("iteration %d: matchScope = %q, want a stable answer across map iteration orders", i, got) + } + } +} diff --git a/cmd/ingestor/maintenance.go b/cmd/ingestor/maintenance.go index 8159d8402..43cbb73a6 100644 --- a/cmd/ingestor/maintenance.go +++ b/cmd/ingestor/maintenance.go @@ -5,47 +5,106 @@ import ( "encoding/json" "fmt" "log" + "runtime" "time" "github.com/meshcore-analyzer/dbschema" ) +// pruneBatchTransmissions bounds how many transmissions (and their child +// observations) one prune transaction deletes before it commits and +// releases the writer lock. +// +// Releasing between batches is the entire point. writerMu serialises every +// wrapped writer call, so while the prune holds it the MQTT ingest path is +// blocked. Go's sync.Mutex switches to FIFO handoff once a waiter has been +// blocked for 1ms (starvation mode), so an ingest goroutine queued behind +// the prune is served at the next batch boundary instead of after the whole +// retention day. +// +// The bound is on transmissions, but hold time scales with the rows actually +// deleted, and each transmission carries an unbounded number of observations. +// At ~16 observations per transmission a batch is ~4k row deletes and a few +// hundred milliseconds; an instance with a denser observation ratio gets a +// proportionally longer hold from the same batch size. 250 also keeps the +// commit count low (a 16k-transmission day is 64 transactions, not 16k). +const pruneBatchTransmissions = 250 + +// pruneAgedTransmissionIDs selects the next batch of transmissions older than +// the cutoff. Both statements of a batch embed it, so they resolve the same +// set: nothing modifies `transmissions` between them inside the transaction. +// +// The ORDER BY must be satisfiable from idx_transmissions_first_seen. That +// PostgreSQL index explicitly includes id as the tiebreaker (SQLite carried +// its rowid implicitly). This lets the LIMIT stop the ordered scan even when +// timestamps tie. Ordering by id alone could scan the entire table when no +// rows have aged out. TestPruneAgedTransmissionIDsHasOrderedIndex pins ordered +// index eligibility for both the batch query and its DELETE callers. +const pruneAgedTransmissionIDs = `SELECT id FROM transmissions WHERE first_seen < $1 ORDER BY first_seen, id LIMIT $2` + +// The two statements of one prune batch. Child observations go first, retaining +// the bounded deletion behavior of the SQLite implementation. +const ( + pruneObservationsBatch = `DELETE FROM observations WHERE transmission_id IN (` + pruneAgedTransmissionIDs + `)` + pruneTransmissionsBatch = `DELETE FROM transmissions WHERE id IN (` + pruneAgedTransmissionIDs + `)` +) + // PruneOldPackets deletes transmissions (and their child observations) // older than `days`. Returns count of transmissions deleted. // // Owned by the ingestor per #1283: the writer process is the only one // allowed to hold the DB write lock; previously this lived in // cmd/server/db.go and raced ingestor INSERTs (SQLITE_BUSY). +// +// Deletion is chunked into bounded transactions so ingest is never blocked +// for longer than a single batch. Deleting a whole retention day in one +// transaction held the writer lock for ~35s on a mesh doing ~260k +// observations/day, stalling MQTT ingest for the same duration. +// +// On error the transmissions deleted by already-committed batches are +// returned alongside it — those rows are gone, so reporting 0 would be +// wrong. func (s *Store) PruneOldPackets(days int) (int64, error) { + return s.pruneOldPackets(days, runtime.Gosched) +} +func (s *Store) pruneOldPackets(days int, betweenBatches func()) (int64, error) { if days <= 0 { return 0, nil } cutoff := time.Now().UTC().AddDate(0, 0, -days).Format(time.RFC3339) - // Tagged for writer-perf visibility (#1340). - var n int64 - err := s.WriterTx("prune_packets", func(tx *sql.Tx) error { - // Delete child observations first (no CASCADE in SQLite). - if _, err := tx.Exec(`DELETE FROM observations WHERE transmission_id IN ( - SELECT id FROM transmissions WHERE first_seen < ? - )`, cutoff); err != nil { - return fmt.Errorf("prune observations: %w", err) - } - - res, err := tx.Exec(`DELETE FROM transmissions WHERE first_seen < ?`, cutoff) + var total int64 + for { + var batch int64 + // Tagged for writer-perf visibility (#1340). + err := s.WriterTx("prune_packets", func(tx *sql.Tx) error { + if _, err := tx.Exec(pruneObservationsBatch, cutoff, pruneBatchTransmissions); err != nil { + return fmt.Errorf("prune observations: %w", err) + } + res, err := tx.Exec(pruneTransmissionsBatch, cutoff, pruneBatchTransmissions) + if err != nil { + return fmt.Errorf("prune transmissions: %w", err) + } + batch, _ = res.RowsAffected() + return nil + }) if err != nil { - return fmt.Errorf("prune transmissions: %w", err) + return total, err } - n, _ = res.RowsAffected() - return nil - }) - if err != nil { - return 0, err + total += batch + // A short batch proves nothing is left below the cutoff: the subquery + // found fewer rows than it was allowed to take. Only a batch that came + // back exactly full needs another pass. + if batch < pruneBatchTransmissions { + break + } + // Yield only after the transaction commits and releases the writer. + betweenBatches() } - if n > 0 { - log.Printf("[prune] deleted %d transmissions older than %d days", n, days) + if total > 0 { + log.Printf("[prune] deleted %d transmissions older than %d days", total, days) } - return n, nil + return total, nil } // PruneOldClientReceptions deletes mobile client-RX coverage rows older than @@ -61,13 +120,13 @@ func (s *Store) PruneOldClientReceptions(days int) (int64, error) { var n int64 err := s.WriterTx("prune_client_receptions", func(tx *sql.Tx) error { - res, err := tx.Exec(`DELETE FROM client_receptions WHERE rx_at < ?`, cutoff) + res, err := tx.Exec(`DELETE FROM client_receptions WHERE rx_at < `+s.parameter(1), cutoff) if err != nil { return fmt.Errorf("prune client_receptions: %w", err) } n, _ = res.RowsAffected() // Drop companion name rows not refreshed within the window. - if _, err := tx.Exec(`DELETE FROM client_observers WHERE last_seen < ?`, cutoff); err != nil { + if _, err := tx.Exec(`DELETE FROM client_observers WHERE last_seen < `+s.parameter(1), cutoff); err != nil { return fmt.Errorf("prune client_observers: %w", err) } return nil @@ -93,7 +152,7 @@ func (s *Store) PruneOldClientRxObservations(days int) (int64, error) { return 0, nil } cutoff := time.Now().UTC().AddDate(0, 0, -days).Format(rxTimeMillisLayout) - res, err := s.db.Exec(`DELETE FROM client_rx_observations WHERE rx_at < ?`, cutoff) + res, err := s.db.Exec(`DELETE FROM client_rx_observations WHERE rx_at < `+s.parameter(1), cutoff) if err != nil { return 0, fmt.Errorf("prune client_rx_observations: %w", err) } @@ -116,7 +175,7 @@ func (s *Store) PruneOldClientRfSamples(days int) (int64, error) { return 0, nil } cutoff := time.Now().UTC().AddDate(0, 0, -days).Format(rxTimeMillisLayout) - res, err := s.db.Exec(`DELETE FROM client_rf_samples WHERE sampled_at < ?`, cutoff) + res, err := s.db.Exec(`DELETE FROM client_rf_samples WHERE sampled_at < `+s.parameter(1), cutoff) if err != nil { return 0, fmt.Errorf("prune client_rf_samples: %w", err) } @@ -127,6 +186,41 @@ func (s *Store) PruneOldClientRfSamples(days int) (int64, error) { return n, nil } +// PruneOldClientDeclaredRegions deletes declared-region observation rows +// (node_declared_regions) older than `days` (by observed_at). Independent of +// clientRxDays/clientRxObsDays/clientRfDays. 0 disables. Owned by the +// ingestor writer (#1283). observed_at on this table is stored at +// millisecond precision via rxTimeMillisLayout, not RFC3339 — the cutoff +// must be formatted the same way so lexicographic comparison against +// stored values stays correct. +func (s *Store) PruneOldClientDeclaredRegions(days int) (int64, error) { + if days <= 0 { + return 0, nil + } + n, err := s.pruneOldClientDeclaredRegionsAt(time.Now().UTC().AddDate(0, 0, -days)) + if err != nil { + return 0, err + } + if n > 0 { + log.Printf("[prune] deleted %d node_declared_regions older than %d days", n, days) + } + return n, nil +} + +// pruneOldClientDeclaredRegionsAt is the cutoff-instant seam behind +// PruneOldClientDeclaredRegions: it exists so tests can pin the cutoff +// deterministically instead of racing time.Now() to place a fixture row on a +// specific side of it. +func (s *Store) pruneOldClientDeclaredRegionsAt(cutoffInstant time.Time) (int64, error) { + cutoff := cutoffInstant.Format(rxTimeMillisLayout) + res, err := s.db.Exec(`DELETE FROM node_declared_regions WHERE observed_at < `+s.parameter(1), cutoff) + if err != nil { + return 0, fmt.Errorf("prune node_declared_regions: %w", err) + } + n, _ := res.RowsAffected() + return n, nil +} + // SoftDeleteBlacklistedObservers marks observers in the blacklist as // inactive=1 so they are hidden from API responses. Owned by ingestor // per #1287. Runs once at startup. @@ -149,7 +243,7 @@ func (s *Store) PruneNeighborEdges(maxAgeDays int) (int64, error) { return 0, nil } cutoff := time.Now().UTC().Add(-time.Duration(maxAgeDays) * 24 * time.Hour).Format(time.RFC3339) - res, err := s.db.Exec("DELETE FROM neighbor_edges WHERE last_seen < ?", cutoff) + res, err := s.db.Exec("DELETE FROM neighbor_edges WHERE last_seen < "+s.parameter(1), cutoff) if err != nil { return 0, fmt.Errorf("prune neighbor_edges: %w", err) } @@ -211,7 +305,7 @@ func (s *Store) BackfillFromPubkey(chunkSize int, yieldDuration time.Duration, p } log.Printf("[backfill] from_pubkey starting: %d ADVERT rows", total) - stmt, err := s.db.Prepare("UPDATE transmissions SET from_pubkey = ? WHERE id = ?") + stmt, err := s.db.Prepare("UPDATE transmissions SET from_pubkey = " + s.parameter(1) + " WHERE id = " + s.parameter(2)) if err != nil { log.Printf("[backfill] from_pubkey prepare: %v", err) return @@ -221,7 +315,7 @@ func (s *Store) BackfillFromPubkey(chunkSize int, yieldDuration time.Duration, p var processed int64 for { rows, err := s.db.Query( - "SELECT id, decoded_json FROM transmissions WHERE from_pubkey IS NULL AND payload_type = 4 LIMIT ?", + "SELECT id, decoded_json FROM transmissions WHERE from_pubkey IS NULL AND payload_type = 4 LIMIT "+s.parameter(1), chunkSize) if err != nil { log.Printf("[backfill] from_pubkey select: %v", err) @@ -245,7 +339,7 @@ func (s *Store) BackfillFromPubkey(chunkSize int, yieldDuration time.Duration, p break } - tx, err := s.db.Begin() + tx, err := beginWrite(s.db, s.Backend()) if err != nil { log.Printf("[backfill] from_pubkey begin tx: %v", err) return diff --git a/cmd/ingestor/mqtt_opts_test.go b/cmd/ingestor/mqtt_opts_test.go index e8aceec45..8241154a2 100644 --- a/cmd/ingestor/mqtt_opts_test.go +++ b/cmd/ingestor/mqtt_opts_test.go @@ -1,8 +1,14 @@ package main import ( + "net" + "regexp" + "sync" "testing" "time" + + mqtt "github.com/eclipse/paho.mqtt.golang" + "github.com/eclipse/paho.mqtt.golang/packets" ) func TestBuildMQTTOpts_ReconnectSettings(t *testing.T) { @@ -45,6 +51,171 @@ func TestBuildMQTTOpts_Credentials(t *testing.T) { } } +// #2013: without SetClientID paho connects with a zero-length ClientID and +// the ingestor's identity depends on what the broker does with that. +func TestBuildMQTTOpts_ClientIDDefaultShape(t *testing.T) { + cases := []struct { + name string + source MQTTSource + want string + }{ + {"sanitized source name", MQTTSource{Broker: "tcp://broker:1883", Name: "local feed/1"}, `^corescope-local-feed-1-[0-9a-f]{6}$`}, + {"broker host without port", MQTTSource{Broker: "ssl://mqtt.example.com:8883"}, `^corescope-mqtt-example-com-[0-9a-f]{6}$`}, + {"no name and no broker host", MQTTSource{Broker: "tcp://:1883"}, `^corescope-[0-9a-f]{6}$`}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := buildMQTTOpts(tc.source).ClientID + if !regexp.MustCompile(tc.want).MatchString(got) { + t.Errorf("ClientID = %q, want match for %s", got, tc.want) + } + }) + } +} + +func TestBuildMQTTOpts_ClientIDUsesConfiguredValue(t *testing.T) { + opts := buildMQTTOpts(MQTTSource{Broker: "tcp://broker:1883", Name: "local", ClientID: "my-ingestor"}) + + if opts.ClientID != "my-ingestor" { + t.Errorf("ClientID = %q, want %q", opts.ClientID, "my-ingestor") + } +} + +func TestBuildMQTTOpts_ClientIDDiffersBetweenSources(t *testing.T) { + a := buildMQTTOpts(MQTTSource{Broker: "tcp://broker:1883", Name: "same"}) + b := buildMQTTOpts(MQTTSource{Broker: "tcp://broker:1883", Name: "same"}) + + if a.ClientID == b.ClientID { + t.Errorf("two unconfigured sources got the same ClientID %q", a.ClientID) + } +} + +// The ID is chosen once in buildMQTTOpts, so every CONNECT the client sends +// must carry it: the first connect, paho's auto-reconnect after the broker +// drops the socket, and the watchdog's force-reconnect on the same client. +func TestBuildMQTTOpts_ClientIDSurvivesReconnects(t *testing.T) { + const wait = 5 * time.Second + broker, connects, drop := fakeMQTTBroker(t) + + opts := buildMQTTOpts(MQTTSource{Broker: broker, Name: "fake broker"}) + up := make(chan struct{}, 8) + opts.SetOnConnectHandler(func(mqtt.Client) { + select { + case up <- struct{}{}: + default: + } + }) + client := mqtt.NewClient(opts) + t.Cleanup(func() { client.Disconnect(250) }) + + var ids []string + awaitConnect := func(step string) { + t.Helper() + select { + case id := <-connects: + ids = append(ids, id) + case <-time.After(wait): + t.Fatalf("%s: no CONNECT within %v (seen %q)", step, wait, ids) + } + select { + case <-up: + case <-time.After(wait): + t.Fatalf("%s: client not connected within %v", step, wait) + } + } + + if tok := client.Connect(); !tok.WaitTimeout(wait) || tok.Error() != nil { + t.Fatalf("connect: timed out or failed: %v", tok.Error()) + } + awaitConnect("first connect") + + drop() + awaitConnect("auto-reconnect") + + buildForceReconnectFn(client, "fake broker")() + awaitConnect("watchdog force-reconnect") + + for i, id := range ids { + if id == "" { + t.Errorf("CONNECT %d carried an empty ClientID (all: %q)", i+1, ids) + } else if id != opts.ClientID { + t.Errorf("CONNECT %d ClientID = %q, want %q (all: %q)", i+1, id, opts.ClientID, ids) + } + } +} + +// fakeMQTTBroker accepts connections on a loopback port, answers every +// CONNECT with an accepted CONNACK and reports its ClientID on connects. drop +// closes every connection accepted so far, as a broker restart would. +func fakeMQTTBroker(t *testing.T) (broker string, connects <-chan string, drop func()) { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + ids := make(chan string, 8) + done := make(chan struct{}) + var mu sync.Mutex + var conns []net.Conn + closed := false + + drop = func() { + mu.Lock() + defer mu.Unlock() + for _, c := range conns { + c.Close() + } + conns = nil + } + t.Cleanup(func() { + close(done) + ln.Close() + mu.Lock() + closed = true + mu.Unlock() + drop() + }) + + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + mu.Lock() + if closed { + mu.Unlock() + c.Close() + return + } + conns = append(conns, c) + mu.Unlock() + go func() { + for { + cp, err := packets.ReadPacket(c) + if err != nil { + return + } + p, ok := cp.(*packets.ConnectPacket) + if !ok { + continue + } + select { + case ids <- p.ClientIdentifier: + case <-done: + return + } + ack := packets.NewControlPacket(packets.Connack).(*packets.ConnackPacket) + if ack.Write(c) != nil { + return + } + } + }() + } + }() + return "tcp://" + ln.Addr().String(), ids, drop +} + func TestBuildMQTTOpts_TLS_InsecureSkipVerify(t *testing.T) { f := false source := MQTTSource{ diff --git a/cmd/ingestor/mqtt_watchdog_1749_test.go b/cmd/ingestor/mqtt_watchdog_1749_test.go index 105c18ad2..43e407829 100644 --- a/cmd/ingestor/mqtt_watchdog_1749_test.go +++ b/cmd/ingestor/mqtt_watchdog_1749_test.go @@ -58,15 +58,8 @@ func TestMQTTStallWatchdog_EscalateOnPersistentDisconnect_1749(t *testing.T) { t.Fatalf("setup: %v", err) } - tick := make(chan time.Time) - done := make(chan struct{}) - defer close(done) - - exited := make(chan struct{}) - go func() { - runLivenessWatchdogLoop(tick, done, threshold, func(args ...any) {}) - close(exited) - }() + tick, stopLoop := startWatchdogTestLoop(t, threshold, func(args ...any) {}) + defer stopLoop() // Feed ticks spanning > (multiplier × threshold) of wall clock so // the escalation path fires. We control the `now` parameter by @@ -114,10 +107,8 @@ func TestMQTTStallWatchdog_DisconnectedEscalationThrottled_1749(t *testing.T) { t.Fatalf("setup: %v", err) } - tick := make(chan time.Time) - done := make(chan struct{}) - defer close(done) - go runLivenessWatchdogLoop(tick, done, threshold, func(args ...any) {}) + tick, stopLoop := startWatchdogTestLoop(t, threshold, func(args ...any) {}) + defer stopLoop() base := time.Now() // Pre-stamp DisconnectedSinceUnix so that the first tick is @@ -259,10 +250,8 @@ func TestMQTTStallWatchdog_LastTickUnixExposed_1749(t *testing.T) { watchdogLastTickUnix.Store(before) }) - tick := make(chan time.Time) - done := make(chan struct{}) - defer close(done) - go runLivenessWatchdogLoop(tick, done, time.Minute, func(args ...any) {}) + tick, stopLoop := startWatchdogTestLoop(t, time.Minute, func(args ...any) {}) + defer stopLoop() stamp := time.Now().Add(48 * time.Hour) // guaranteed > before select { diff --git a/cmd/ingestor/mqtt_watchdog_1810_test.go b/cmd/ingestor/mqtt_watchdog_1810_test.go index aeefd4a0f..b68fa75d3 100644 --- a/cmd/ingestor/mqtt_watchdog_1810_test.go +++ b/cmd/ingestor/mqtt_watchdog_1810_test.go @@ -212,10 +212,8 @@ func TestWatchdog_EscalationWarnThrottled_1810(t *testing.T) { } } - tick := make(chan time.Time) - done := make(chan struct{}) - defer close(done) - go runLivenessWatchdogLoop(tick, done, threshold, emit) + tick, stopLoop := startWatchdogTestLoop(t, threshold, emit) + defer stopLoop() base := time.Now() // First tick: stamps DisconnectedSinceUnix (no escalation yet). diff --git a/cmd/ingestor/mqtt_watchdog_testhelpers_test.go b/cmd/ingestor/mqtt_watchdog_testhelpers_test.go index 348c2f2d1..1db9b99a5 100644 --- a/cmd/ingestor/mqtt_watchdog_testhelpers_test.go +++ b/cmd/ingestor/mqtt_watchdog_testhelpers_test.go @@ -1,6 +1,7 @@ package main import ( + "sync" "testing" "time" ) @@ -45,3 +46,31 @@ func sendTickOrFail(t *testing.T, tick chan<- time.Time, stamp time.Time, timeou t.Fatalf("%s: tick blocked after %s — loop dead?", label, timeout) } } + +// startWatchdogTestLoop is setupWatchdogTestLoop for the common case: a test +// that wants the loop running for its own duration and nothing more. The +// returned stop closes done AND waits for the loop goroutine to return. It is +// safe to call more than once. +// +// Waiting is the part that must not be skipped. Closing done only asks the +// loop to stop; the goroutine can still be inside a scan, and that scan walks +// the package-level livenessRegistry. The next test registers its own source +// there, so a loop that has not returned yet will process that source on a +// tick carrying the previous test's fabricated clock. +// +// That is measured, not theoretical. With four call sites closing done and +// walking away, TestMQTTStallWatchdog_DisconnectedEscalationThrottled_1749 +// failed 2 to 3 times per 20 runs of the watchdog tests and never once in 50 +// runs on its own. Debug tracing showed two loops reaching maybeForceReconnect +// for the same source with tick clocks 420s apart, both reading +// LastForceReconnectUnix as 0 before either wrote it, so the throttle the test +// asserts on let both through. +func startWatchdogTestLoop(t *testing.T, threshold time.Duration, emit func(...any)) (tick chan time.Time, stop func()) { + t.Helper() + tick, done, exited := setupWatchdogTestLoop(t, threshold, emit) + var once sync.Once + return tick, func() { + once.Do(func() { close(done) }) + <-exited + } +} diff --git a/cmd/ingestor/multibyte_persist.go b/cmd/ingestor/multibyte_persist.go index d9938da55..b24058b4f 100644 --- a/cmd/ingestor/multibyte_persist.go +++ b/cmd/ingestor/multibyte_persist.go @@ -76,7 +76,7 @@ func (s *Store) RunMultibyteCapPersist() (MultibyteCapPersistStats, error) { return stats, nil } - tx, err := s.db.Begin() + tx, err := beginWrite(s.db, s.Backend()) if err != nil { return stats, err } @@ -85,19 +85,19 @@ func (s *Store) RunMultibyteCapPersist() (MultibyteCapPersistStats, error) { // inactive_nodes. The pre-#1386 implementation issued one UPDATE // against each table per entry — 50% guaranteed-empty. We now // look up the table once, then issue the matching UPDATE. - stmtN, err := tx.Prepare(`UPDATE nodes SET multibyte_sup=?, multibyte_evidence=? WHERE public_key=?`) + stmtN, err := tx.Prepare(`UPDATE nodes SET multibyte_sup=` + s.parameter(1) + `, multibyte_evidence=` + s.parameter(2) + ` WHERE public_key=` + s.parameter(3)) if err != nil { return stats, err } defer stmtN.Close() - stmtI, err := tx.Prepare(`UPDATE inactive_nodes SET multibyte_sup=?, multibyte_evidence=? WHERE public_key=?`) + stmtI, err := tx.Prepare(`UPDATE inactive_nodes SET multibyte_sup=` + s.parameter(1) + `, multibyte_evidence=` + s.parameter(2) + ` WHERE public_key=` + s.parameter(3)) if err != nil { return stats, err } defer stmtI.Close() // Membership probe: one indexed PK lookup. Cheap; avoids the // guaranteed-miss second UPDATE. - stmtProbe, err := tx.Prepare(`SELECT 1 FROM nodes WHERE public_key=? LIMIT 1`) + stmtProbe, err := tx.Prepare(`SELECT 1 FROM nodes WHERE public_key=` + s.parameter(1) + ` LIMIT 1`) if err != nil { return stats, err } @@ -187,24 +187,9 @@ func containsCI(s, sub string) bool { // RunMultibyteCapPersist on legacy DBs that pre-date the // internal/dbschema migration (#1386). func (s *Store) hasMultibyteSupColumns() bool { - rows, err := s.db.Query(`PRAGMA table_info(nodes)`) - if err != nil { - return false - } - defer rows.Close() - for rows.Next() { - var cid int - var name, ctype string - var notnull, pk int - var dflt interface{} - if err := rows.Scan(&cid, &name, &ctype, ¬null, &dflt, &pk); err != nil { - return false - } - if name == "multibyte_sup" { - return true - } - } - return false + var found bool + _ = s.db.QueryRow(s.nativeSQL(`SELECT EXISTS(SELECT 1 FROM pragma_table_info('nodes') WHERE name='multibyte_sup')`, `SELECT EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema=current_schema() AND table_name='nodes' AND column_name='multibyte_sup')`)).Scan(&found) + return found } // multibyteStatusToInt mirrors the mapping the server used before relocation. diff --git a/cmd/ingestor/multibyte_persist_helpers_test.go b/cmd/ingestor/multibyte_persist_helpers_test.go index aee8078aa..bee01ce3a 100644 --- a/cmd/ingestor/multibyte_persist_helpers_test.go +++ b/cmd/ingestor/multibyte_persist_helpers_test.go @@ -5,15 +5,16 @@ import ( "database/sql" "log" "strings" + "sync" "testing" ) // captureLogs redirects the standard logger to a buffer for the // duration of the test and returns the buffer. Restores the previous // writer when the test ends. -func captureLogs(t *testing.T) *bytes.Buffer { +func captureLogs(t *testing.T) *syncBuffer { t.Helper() - buf := &bytes.Buffer{} + buf := &syncBuffer{} prevWriter := log.Writer() prevFlags := log.Flags() log.SetOutput(buf) @@ -24,31 +25,41 @@ func captureLogs(t *testing.T) *bytes.Buffer { return buf } +// syncBuffer is a bytes.Buffer that may be read while a background goroutine +// is still logging into it. log.Logger serialises its own writes, but the test +// reading buf.String() sits outside that lock, and RunAsyncMigration keeps +// logging after the call that started it returns. Without this the read and +// the write race, which is a real data race and not a test artefact: it can +// panic on a buffer grow, not merely trip -race. +type syncBuffer struct { + mu sync.Mutex + buf bytes.Buffer +} + +func (b *syncBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.Write(p) +} + +func (b *syncBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.String() +} + // logContains reports whether the captured log buffer contains substr // (case-insensitive). -func logContains(buf *bytes.Buffer, substr string) bool { +func logContains(buf *syncBuffer, substr string) bool { return strings.Contains(strings.ToLower(buf.String()), strings.ToLower(substr)) } // columnExists reports whether the named column exists on the table. func columnExists(t *testing.T, db *sql.DB, table, col string) bool { t.Helper() - rows, err := db.Query("PRAGMA table_info(" + table + ")") - if err != nil { - t.Fatalf("PRAGMA table_info(%s): %v", table, err) - } - defer rows.Close() - for rows.Next() { - var cid int - var name, ctype string - var notnull, pk int - var dfltValue sql.NullString - if err := rows.Scan(&cid, &name, &ctype, ¬null, &dfltValue, &pk); err != nil { - t.Fatalf("scan PRAGMA: %v", err) - } - if name == col { - return true - } + var found bool + if err := db.QueryRow(testNativeSQL(`SELECT EXISTS(SELECT 1 FROM pragma_table_info(?1) WHERE name=?2)`, `SELECT EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema=current_schema() AND table_name=$1 AND column_name=$2)`), table, col).Scan(&found); err != nil { + t.Fatal(err) } - return false + return found } diff --git a/cmd/ingestor/multibyte_persist_test.go b/cmd/ingestor/multibyte_persist_test.go index 065a2381c..b270e6c4f 100644 --- a/cmd/ingestor/multibyte_persist_test.go +++ b/cmd/ingestor/multibyte_persist_test.go @@ -23,7 +23,7 @@ import ( func TestRunMultibyteCapPersist_AppliesSnapshot(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "test.db") - store, err := OpenStore(dbPath) + store, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore: %v", err) } @@ -105,7 +105,7 @@ func TestRunMultibyteCapPersist_AppliesSnapshot(t *testing.T) { func TestRunMultibyteCapPersist_NoSnapshot_NoOp(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "test.db") - store, err := OpenStore(dbPath) + store, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore: %v", err) } @@ -135,7 +135,7 @@ func TestRunMultibyteCapPersist_RoundTrip(t *testing.T) { dbPath := filepath.Join(dir, "test.db") // --- Phase 1: open store, seed, persist snapshot --- - store, err := OpenStore(dbPath) + store, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore: %v", err) } @@ -172,7 +172,7 @@ func TestRunMultibyteCapPersist_RoundTrip(t *testing.T) { } // --- Phase 2: fresh Store, verify persisted state survived --- - store2, err := OpenStore(dbPath) + store2, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore (reopen): %v", err) } @@ -211,7 +211,7 @@ func TestRunMultibyteCapPersist_RoundTrip(t *testing.T) { func TestRunMultibyteCapPersist_MalformedSnapshot(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "test.db") - store, err := OpenStore(dbPath) + store, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore: %v", err) } @@ -257,7 +257,7 @@ func TestRunMultibyteCapPersist_MalformedSnapshot(t *testing.T) { func TestRunMultibyteCapPersist_MissingSchemaColumns(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "test.db") - store, err := OpenStore(dbPath) + store, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore: %v", err) } @@ -270,7 +270,7 @@ func TestRunMultibyteCapPersist_MissingSchemaColumns(t *testing.T) { `ALTER TABLE inactive_nodes DROP COLUMN multibyte_sup`, `ALTER TABLE inactive_nodes DROP COLUMN multibyte_evidence`, } { - if _, err := store.db.Exec(stmt); err != nil { + if _, err := testAdmin(t, store).Exec(stmt); err != nil { t.Fatalf("simulate legacy DB (%q): %v", stmt, err) } } @@ -315,7 +315,7 @@ func TestRunMultibyteCapPersist_MissingSchemaColumns(t *testing.T) { func TestRunMultibyteCapPersist_PreservesConfirmedOnUnknown(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "test.db") - store, err := OpenStore(dbPath) + store, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore: %v", err) } diff --git a/cmd/ingestor/neighbor_builder.go b/cmd/ingestor/neighbor_builder.go index 7a49ecc9a..01ba7369e 100644 --- a/cmd/ingestor/neighbor_builder.go +++ b/cmd/ingestor/neighbor_builder.go @@ -20,7 +20,7 @@ const NeighborEdgesBuilderInterval = 60 * time.Second // neighborBuilderMaxBatch caps how many observation rows a single // delta tick may process (#1339). With max_open_conns=1, an unbounded -// scan on a multi-million-row table holds the SQLite write lock for +// scan on a multi-million-row table holds the writer lock for // minutes and starves MQTT ingest. The cap keeps each tick bounded; // if a backlog accumulates, successive ticks drain it 50k rows at a // time without ever blocking ingest for long. @@ -197,9 +197,9 @@ func (s *Store) buildAndPersistNeighborEdges(trust *packetpath.TrustConfig) (int FROM observations o JOIN transmissions t ON t.id = o.transmission_id LEFT JOIN observers obs ON obs.rowid = o.observer_idx - WHERE o.timestamp > ? + WHERE o.timestamp > `+s.parameter(1)+` ORDER BY o.timestamp - LIMIT ?`, watermarkEpoch, neighborBuilderMaxBatch) + LIMIT `+s.parameter(2), watermarkEpoch, neighborBuilderMaxBatch) if err != nil { return 0, fmt.Errorf("scan observations: %w", err) } @@ -254,36 +254,58 @@ func (s *Store) buildAndPersistNeighborEdges(trust *packetpath.TrustConfig) (int return 0, nil } - // Wrap the whole edge-persist tx under writer-perf instrumentation - // (#1340). Slow neighbor-builder ticks (the #1339 root cause) now - // show up on /api/perf under component=neighbor_builder. - var inserted int - err = s.WriterTx("neighbor_builder", func(tx *sql.Tx) error { - stmt, err := tx.Prepare(`INSERT INTO neighbor_edges (node_a, node_b, count, last_seen) - VALUES (?, ?, 1, ?) - ON CONFLICT(node_a, node_b) DO UPDATE SET - count = count + 1, - last_seen = MAX(last_seen, excluded.last_seen)`) - if err != nil { - return fmt.Errorf("prepare: %w", err) + // Aggregate repeated pairs before taking the writer lock. Counts and the + // latest timestamp are additive; the working set is bounded by this scan's + // candidate slice. SQL batches cap parameters and server round trips. + type countedEdge struct { + a, b, ts string + count int + } + counts := make(map[[2]string]*countedEdge) + for _, e := range edges { + key := [2]string{e.a, e.b} + if current := counts[key]; current != nil { + current.count++ + if e.ts > current.ts { + current.ts = e.ts + } + } else { + counts[key] = &countedEdge{e.a, e.b, e.ts, 1} } - defer stmt.Close() - var firstErr error - for _, e := range edges { - if _, err := stmt.Exec(e.a, e.b, e.ts); err != nil && firstErr == nil { - firstErr = err + } + err = s.WriterTx("neighbor_builder", func(tx *sql.Tx) error { + const batchSize = 1000 + values := make([]string, 0, batchSize) + args := make([]any, 0, batchSize*4) + flush := func() error { + if len(values) == 0 { + return nil } + _, err := tx.Exec(`INSERT INTO neighbor_edges(node_a,node_b,count,last_seen) VALUES `+strings.Join(values, ",")+s.nativeSQL(` + ON CONFLICT(node_a,node_b) DO UPDATE SET count=neighbor_edges.count+excluded.count, + last_seen=CASE WHEN neighbor_edges.last_seen IS NULL OR excluded.last_seen IS NULL THEN NULL ELSE MAX(neighbor_edges.last_seen,excluded.last_seen) END`, ` + ON CONFLICT(node_a,node_b) DO UPDATE SET count=neighbor_edges.count+excluded.count, + last_seen=CASE WHEN neighbor_edges.last_seen IS NULL OR excluded.last_seen IS NULL THEN NULL ELSE GREATEST(neighbor_edges.last_seen,excluded.last_seen) END`), args...) + values = values[:0] + args = args[:0] + return err } - if firstErr != nil { - return fmt.Errorf("upsert: %w", firstErr) + for _, e := range counts { + n := len(args) + values = append(values, fmt.Sprintf("(%s,%s,%s,%s)", s.parameter(n+1), s.parameter(n+2), s.parameter(n+3), s.parameter(n+4))) + args = append(args, e.a, e.b, e.count, e.ts) + if len(values) == batchSize { + if err := flush(); err != nil { + return err + } + } } - inserted = len(edges) - return nil + return flush() }) if err != nil { return 0, err } - return inserted, nil + return len(edges), nil } // canonEdge orders the pair so node_a <= node_b (matches the existing diff --git a/cmd/ingestor/neighbor_builder_delta_test.go b/cmd/ingestor/neighbor_builder_delta_test.go index 4e530953e..a60792a5f 100644 --- a/cmd/ingestor/neighbor_builder_delta_test.go +++ b/cmd/ingestor/neighbor_builder_delta_test.go @@ -29,27 +29,27 @@ func TestNeighborEdgesBuilderDeltaScan(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "delta.db") - store, err := OpenStore(dbPath) + store, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore: %v", err) } defer store.Close() if _, err := store.db.Exec( - `INSERT INTO nodes (public_key, name) VALUES (?, ?), (?, ?)`, + `INSERT INTO nodes (public_key, name) VALUES ($1, $2), ($3, $4)`, "aaaaaaaaaa", "from-node", "bbbbbbbbbb", "first-hop", ); err != nil { t.Fatal(err) } if _, err := store.db.Exec( - `INSERT INTO observers (id, name) VALUES (?, ?)`, + `INSERT INTO observers (id, name) VALUES ($1, $2)`, "obs-1", "observer-1", ); err != nil { t.Fatal(err) } var obsRowid int64 - if err := store.db.QueryRow(`SELECT rowid FROM observers WHERE id = ?`, "obs-1").Scan(&obsRowid); err != nil { + if err := store.db.QueryRow(`SELECT rowid FROM observers WHERE id = $1`, "obs-1").Scan(&obsRowid); err != nil { t.Fatal(err) } @@ -58,34 +58,53 @@ func TestNeighborEdgesBuilderDeltaScan(t *testing.T) { const baselineStartTs int64 = 1735689600 // 2025-01-01 UTC baselineMaxTs := baselineStartTs + int64(baseline) - 1 - tx, err := store.db.Begin() - if err != nil { - t.Fatal(err) - } - txStmt, err := tx.Prepare(`INSERT INTO transmissions - (raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json, from_pubkey) - VALUES ('', ?, ?, 0, ?, 0, '{}', 'aaaaaaaaaa')`) - if err != nil { - t.Fatal(err) - } - obsStmt, err := tx.Prepare(`INSERT INTO observations - (transmission_id, observer_idx, path_json, timestamp) VALUES (?, ?, '["bb"]', ?)`) - if err != nil { - t.Fatal(err) - } - for i := 0; i < baseline; i++ { - res, err := txStmt.Exec(fmt.Sprintf("h%d", i), baselineStartTs+int64(i), payloadADVERT) - if err != nil { - t.Fatal(err) + // Seed the same 100K logical rows in one transaction without per-row RPCs. + seed := func(prefix string, start int64, count int) { + t.Helper() + if store.Backend() == "sqlite" { + tx, err := store.db.Begin() + if err != nil { + t.Fatal(err) + } + defer tx.Rollback() + parents, err := tx.Prepare(`INSERT INTO transmissions(raw_hex,hash,first_seen,route_type,payload_type,payload_version,decoded_json,from_pubkey) VALUES('',?1,?2,0,?3,0,'{}','aaaaaaaaaa')`) + if err != nil { + t.Fatal(err) + } + defer parents.Close() + children, err := tx.Prepare(`INSERT INTO observations(transmission_id,observer_idx,path_json,timestamp) VALUES(?1,?2,'["bb"]',?3)`) + if err != nil { + t.Fatal(err) + } + defer children.Close() + for i := 0; i < count; i++ { + stamp := start + int64(i) + res, err := parents.Exec(fmt.Sprint(prefix, i), time.Unix(stamp, 0).UTC().Format(time.RFC3339), payloadADVERT) + if err != nil { + t.Fatal(err) + } + id, _ := res.LastInsertId() + if _, err = children.Exec(id, obsRowid, stamp); err != nil { + t.Fatal(err) + } + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + return } - txID, _ := res.LastInsertId() - if _, err := obsStmt.Exec(txID, obsRowid, baselineStartTs+int64(i)); err != nil { + + _, err := store.db.Exec(`WITH inserted AS ( + INSERT INTO transmissions(raw_hex,hash,first_seen,route_type,payload_type,payload_version,decoded_json,from_pubkey) + SELECT '',$1::text||n::text,to_char(to_timestamp($2::bigint+n) AT TIME ZONE 'UTC','YYYY-MM-DD"T"HH24:MI:SS"Z"'),0,$3,0,'{}','aaaaaaaaaa' + FROM generate_series(0,$4::integer-1) n RETURNING id,hash + ) INSERT INTO observations(transmission_id,observer_idx,path_json,timestamp) + SELECT id,$5,'["bb"]',$2::bigint+substring(hash,2)::bigint FROM inserted`, prefix, start, payloadADVERT, count, obsRowid) + if err != nil { t.Fatal(err) } } - if err := tx.Commit(); err != nil { - t.Fatal(err) - } + seed("h", baselineStartTs, baseline) // Initial warm-up: drain to completion (StartNeighborEdgesBuilder // does the same — call directly so the test doesn't depend on the @@ -136,34 +155,7 @@ func TestNeighborEdgesBuilderDeltaScan(t *testing.T) { // than baselineMaxTs. const delta = 100 deltaStartTs := baselineMaxTs + 1 - tx2, err := store.db.Begin() - if err != nil { - t.Fatal(err) - } - txStmt2, err := tx2.Prepare(`INSERT INTO transmissions - (raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json, from_pubkey) - VALUES ('', ?, ?, 0, ?, 0, '{}', 'aaaaaaaaaa')`) - if err != nil { - t.Fatal(err) - } - obsStmt2, err := tx2.Prepare(`INSERT INTO observations - (transmission_id, observer_idx, path_json, timestamp) VALUES (?, ?, '["bb"]', ?)`) - if err != nil { - t.Fatal(err) - } - for i := 0; i < delta; i++ { - res, err := txStmt2.Exec(fmt.Sprintf("d%d", i), deltaStartTs+int64(i), payloadADVERT) - if err != nil { - t.Fatal(err) - } - txID, _ := res.LastInsertId() - if _, err := obsStmt2.Exec(txID, obsRowid, deltaStartTs+int64(i)); err != nil { - t.Fatal(err) - } - } - if err := tx2.Commit(); err != nil { - t.Fatal(err) - } + seed("d", deltaStartTs, delta) deltaStart := time.Now() n3, err := store.buildAndPersistNeighborEdges(trustAllPrefixes()) @@ -184,6 +176,8 @@ func TestNeighborEdgesBuilderDeltaScan(t *testing.T) { t.Fatalf("delta build of %d rows took %v; expected <500ms. (#1339)", delta, deltaDur) } + t.Logf("100K baseline: empty delta=%s; 100-row delta=%s", noopDur, deltaDur) + // Sanity: MAX(last_seen) advanced. var maxLastSeen2 string if err := store.db.QueryRow(`SELECT MAX(last_seen) FROM neighbor_edges`).Scan(&maxLastSeen2); err != nil { diff --git a/cmd/ingestor/neighbor_builder_test.go b/cmd/ingestor/neighbor_builder_test.go index 5cd777df9..c4dead36b 100644 --- a/cmd/ingestor/neighbor_builder_test.go +++ b/cmd/ingestor/neighbor_builder_test.go @@ -21,7 +21,7 @@ func TestNeighborEdgesBuilderUpsertsFromObservations(t *testing.T) { // Open via the ingestor's normal opener so applySchema and // dbschema.Apply both run (the builder requires neighbor_edges + // observers.iata etc.). - store, err := OpenStore(dbPath) + store, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore: %v", err) } @@ -29,7 +29,7 @@ func TestNeighborEdgesBuilderUpsertsFromObservations(t *testing.T) { // Seed two nodes whose pubkey prefixes will be used as hops. if _, err := store.db.Exec( - `INSERT INTO nodes (public_key, name) VALUES (?, ?), (?, ?)`, + `INSERT INTO nodes (public_key, name) VALUES ($1, $2), ($3, $4)`, "aaaaaaaaaa", "from-node", "bbbbbbbbbb", "first-hop", ); err != nil { @@ -38,31 +38,31 @@ func TestNeighborEdgesBuilderUpsertsFromObservations(t *testing.T) { // Seed one observer. if _, err := store.db.Exec( - `INSERT INTO observers (id, name) VALUES (?, ?)`, + `INSERT INTO observers (id, name) VALUES ($1, $2)`, "obs-1", "observer-1", ); err != nil { t.Fatal(err) } var obsRowid int64 - if err := store.db.QueryRow(`SELECT rowid FROM observers WHERE id = ?`, "obs-1").Scan(&obsRowid); err != nil { + if err := store.db.QueryRow(`SELECT rowid FROM observers WHERE id = $1`, "obs-1").Scan(&obsRowid); err != nil { t.Fatal(err) } // Insert one ADVERT transmission with from_pubkey = aaaaa… - res, err := store.db.Exec( + var txID int64 + err = store.db.QueryRow( `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json, from_pubkey) - VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) RETURNING id`, "", "h1", "2026-01-01T00:00:00Z", 0, payloadADVERT, 0, "{}", "aaaaaaaaaa", - ) + ).Scan(&txID) if err != nil { t.Fatal(err) } - txID, _ := res.LastInsertId() // Insert one observation whose path[0] = "bb" (2-hex prefix unique // to bbbbb… in the nodes table). Expected edge: a↔b. if _, err := store.db.Exec( - `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp) VALUES (?, ?, ?, ?)`, + `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp) VALUES ($1, $2, $3, $4)`, txID, obsRowid, `["bb"]`, int64(1735689600), ); err != nil { t.Fatal(err) @@ -77,7 +77,7 @@ func TestNeighborEdgesBuilderUpsertsFromObservations(t *testing.T) { } var got int - if err := store.db.QueryRow(`SELECT COUNT(*) FROM neighbor_edges WHERE node_a = ? AND node_b = ?`, "aaaaaaaaaa", "bbbbbbbbbb").Scan(&got); err != nil { + if err := store.db.QueryRow(`SELECT COUNT(*) FROM neighbor_edges WHERE node_a = $1 AND node_b = $2`, "aaaaaaaaaa", "bbbbbbbbbb").Scan(&got); err != nil { t.Fatal(err) } if got != 1 { @@ -97,14 +97,14 @@ func TestNeighborEdgesBuilderUpsertsFromAnonReqEphemeralPubKey(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "build.db") - store, err := OpenStore(dbPath) + store, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore: %v", err) } defer store.Close() if _, err := store.db.Exec( - `INSERT INTO nodes (public_key, name) VALUES (?, ?), (?, ?)`, + `INSERT INTO nodes (public_key, name) VALUES ($1, $2), ($3, $4)`, "aaaaaaaaaa", "sender", "bbbbbbbbbb", "first-hop", ); err != nil { @@ -112,31 +112,31 @@ func TestNeighborEdgesBuilderUpsertsFromAnonReqEphemeralPubKey(t *testing.T) { } if _, err := store.db.Exec( - `INSERT INTO observers (id, name) VALUES (?, ?)`, + `INSERT INTO observers (id, name) VALUES ($1, $2)`, "obs-1", "observer-1", ); err != nil { t.Fatal(err) } var obsRowid int64 - if err := store.db.QueryRow(`SELECT rowid FROM observers WHERE id = ?`, "obs-1").Scan(&obsRowid); err != nil { + if err := store.db.QueryRow(`SELECT rowid FROM observers WHERE id = $1`, "obs-1").Scan(&obsRowid); err != nil { t.Fatal(err) } // ANON_REQ transmission: from_pubkey left NULL (as real ingest does — // only ADVERT populates it at write time), sender identity carried in // decoded_json.ephemeralPubKey instead. - res, err := store.db.Exec( + var txID int64 + err = store.db.QueryRow( `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) - VALUES (?, ?, ?, ?, ?, ?, ?)`, + VALUES ($1, $2, $3, $4, $5, $6, $7) RETURNING id`, "", "h2", "2026-01-01T00:00:00Z", 0, payloadAnonReq, 0, `{"ephemeralPubKey":"aaaaaaaaaa"}`, - ) + ).Scan(&txID) if err != nil { t.Fatal(err) } - txID, _ := res.LastInsertId() if _, err := store.db.Exec( - `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp) VALUES (?, ?, ?, ?)`, + `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp) VALUES ($1, $2, $3, $4)`, txID, obsRowid, `["bb"]`, int64(1735689600), ); err != nil { t.Fatal(err) @@ -151,7 +151,7 @@ func TestNeighborEdgesBuilderUpsertsFromAnonReqEphemeralPubKey(t *testing.T) { } var got int - if err := store.db.QueryRow(`SELECT COUNT(*) FROM neighbor_edges WHERE node_a = ? AND node_b = ?`, "aaaaaaaaaa", "bbbbbbbbbb").Scan(&got); err != nil { + if err := store.db.QueryRow(`SELECT COUNT(*) FROM neighbor_edges WHERE node_a = $1 AND node_b = $2`, "aaaaaaaaaa", "bbbbbbbbbb").Scan(&got); err != nil { t.Fatal(err) } if got != 1 { @@ -169,14 +169,14 @@ func TestNeighborEdgesBuilderExcludesOtherNonAdvertTypes(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "build.db") - store, err := OpenStore(dbPath) + store, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore: %v", err) } defer store.Close() if _, err := store.db.Exec( - `INSERT INTO nodes (public_key, name) VALUES (?, ?), (?, ?)`, + `INSERT INTO nodes (public_key, name) VALUES ($1, $2), ($3, $4)`, "aaaaaaaaaa", "sender", "bbbbbbbbbb", "first-hop", ); err != nil { @@ -184,29 +184,29 @@ func TestNeighborEdgesBuilderExcludesOtherNonAdvertTypes(t *testing.T) { } if _, err := store.db.Exec( - `INSERT INTO observers (id, name) VALUES (?, ?)`, + `INSERT INTO observers (id, name) VALUES ($1, $2)`, "obs-1", "observer-1", ); err != nil { t.Fatal(err) } var obsRowid int64 - if err := store.db.QueryRow(`SELECT rowid FROM observers WHERE id = ?`, "obs-1").Scan(&obsRowid); err != nil { + if err := store.db.QueryRow(`SELECT rowid FROM observers WHERE id = $1`, "obs-1").Scan(&obsRowid); err != nil { t.Fatal(err) } const payloadREQ = 2 - res, err := store.db.Exec( + var txID int64 + err = store.db.QueryRow( `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json, from_pubkey) - VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) RETURNING id`, "", "h3", "2026-01-01T00:00:00Z", 0, payloadREQ, 0, "{}", "aaaaaaaaaa", - ) + ).Scan(&txID) if err != nil { t.Fatal(err) } - txID, _ := res.LastInsertId() if _, err := store.db.Exec( - `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp) VALUES (?, ?, ?, ?)`, + `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp) VALUES ($1, $2, $3, $4)`, txID, obsRowid, `["bb"]`, int64(1735689600), ); err != nil { t.Fatal(err) @@ -217,7 +217,7 @@ func TestNeighborEdgesBuilderExcludesOtherNonAdvertTypes(t *testing.T) { } var got int - if err := store.db.QueryRow(`SELECT COUNT(*) FROM neighbor_edges WHERE node_a = ? AND node_b = ?`, "aaaaaaaaaa", "bbbbbbbbbb").Scan(&got); err != nil { + if err := store.db.QueryRow(`SELECT COUNT(*) FROM neighbor_edges WHERE node_a = $1 AND node_b = $2`, "aaaaaaaaaa", "bbbbbbbbbb").Scan(&got); err != nil { t.Fatal(err) } if got != 0 { @@ -240,30 +240,30 @@ func trustAllPrefixes() *packetpath.TrustConfig { func seedTrustFixture(t *testing.T, store *Store, hop string) { t.Helper() if _, err := store.db.Exec( - `INSERT INTO nodes (public_key, name) VALUES (?, ?), (?, ?)`, + `INSERT INTO nodes (public_key, name) VALUES ($1, $2), ($3, $4)`, "aaaaaaaaaa", "from-node", "bbbbbbbbbb", "first-hop", ); err != nil { t.Fatal(err) } - if _, err := store.db.Exec(`INSERT INTO observers (id, name) VALUES (?, ?)`, "obs-1", "observer-1"); err != nil { + if _, err := store.db.Exec(`INSERT INTO observers (id, name) VALUES ($1, $2)`, "obs-1", "observer-1"); err != nil { t.Fatal(err) } var obsRowid int64 - if err := store.db.QueryRow(`SELECT rowid FROM observers WHERE id = ?`, "obs-1").Scan(&obsRowid); err != nil { + if err := store.db.QueryRow(`SELECT rowid FROM observers WHERE id = $1`, "obs-1").Scan(&obsRowid); err != nil { t.Fatal(err) } - res, err := store.db.Exec( + var txID int64 + err := store.db.QueryRow( `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json, from_pubkey) - VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) RETURNING id`, "", "h1", "2026-01-01T00:00:00Z", 0, payloadADVERT, 0, "{}", "aaaaaaaaaa", - ) + ).Scan(&txID) if err != nil { t.Fatal(err) } - txID, _ := res.LastInsertId() if _, err := store.db.Exec( - `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp) VALUES (?, ?, ?, ?)`, + `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp) VALUES ($1, $2, $3, $4)`, txID, obsRowid, `["`+hop+`"]`, int64(1735689600), ); err != nil { t.Fatal(err) @@ -279,7 +279,7 @@ func seedTrustFixture(t *testing.T, store *Store, hop string) { // one byte) a later-joining repeater sharing that byte turns today's // "unique" resolution into a wrong edge. func TestNeighborEdgesBuilderPathTrustExcludesOneByte(t *testing.T) { - store, err := OpenStore(filepath.Join(t.TempDir(), "trust1.db")) + store, err := openPostgresTestStore(t, filepath.Join(t.TempDir(), "trust1.db")) if err != nil { t.Fatalf("OpenStore: %v", err) } @@ -297,7 +297,7 @@ func TestNeighborEdgesBuilderPathTrustExcludesOneByte(t *testing.T) { var got int if err := store.db.QueryRow( - `SELECT COUNT(*) FROM neighbor_edges WHERE node_a = ? AND node_b = ?`, + `SELECT COUNT(*) FROM neighbor_edges WHERE node_a = $1 AND node_b = $2`, "aaaaaaaaaa", "bbbbbbbbbb").Scan(&got); err != nil { t.Fatal(err) } @@ -311,7 +311,7 @@ func TestNeighborEdgesBuilderPathTrustExcludesOneByte(t *testing.T) { // so the threshold narrows the evidence base rather than disabling the // builder. func TestNeighborEdgesBuilderPathTrustAllowsTwoByte(t *testing.T) { - store, err := OpenStore(filepath.Join(t.TempDir(), "trust2.db")) + store, err := openPostgresTestStore(t, filepath.Join(t.TempDir(), "trust2.db")) if err != nil { t.Fatalf("OpenStore: %v", err) } @@ -324,7 +324,7 @@ func TestNeighborEdgesBuilderPathTrustAllowsTwoByte(t *testing.T) { var got int if err := store.db.QueryRow( - `SELECT COUNT(*) FROM neighbor_edges WHERE node_a = ? AND node_b = ?`, + `SELECT COUNT(*) FROM neighbor_edges WHERE node_a = $1 AND node_b = $2`, "aaaaaaaaaa", "bbbbbbbbbb").Scan(&got); err != nil { t.Fatal(err) } diff --git a/cmd/ingestor/neighbors_guard.go b/cmd/ingestor/neighbors_guard.go new file mode 100644 index 000000000..3142074ba --- /dev/null +++ b/cmd/ingestor/neighbors_guard.go @@ -0,0 +1,33 @@ +package main + +import "time" + +// Limits on the unauthenticated /neighbors report (#1865). Any MQTT publisher +// can send one, so the values it carries are bounded before they reach the +// nodes table. +const ( + // maxConfiguredScopeLen caps the normalised comma-separated scope list. + // Real repeaters carry a handful of short region names. + maxConfiguredScopeLen = 256 + // maxReportFuture is how far ahead of our clock a report may be stamped + // and still be accepted. Allows ordinary clock skew, rejects a timestamp + // chosen to win last-write-wins forever. + maxReportFuture = 5 * time.Minute +) + +// reportNow is swapped in tests. +var reportNow = time.Now + +// reportTooFarInFuture reports whether a canonical RFC3339 timestamp (the +// output of normalizeReportTS) is more than maxReportFuture ahead of now. +// Empty or unparseable input is not "in the future". +func reportTooFarInFuture(canonical string) bool { + if canonical == "" { + return false + } + t, err := time.Parse(time.RFC3339, canonical) + if err != nil { + return false + } + return t.After(reportNow().Add(maxReportFuture)) +} diff --git a/cmd/ingestor/neighbors_guard_test.go b/cmd/ingestor/neighbors_guard_test.go new file mode 100644 index 000000000..77cf0db97 --- /dev/null +++ b/cmd/ingestor/neighbors_guard_test.go @@ -0,0 +1,94 @@ +package main + +import ( + "strings" + "testing" + "time" +) + +// A /neighbors report can come from any MQTT publisher, so the values it +// carries must be bounded. These tests cover the three guards: a timestamp +// far in the future (which would lock last-write-wins), a non-hex pubkey, and +// an oversized scope list. + +func TestNeighborsReportFutureTimestampIsDropped(t *testing.T) { + store := openNeighborsStore(t) + pk := "ee00000000000000000000000000000000000000000000000000000000000001" + seedNode(t, store, pk) + + fixed := time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC) + reportNow = func() time.Time { return fixed } + t.Cleanup(func() { reportNow = time.Now }) + + // A report stamped a year ahead must not be written... + if err := store.UpdateNodeConfiguredScope(pk, "evil", "2027-10-07T12:00:00Z"); err != nil { + t.Fatal(err) + } + if sc, _ := configuredScope(t, store, pk); sc.Valid && sc.String != "" { + t.Fatalf("future-stamped report was stored: %q", sc.String) + } + // ...and a genuine report with a normal timestamp must still land after it. + if err := store.UpdateNodeConfiguredScope(pk, "eu", "2026-10-07T11:59:00Z"); err != nil { + t.Fatal(err) + } + if sc, _ := configuredScope(t, store, pk); sc.String != "#eu" { + t.Fatalf("genuine report blocked: configured_scope = %q, want '#eu'", sc.String) + } + // Ordinary clock skew (inside maxReportFuture) is still accepted. + if err := store.UpdateNodeConfiguredScope(pk, "de", "2026-10-07T12:03:00Z"); err != nil { + t.Fatal(err) + } + if sc, _ := configuredScope(t, store, pk); sc.String != "#de" { + t.Fatalf("slightly-ahead report rejected: configured_scope = %q, want '#de'", sc.String) + } +} + +func TestNeighborsReportIgnoresNonHexPubkeys(t *testing.T) { + store := openNeighborsStore(t) + pk := "ee00000000000000000000000000000000000000000000000000000000000002" + seedNode(t, store, pk) + + msg := map[string]interface{}{ + "timestamp": "2026-10-06T12:00:00Z", + "origin_id": "not-a-pubkey", + "self": map[string]interface{}{"scopes": "eu"}, + "neighbors": []interface{}{ + map[string]interface{}{"pubkey": "../etc/passwd", "status": "responded", "scopes": "eu"}, + map[string]interface{}{"pubkey": strings.ToUpper(pk), "status": "responded", "scopes": "dk"}, + }, + } + handleNeighborsReport(store, "test", "not-a-pubkey-either", msg) + + // The valid neighbor was written; nothing errored on the junk keys. + if sc, _ := configuredScope(t, store, pk); sc.String != "#dk" { + t.Fatalf("valid neighbor not written: %q", sc.String) + } + var n int + if err := store.db.QueryRow(`SELECT COUNT(*) FROM nodes WHERE configured_scope IS NOT NULL AND configured_scope != ''`).Scan(&n); err != nil { + t.Fatal(err) + } + if n != 1 { + t.Fatalf("expected exactly 1 node with a configured scope, got %d", n) + } +} + +func TestNeighborsReportOversizedScopeIsDropped(t *testing.T) { + store := openNeighborsStore(t) + pk := "ee00000000000000000000000000000000000000000000000000000000000003" + seedNode(t, store, pk) + + huge := strings.Repeat("region,", 200) // ~1.6 KB after normalisation + if err := store.UpdateNodeConfiguredScope(pk, huge, "2026-10-06T12:00:00Z"); err != nil { + t.Fatal(err) + } + if sc, _ := configuredScope(t, store, pk); sc.Valid && sc.String != "" { + t.Fatalf("oversized scope list was stored (%d bytes)", len(sc.String)) + } + // A normal list is unaffected. + if err := store.UpdateNodeConfiguredScope(pk, "eu,dk,dk-aarhus", "2026-10-06T12:00:00Z"); err != nil { + t.Fatal(err) + } + if sc, _ := configuredScope(t, store, pk); sc.String != "#eu,#dk,#dk-aarhus" { + t.Fatalf("normal scope list mangled: %q", sc.String) + } +} diff --git a/cmd/ingestor/observer_fields.go b/cmd/ingestor/observer_fields.go new file mode 100644 index 000000000..bb8195043 --- /dev/null +++ b/cmd/ingestor/observer_fields.go @@ -0,0 +1,24 @@ +package main + +import "unicode/utf8" + +// Observer-supplied strings come straight from the MQTT topic (id, IATA) or +// the status JSON (origin/name, model, firmware, client version, radio). Any +// publisher controls them and nothing upstream bounds their length, so cap +// them before they reach the observers table. Values are truncated, not +// dropped, so a legitimate observer with a long name still shows up. +const ( + maxObserverIDLen = 128 // ids are 64-hex pubkeys in practice + maxObserverTextLen = 128 // name, model, firmware, client version, radio + maxObserverIATALen = 16 // region codes are 3–8 chars +) + +// clampObserverField strips control characters and truncates to max runes. +func clampObserverField(s string, max int) string { + s = sanitizeName(s) + if utf8.RuneCountInString(s) <= max { + return s + } + r := []rune(s) + return string(r[:max]) +} diff --git a/cmd/ingestor/observer_fields_test.go b/cmd/ingestor/observer_fields_test.go new file mode 100644 index 000000000..eb23fcf52 --- /dev/null +++ b/cmd/ingestor/observer_fields_test.go @@ -0,0 +1,47 @@ +package main + +import ( + "strings" + "testing" + "unicode/utf8" +) + +func TestClampObserverField(t *testing.T) { + if got := clampObserverField("Observer One", 128); got != "Observer One" { + t.Fatalf("short value changed: %q", got) + } + long := strings.Repeat("x", 5000) + if got := clampObserverField(long, 128); utf8.RuneCountInString(got) != 128 { + t.Fatalf("long value not truncated to 128 runes: %d", utf8.RuneCountInString(got)) + } + // Truncation counts runes, not bytes, so a multi-byte name is not cut mid-character. + multi := strings.Repeat("é", 200) + got := clampObserverField(multi, 128) + if utf8.RuneCountInString(got) != 128 || !utf8.ValidString(got) { + t.Fatalf("multi-byte truncation broke the string: %d runes valid=%v", utf8.RuneCountInString(got), utf8.ValidString(got)) + } + if got := clampObserverField("a\x00b\x1fc", 128); got != "abc" { + t.Fatalf("control characters not stripped: %q", got) + } +} + +func TestExtractObserverMetaCapsStrings(t *testing.T) { + long := strings.Repeat("m", 1000) + meta := extractObserverMeta(map[string]interface{}{ + "model": long, + "firmware": long, + "client_version": long, + "radio": long, + }) + if meta == nil { + t.Fatal("meta nil") + } + for name, v := range map[string]*string{"model": meta.Model, "firmware": meta.Firmware, "client_version": meta.ClientVersion, "radio": meta.Radio} { + if v == nil { + t.Fatalf("%s not set", name) + } + if n := utf8.RuneCountInString(*v); n != maxObserverTextLen { + t.Fatalf("%s not capped: %d runes", name, n) + } + } +} diff --git a/cmd/ingestor/observer_lastseen_1465_test.go b/cmd/ingestor/observer_lastseen_1465_test.go index dedd7ab76..9c3809b4f 100644 --- a/cmd/ingestor/observer_lastseen_1465_test.go +++ b/cmd/ingestor/observer_lastseen_1465_test.go @@ -31,7 +31,7 @@ func TestStatusMessage_ObserverLastSeen_AlwaysIngestTime_PastEnvelope_1465(t *te after := time.Now().Unix() var lastSeen string - if err := store.db.QueryRow(`SELECT last_seen FROM observers WHERE id = ?`, "obs-past").Scan(&lastSeen); err != nil { + if err := store.db.QueryRow(`SELECT last_seen FROM observers WHERE id = $1`, "obs-past").Scan(&lastSeen); err != nil { t.Fatalf("scan last_seen: %v", err) } ls, err := time.Parse(time.RFC3339, lastSeen) @@ -61,7 +61,7 @@ func TestStatusMessage_ObserverLastSeen_AlwaysIngestTime_FutureEnvelope_1465(t * after := time.Now().Unix() var lastSeen string - if err := store.db.QueryRow(`SELECT last_seen FROM observers WHERE id = ?`, "obs-future").Scan(&lastSeen); err != nil { + if err := store.db.QueryRow(`SELECT last_seen FROM observers WHERE id = $1`, "obs-future").Scan(&lastSeen); err != nil { t.Fatalf("scan last_seen: %v", err) } ls, err := time.Parse(time.RFC3339, lastSeen) @@ -94,7 +94,7 @@ func TestPacketMessage_ObserverLastSeen_AlwaysIngestTime_PastEnvelope_1465(t *te after := time.Now().Unix() var lastSeen string - if err := store.db.QueryRow(`SELECT last_seen FROM observers WHERE id = ?`, "obs-pkt").Scan(&lastSeen); err != nil { + if err := store.db.QueryRow(`SELECT last_seen FROM observers WHERE id = $1`, "obs-pkt").Scan(&lastSeen); err != nil { t.Fatalf("scan last_seen: %v", err) } ls, err := time.Parse(time.RFC3339, lastSeen) diff --git a/cmd/ingestor/observer_purge_test.go b/cmd/ingestor/observer_purge_test.go index 8a3f6a6a6..5646ba7a2 100644 --- a/cmd/ingestor/observer_purge_test.go +++ b/cmd/ingestor/observer_purge_test.go @@ -13,11 +13,11 @@ func staleObserver(t *testing.T, s *Store, id string) int64 { t.Fatal(err) } old := time.Now().UTC().AddDate(0, 0, -90).Format(time.RFC3339) - if _, err := s.db.Exec(`UPDATE observers SET last_seen = ?, inactive = 1 WHERE id = ?`, old, id); err != nil { + if _, err := s.db.Exec(`UPDATE observers SET last_seen = $1, inactive = 1 WHERE id = $2`, old, id); err != nil { t.Fatal(err) } var rowid int64 - if err := s.db.QueryRow(`SELECT rowid FROM observers WHERE id = ?`, id).Scan(&rowid); err != nil { + if err := s.db.QueryRow(`SELECT rowid FROM observers WHERE id = $1`, id).Scan(&rowid); err != nil { t.Fatal(err) } return rowid @@ -26,7 +26,7 @@ func staleObserver(t *testing.T, s *Store, id string) int64 { func observerExists(t *testing.T, s *Store, id string) bool { t.Helper() var n int - if err := s.db.QueryRow(`SELECT COUNT(*) FROM observers WHERE id = ?`, id).Scan(&n); err != nil { + if err := s.db.QueryRow(`SELECT COUNT(*) FROM observers WHERE id = $1`, id).Scan(&n); err != nil { t.Fatal(err) } return n > 0 @@ -56,13 +56,13 @@ func TestPurgeStaleObserversKeepsObserverWithObservations(t *testing.T) { rowid := staleObserver(t, store, "obs-referenced") if _, err := store.db.Exec( - `INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (1, 'AA', 'h1', ?)`, + `INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (1, 'AA', 'h1', $1)`, time.Now().UTC().Format(time.RFC3339), ); err != nil { t.Fatal(err) } if _, err := store.db.Exec( - `INSERT INTO observations (transmission_id, observer_idx, timestamp) VALUES (1, ?, 0)`, rowid, + `INSERT INTO observations (transmission_id, observer_idx, timestamp) VALUES (1, $1, 0)`, rowid, ); err != nil { t.Fatal(err) } @@ -95,7 +95,7 @@ func TestPurgeStaleObserversKeepsObserverWithMetrics(t *testing.T) { staleObserver(t, store, "obs-metrics") if _, err := store.db.Exec( - `INSERT INTO observer_metrics (observer_id, timestamp) VALUES ('obs-metrics', ?)`, + `INSERT INTO observer_metrics (observer_id, timestamp) VALUES ('obs-metrics', $1)`, time.Now().UTC().Format(time.RFC3339), ); err != nil { t.Fatal(err) @@ -143,7 +143,7 @@ func TestPurgeStaleObserversKeepsActiveObserver(t *testing.T) { t.Fatal(err) } old := time.Now().UTC().AddDate(0, 0, -90).Format(time.RFC3339) - if _, err := store.db.Exec(`UPDATE observers SET last_seen = ? WHERE id = ?`, old, "obs-active"); err != nil { + if _, err := store.db.Exec(`UPDATE observers SET last_seen = $1 WHERE id = $2`, old, "obs-active"); err != nil { t.Fatal(err) } @@ -166,7 +166,7 @@ func TestPurgeStaleObserversKeepsObserverInsideWindow(t *testing.T) { } recent := time.Now().UTC().AddDate(0, 0, -20).Format(time.RFC3339) if _, err := store.db.Exec( - `UPDATE observers SET last_seen = ?, inactive = 1 WHERE id = ?`, recent, "obs-recent", + `UPDATE observers SET last_seen = $1, inactive = 1 WHERE id = $2`, recent, "obs-recent", ); err != nil { t.Fatal(err) } diff --git a/cmd/ingestor/observer_status_meta_test.go b/cmd/ingestor/observer_status_meta_test.go index a08f3db33..47a5ceeb8 100644 --- a/cmd/ingestor/observer_status_meta_test.go +++ b/cmd/ingestor/observer_status_meta_test.go @@ -53,7 +53,7 @@ func TestStatusMessageMetadataPersisted_Issue1044(t *testing.T) { } // Now drive the meta through UpsertObserver and verify the row. - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } diff --git a/cmd/ingestor/postgres_config_test.go b/cmd/ingestor/postgres_config_test.go new file mode 100644 index 000000000..abaec5c01 --- /dev/null +++ b/cmd/ingestor/postgres_config_test.go @@ -0,0 +1,64 @@ +package main + +import ( + "github.com/meshcore-analyzer/dbconfig" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestPostgresConfigurationPrecedence(t *testing.T) { + for _, tc := range []struct{ name, generic, writer, want string }{ + {"config", "", "", "postgres://configured@localhost/telemetry"}, + {"generic", "postgres://generic@localhost/telemetry", "", "postgres://generic@localhost/telemetry"}, + {"writer", "postgres://generic@localhost/telemetry", "postgres://writer@localhost/telemetry", "postgres://writer@localhost/telemetry"}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Setenv("CORESCOPE_DATABASE_URL", tc.generic) + t.Setenv("CORESCOPE_WRITER_DATABASE_URL", tc.writer) + t.Setenv("CORESCOPE_STATE_DIR", "private-state") + t.Setenv("CORESCOPE_APPROVED_CHANNELS_DATABASE_URL", "postgres://channels@localhost/accounts") + path := filepath.Join(t.TempDir(), "config.json") + if err := os.WriteFile(path, []byte(`{"databaseURL":"postgres://configured@localhost/telemetry","dbPath":"legacy.db"}`), 0600); err != nil { + t.Fatal(err) + } + cfg, err := LoadConfig(path) + if err != nil { + t.Fatal(err) + } + if cfg.DatabaseURL != tc.want || cfg.StateDir != "private-state" || cfg.ApprovedChannelsURL() != "postgres://channels@localhost/accounts" { + t.Fatal("PostgreSQL overrides did not resolve correctly") + } + }) + } +} + +func TestSelectedPostgresCannotFallBackToSQLiteFiles(t *testing.T) { + path := filepath.Join(t.TempDir(), "missing", "database.db") + if s, err := OpenStoreStorage(dbconfig.Storage{Backend: dbconfig.Postgres, DBPath: path}, nil); err == nil { + s.Close() + t.Fatal("missing PostgreSQL URL fell back to SQLite") + } + if _, err := os.Stat(filepath.Dir(path)); !os.IsNotExist(err) { + t.Fatal("legacy path created a directory") + } +} + +func TestOpenStoreConnectionErrorDoesNotLeakCredentials(t *testing.T) { + _, err := OpenStore("postgres://reader:do-not-log-this@localhost:1/test?sslmode=disable") + if err == nil { + t.Fatal("closed port unexpectedly connected") + } + if strings.Contains(err.Error(), "do-not-log-this") || strings.Contains(err.Error(), "postgres://") { + t.Fatal("connection error leaked credentials") + } +} + +func TestStatsDefaultUsesStateDirectory(t *testing.T) { + t.Setenv("CORESCOPE_INGESTOR_STATS", "") + t.Setenv("CORESCOPE_STATE_DIR", "private-state") + if got := statsFilePath(); got != filepath.Join("private-state", "ingestor-stats.json") { + t.Fatalf("stats path %q", got) + } +} diff --git a/cmd/ingestor/postgres_test.go b/cmd/ingestor/postgres_test.go new file mode 100644 index 000000000..dcb997fff --- /dev/null +++ b/cmd/ingestor/postgres_test.go @@ -0,0 +1,189 @@ +package main + +import ( + "context" + "database/sql" + "net/url" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/pgutil" + "github.com/meshcore-analyzer/pgutil/pgtest" +) + +var testDatabases sync.Map + +func openPostgresTestStore(t testing.TB, key string) (*Store, error) { + return openPostgresTestStoreInterval(t, key, 300) +} +func openPostgresTestStoreInterval(t testing.TB, key string, interval int) (*Store, error) { + if testBackend(t) == dbconfig.SQLite { + return OpenStoreWithInterval(key, interval) + } + stateDir := filepath.Dir(key) + if strings.HasPrefix(key, "postgres") { + stateDir = t.TempDir() + } + return OpenStoreWithState(testPostgresURL(t, key), stateDir, interval) +} + +// A historical fixture path is only a test key. Runtime OpenStore accepts +// PostgreSQL URLs exclusively, with a real restricted writer credential. +func testPostgresURL(t testing.TB, key string) string { + if testBackend(t) != dbconfig.Postgres { + t.Skip("PostgreSQL matrix not selected") + } + t.Helper() + if strings.HasPrefix(key, "postgres://") || strings.HasPrefix(key, "postgresql://") { + return key + } + cacheKey := t.Name() + "|" + key + if v, ok := testDatabases.Load(cacheKey); ok { + return v.(string) + } + owner := pgtest.NewSchema(t) + db, err := pgutil.Open(owner, false) + if err != nil { + t.Fatal(err) + } + if err = dbschema.Apply(db, nil); err != nil { + db.Close() + t.Fatal(err) + } + u, _ := url.Parse(owner) + schema := u.Query().Get("search_path") + role := schema + "_writer" + for _, stmt := range []string{`CREATE ROLE ` + role + ` LOGIN PASSWORD '` + role + `'`, + `GRANT USAGE ON SCHEMA ` + schema + ` TO ` + role, + `GRANT SELECT,INSERT,UPDATE,DELETE ON ALL TABLES IN SCHEMA ` + schema + ` TO ` + role, + `GRANT USAGE,SELECT ON ALL SEQUENCES IN SCHEMA ` + schema + ` TO ` + role, + `REVOKE INSERT,UPDATE,DELETE ON ` + schema + `.corescope_schema FROM ` + role} { + if _, err = db.Exec(stmt); err != nil { + db.Close() + t.Fatal(err) + } + } + u.User = url.UserPassword(role, role) + dsn := u.String() + testDatabases.Store(cacheKey, dsn) + t.Cleanup(func() { + testDatabases.Delete(cacheKey) + defer db.Close() + if _, err := db.Exec(`DROP OWNED BY ` + role); err != nil { + t.Error(err) + } + if _, err := db.Exec(`DROP ROLE ` + role); err != nil { + t.Error(err) + } + }) + return dsn +} + +func testAdmin(t testing.TB, s *Store) *sql.DB { + if s.Backend() == dbconfig.SQLite { + return s.db + } + t.Helper() + var schema string + if err := s.db.QueryRow(`SELECT current_schema()`).Scan(&schema); err != nil { + t.Fatal(err) + } + u, _ := url.Parse(os.Getenv("CORESCOPE_TEST_POSTGRES_URL")) + q := u.Query() + q.Set("search_path", schema) + u.RawQuery = q.Encode() + db, err := pgutil.Open(u.String(), false) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { db.Close() }) + return db +} + +func TestPostgresPacketCommitOrdering(t *testing.T) { + s, err := OpenStore(testPostgresURL(t, "cursor")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + owner := testAdmin(t, s) + tx, err := beginWrite(owner) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback() + var earlier int64 + if err = tx.QueryRow(`INSERT INTO transmissions(raw_hex,hash,first_seen) VALUES('AA','delayed','2026-01-01T00:00:00Z') RETURNING id`).Scan(&earlier); err != nil { + t.Fatal(err) + } + done := make(chan error, 1) + go func() { + _, e := s.InsertTransmission(&PacketData{Hash: "following", RawHex: "BB", Timestamp: "2026-01-01T00:00:01Z"}) + done <- e + }() + select { + case err := <-done: + t.Fatalf("later ingest passed unfinished transaction: %v", err) + case <-time.After(75 * time.Millisecond): + } + if err = tx.Commit(); err != nil { + t.Fatal(err) + } + select { + case err := <-done: + if err != nil { + t.Fatal(err) + } + case <-time.After(3 * time.Second): + t.Fatal("writer did not resume") + } + var later int64 + if err = s.db.QueryRow(`SELECT id FROM transmissions WHERE hash='following'`).Scan(&later); err != nil { + t.Fatal(err) + } + if later <= earlier { + t.Fatalf("cursor order %d then %d", earlier, later) + } +} + +func TestPostgresRuntimeCannotMigrate(t *testing.T) { + s, err := OpenStore(testPostgresURL(t, "roles")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + for _, q := range []string{`CREATE TABLE forbidden(id int)`, `UPDATE corescope_schema SET ready=false`, `DROP TABLE transmissions`} { + if _, err = s.db.Exec(q); err == nil { + t.Errorf("runtime role allowed %s", q) + } + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if err = s.db.PingContext(ctx); err != nil { + t.Fatal(err) + } +} + +func TestPostgresNaiveClockUpsert(t *testing.T) { + s := newTestStore(t) + now := time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC) + for i, at := range []time.Time{now, now.Add(time.Hour), now.Add(48 * time.Hour)} { + if err := s.RecordNaiveSkew("clock-observer", -90, at); err != nil { + t.Fatal(err) + } + var count, skew int + if err := s.db.QueryRow(`SELECT clock_skew_count_24h,clock_skew_seconds FROM observers WHERE id='clock-observer'`).Scan(&count, &skew); err != nil { + t.Fatal(err) + } + want := []int{1, 2, 1}[i] + if count != want || skew != -90 { + t.Fatalf("event %d: count=%d skew=%d", i, count, skew) + } + } +} diff --git a/cmd/ingestor/pragma_parity_test.go b/cmd/ingestor/pragma_parity_test.go new file mode 100644 index 000000000..bfa5c48bc --- /dev/null +++ b/cmd/ingestor/pragma_parity_test.go @@ -0,0 +1,24 @@ +package main + +import "testing" + +// Durable settings are measured through the runtime connection, not logs. +func TestOpenStoreDurability(t *testing.T) { + s := newTestStore(t) + settings := map[string]string{"SHOW fsync": "on", "SHOW full_page_writes": "on", "SHOW synchronous_commit": "on"} + if s.Backend() == "sqlite" { + settings = map[string]string{"PRAGMA journal_mode": "wal", "PRAGMA synchronous": "2", "PRAGMA foreign_keys": "1"} + } + for setting, want := range settings { + var got string + if err := s.db.QueryRow(setting).Scan(&got); err != nil { + t.Fatal(err) + } + if got != want { + t.Errorf("%s=%s; require on", setting, got) + } + } + if _, err := s.db.Exec(`INSERT INTO observations(transmission_id,timestamp) VALUES(999999,1)`); err == nil { + t.Fatal("foreign key not enforced") + } +} diff --git a/cmd/ingestor/prune_chunked_test.go b/cmd/ingestor/prune_chunked_test.go new file mode 100644 index 000000000..f3e186e2d --- /dev/null +++ b/cmd/ingestor/prune_chunked_test.go @@ -0,0 +1,288 @@ +package main + +import ( + "fmt" + "path/filepath" + "strings" + "testing" + "time" +) + +// seedAgedTransmissions inserts n transmissions stamped `age` days in the +// past, each with obsPerTx child observations, and returns nothing — the +// caller asserts against the store. Kept local to this file so the batching +// tests own their fixture shape. +func seedAgedTransmissions(t *testing.T, store *Store, n, obsPerTx, ageDays int) { + t.Helper() + ts := time.Now().UTC().AddDate(0, 0, -ageDays).Format(time.RFC3339) + for i := 0; i < n; i++ { + var txID int64 + err := store.db.QueryRow( + `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) + VALUES ($1, $2, $3, 0, 1, 1, '{}') RETURNING id`, + "AA", fmt.Sprintf("h%d-%d", ageDays, i), ts, + ).Scan(&txID) + if err != nil { + t.Fatalf("seed tx %d: %v", i, err) + } + if err != nil { + t.Fatalf("seed tx %d LastInsertId: %v", i, err) + } + for j := 0; j < obsPerTx; j++ { + if _, err := store.db.Exec( + `INSERT INTO observations (transmission_id, observer_idx, direction, snr, rssi, score, path_json, timestamp) + VALUES ($1, $2, 'rx', 1.0, -100, 0, '[]', $3)`, + txID, j, time.Now().Unix(), + ); err != nil { + t.Fatalf("seed obs %d/%d: %v", i, j, err) + } + } + } +} + +func countRows(t *testing.T, store *Store, table string) int { + t.Helper() + var n int + if err := store.db.QueryRow(`SELECT COUNT(*) FROM ` + table).Scan(&n); err != nil { + t.Fatalf("count %s: %v", table, err) + } + return n +} + +func openPruneStore(t *testing.T, name string) *Store { + t.Helper() + store, err := openPostgresTestStore(t, filepath.Join(t.TempDir(), name)) + if err != nil { + t.Fatalf("OpenStore: %v", err) + } + t.Cleanup(func() { store.Close() }) + return store +} + +// TestPruneOldPacketsDeletesInBoundedBatches is the perf assertion for the +// chunked prune: it enforces that a retention day is deleted across many +// bounded transactions rather than one long-held writer lock. +// +// The old implementation issued exactly ONE WriterTx regardless of row +// count, holding writerMu — and therefore blocking MQTT ingest — for the +// whole delete. Measured on a production instance doing ~260k +// observations/day that was prune_packets hold_ms_p50 = 33905, with +// mqtt_handler wait_ms_max = 35797. +// +// Counting prune_packets transactions is the deterministic proxy for that +// characteristic: N transactions means the lock was released N-1 times, so +// the worst-case ingest stall is one batch rather than the full day. +func TestPruneOldPacketsDeletesInBoundedBatches(t *testing.T) { + store := openPruneStore(t, "prune-batched.db") + + // Two full batches plus a partial one. The partial batch is what ends the + // loop, so no terminating empty batch should run. + const aged = pruneBatchTransmissions*2 + 37 + seedAgedTransmissions(t, store, aged, 2, 10) + + ResetWriterStatsForTest() + + n, err := store.PruneOldPackets(5) + if err != nil { + t.Fatalf("PruneOldPackets: %v", err) + } + if n != aged { + t.Fatalf("expected %d transmissions pruned, got %d", aged, n) + } + + // 2 full batches + 1 partial batch, which proves nothing is left. + wantTx := int64(3) + got := store.WriterStatsSnapshot()["prune_packets"] + if got.Count != wantTx { + t.Fatalf("expected %d prune_packets transactions for %d rows at batch size %d, got %d "+ + "(1 means the delete was not chunked and holds the writer lock for the whole retention day)", + wantTx, aged, pruneBatchTransmissions, got.Count) + } + + if remaining := countRows(t, store, "transmissions"); remaining != 0 { + t.Fatalf("expected all aged transmissions gone, %d remain", remaining) + } + if remaining := countRows(t, store, "observations"); remaining != 0 { + t.Fatalf("expected all child observations gone, %d remain", remaining) + } +} + +// TestPruneOldPacketsExactMultipleTerminates covers the one case the +// short-batch exit cannot catch on its own: when the aged rows are an exact +// multiple of the batch size, the last batch comes back full, so the loop +// must run one more — empty — batch to learn that nothing is left. +func TestPruneOldPacketsExactMultipleTerminates(t *testing.T) { + store := openPruneStore(t, "prune-exact.db") + + const aged = pruneBatchTransmissions * 2 + seedAgedTransmissions(t, store, aged, 1, 10) + + ResetWriterStatsForTest() + + n, err := store.PruneOldPackets(5) + if err != nil { + t.Fatalf("PruneOldPackets: %v", err) + } + if n != aged { + t.Fatalf("expected %d pruned, got %d", aged, n) + } + + // 2 full batches + 1 empty batch that finds nothing and ends the loop. + if got := store.WriterStatsSnapshot()["prune_packets"].Count; got != 3 { + t.Fatalf("expected 3 prune_packets transactions for an exact multiple of the batch size, got %d", got) + } + if remaining := countRows(t, store, "transmissions"); remaining != 0 { + t.Fatalf("expected all aged transmissions gone, %d remain", remaining) + } + if remaining := countRows(t, store, "observations"); remaining != 0 { + t.Fatalf("expected all child observations gone, %d remain", remaining) + } +} + +// TestPruneOldPacketsSpansBatchesAndKeepsFreshRows covers the correctness +// risk the batching introduces: with a LIMIT on both statements, a cutoff +// that straddles several batches must still delete every aged row and no +// fresh one, and must not orphan observations. +func TestPruneOldPacketsSpansBatchesAndKeepsFreshRows(t *testing.T) { + store := openPruneStore(t, "prune-mixed.db") + + const aged = pruneBatchTransmissions + 11 + const fresh = 17 + const obsPerTx = 3 + seedAgedTransmissions(t, store, aged, obsPerTx, 10) + seedAgedTransmissions(t, store, fresh, obsPerTx, 0) + + n, err := store.PruneOldPackets(5) + if err != nil { + t.Fatalf("PruneOldPackets: %v", err) + } + if n != aged { + t.Fatalf("expected %d pruned, got %d", aged, n) + } + + if remaining := countRows(t, store, "transmissions"); remaining != fresh { + t.Fatalf("expected %d fresh transmissions kept, got %d", fresh, remaining) + } + if remaining := countRows(t, store, "observations"); remaining != fresh*obsPerTx { + t.Fatalf("expected %d observations kept, got %d", fresh*obsPerTx, remaining) + } + + // No observation may reference a transmission that is gone. + var orphans int + if err := store.db.QueryRow(` + SELECT COUNT(*) FROM observations o + LEFT JOIN transmissions t ON t.id = o.transmission_id + WHERE t.id IS NULL`).Scan(&orphans); err != nil { + t.Fatalf("orphan check: %v", err) + } + if orphans != 0 { + t.Fatalf("expected no orphaned observations, got %d", orphans) + } +} + +// TestPruneOldPacketsDisabledTakesNoWriterLock pins the 0/negative +// short-circuit: retention disabled must not open a transaction at all. +func TestPruneOldPacketsDisabledTakesNoWriterLock(t *testing.T) { + store := openPruneStore(t, "prune-disabled.db") + seedAgedTransmissions(t, store, 5, 1, 10) + + ResetWriterStatsForTest() + + for _, days := range []int{0, -1} { + n, err := store.PruneOldPackets(days) + if err != nil { + t.Fatalf("PruneOldPackets(%d): %v", days, err) + } + if n != 0 { + t.Fatalf("PruneOldPackets(%d) = %d, want 0", days, n) + } + } + + if got := store.WriterStatsSnapshot()["prune_packets"].Count; got != 0 { + t.Fatalf("expected no prune_packets transactions when retention is disabled, got %d", got) + } + if remaining := countRows(t, store, "transmissions"); remaining != 5 { + t.Fatalf("expected 5 transmissions untouched, got %d", remaining) + } +} + +// TestPruneOldPacketsNothingToDeleteRunsOneEmptyBatch documents the +// steady-state cost when nothing has aged out yet: a single empty batch, then +// the loop exits. That batch is only cheap because the subquery is walked off +// idx_transmissions_first_seen — see +// TestPruneAgedTransmissionIDsHasOrderedIndex. +func TestPruneOldPacketsNothingToDeleteRunsOneEmptyBatch(t *testing.T) { + store := openPruneStore(t, "prune-noop.db") + seedAgedTransmissions(t, store, 9, 2, 0) + + ResetWriterStatsForTest() + + n, err := store.PruneOldPackets(5) + if err != nil { + t.Fatalf("PruneOldPackets: %v", err) + } + if n != 0 { + t.Fatalf("expected 0 pruned, got %d", n) + } + if got := store.WriterStatsSnapshot()["prune_packets"].Count; got != 1 { + t.Fatalf("expected exactly 1 empty prune_packets transaction, got %d", got) + } + if remaining := countRows(t, store, "transmissions"); remaining != 9 { + t.Fatalf("expected 9 transmissions kept, got %d", remaining) + } +} + +// TestPruneAgedTransmissionIDsHasOrderedIndex pins the query plan of the +// batch subquery and of both statements that embed it. +// +// Ordering the batch by id instead of first_seen makes SQLite drop +// idx_transmissions_first_seen for a rowid SCAN. On a 1M-row table that took +// the terminating, nothing-left batch from ~10µs to ~73ms — once per statement, +// under writerMu, in the state an instance is in whenever nothing has aged out. +// Transaction counts cannot see that regression, so the plan is the assertion. +func TestPruneAgedTransmissionIDsHasOrderedIndex(t *testing.T) { + store := openPruneStore(t, "prune-plan.db") + seedAgedTransmissions(t, store, 20, 2, 10) + + cutoff := time.Now().UTC().AddDate(0, 0, -5).Format(time.RFC3339) + for name, q := range map[string]string{ + "batch subquery": pruneAgedTransmissionIDs, + "observations delete": pruneObservationsBatch, + "transmissions delete": pruneTransmissionsBatch, + } { + if testBackendValue() == "postgres" { + if _, err := store.db.Exec(`SET enable_seqscan=off; SET enable_bitmapscan=off`); err != nil { + t.Fatal(err) + } + } + rows, err := store.db.Query(testNativeSQL(`EXPLAIN QUERY PLAN `, "EXPLAIN (COSTS OFF) ")+q, cutoff, pruneBatchTransmissions) + if err != nil { + t.Fatalf(testNativeSQL(`%s: EXPLAIN QUERY PLAN: %v`, "%s: EXPLAIN (COSTS OFF): %v"), name, err) + } + var steps []string + for rows.Next() { + var detail string + if err := scanTestPlan(rows, &detail); err != nil { + rows.Close() + t.Fatalf("%s: scan plan row: %v", name, err) + } + steps = append(steps, detail) + } + if err := rows.Err(); err != nil { + rows.Close() + t.Fatalf("%s: plan rows: %v", name, err) + } + rows.Close() + plan := strings.Join(steps, " | ") + + if !strings.Contains(plan, "idx_transmissions_first_seen") { + t.Errorf("%s: plan does not use idx_transmissions_first_seen: %s", name, plan) + } + if strings.Contains(plan, "Seq Scan on transmissions") { + t.Errorf("%s: plan scans transmissions, so the empty terminating batch walks the whole table under writerMu: %s", name, plan) + } + if strings.Contains(plan, "Sort Key:") { + t.Errorf("%s: ordered-index eligibility query unexpectedly requires a sort: %s", name, plan) + } + } +} diff --git a/cmd/ingestor/prune_geofilter.go b/cmd/ingestor/prune_geofilter.go index 38f75857f..04316415d 100644 --- a/cmd/ingestor/prune_geofilter.go +++ b/cmd/ingestor/prune_geofilter.go @@ -1,7 +1,7 @@ // Package main: ingestor-side processor for prune-request marker files // written by the read-only server (see internal/prunequeue). // -// The server cannot DELETE because it opens SQLite mode=ro (#1283/#1289). +// The server cannot DELETE because it uses a restricted PostgreSQL reader (#1283/#1289). // Instead, the server writes request-.json under /prune-requests/ // and the ingestor consumes it here. package main @@ -22,7 +22,7 @@ func (s *Store) DeleteNodesByPubkeys(pubkeys []string) (int64, error) { if len(pubkeys) == 0 { return 0, nil } - // Chunk to keep statements under SQLite's variable limit (default 999). + // Bound each delete batch and the number of bind parameters. const chunk = 500 var total int64 for start := 0; start < len(pubkeys); start += chunk { @@ -31,8 +31,11 @@ func (s *Store) DeleteNodesByPubkeys(pubkeys []string) (int64, error) { end = len(pubkeys) } batch := pubkeys[start:end] - placeholders := strings.Repeat("?,", len(batch)) - placeholders = placeholders[:len(placeholders)-1] + slots := make([]string, len(batch)) + for i := range slots { + slots[i] = fmt.Sprintf("%s", s.parameter(i+1)) + } + placeholders := strings.Join(slots, ",") args := make([]interface{}, len(batch)) for i, pk := range batch { args[i] = pk @@ -59,7 +62,7 @@ func (s *Store) DeleteNodesByPubkeys(pubkeys []string) (int64, error) { } // RunPendingPruneRequests scans the prune-requests/ directory next to the -// SQLite database and processes any request-.json markers written by +// state directory and processes any request-.json markers written by // the server. Each request is honored verbatim — the server is responsible // for the TOCTOU snapshot (only pubkeys that were still outside the // geofilter at confirm time). After running DELETE, the ingestor writes diff --git a/cmd/ingestor/prune_geofilter_test.go b/cmd/ingestor/prune_geofilter_test.go index f12654c89..def2e353c 100644 --- a/cmd/ingestor/prune_geofilter_test.go +++ b/cmd/ingestor/prune_geofilter_test.go @@ -12,7 +12,7 @@ func TestRunPendingPruneRequests(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "test.db") - store, err := OpenStore(dbPath) + store, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore: %v", err) } @@ -67,7 +67,7 @@ func TestRunPendingPruneRequests(t *testing.T) { func TestRunPendingPruneRequests_EmptyQueueIsNoop(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "test.db") - store, err := OpenStore(dbPath) + store, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore: %v", err) } diff --git a/cmd/ingestor/refresh_planner_stats_test.go b/cmd/ingestor/refresh_planner_stats_test.go new file mode 100644 index 000000000..a04801709 --- /dev/null +++ b/cmd/ingestor/refresh_planner_stats_test.go @@ -0,0 +1,311 @@ +package main + +import ( + "bytes" + "github.com/meshcore-analyzer/dbconfig" + "log" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestAnalysisLimitConfigDefault_Issue2058(t *testing.T) { + cases := []struct { + name string + cfg *Config + want int + }{ + // Zero means "no limit" to SQLite, so an unset config must not be + // passed through as 0: that would turn a 2 second refresh into the + // 242.9s unbounded ANALYZE measured on the staging database. + {"no db section", &Config{}, 10000}, + {"db section, limit unset", &Config{DB: &dbconfig.DBConfig{}}, 10000}, + {"explicit limit", &Config{DB: &dbconfig.DBConfig{AnalysisLimit: 1000}}, 1000}, + {"disabled", &Config{DB: &dbconfig.DBConfig{AnalysisLimit: -1}}, -1}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := tc.cfg.AnalysisLimit(); got != tc.want { + t.Errorf("AnalysisLimit() = %d, want %d", got, tc.want) + } + }) + } +} + +// The default is not a free choice: 400 and 1000 were measured to leave the +// channel-query plan unchanged on the staging database, so a well-meant edit +// back to SQLite's documented 400 would quietly return this to a no-op. +func TestAnalysisLimitDefaultIsHighEnoughToMatter_Issue2058(t *testing.T) { + const measuredIneffective = 1000 + if got := (&Config{}).AnalysisLimit(); got <= measuredIneffective { + t.Errorf("default analysis_limit is %d; %d and below were measured to leave the plan unchanged on a 9.4 GB database", + got, measuredIneffective) + } +} + +func TestAnalysisLimitSurvivesTheConfigFile_Issue2058(t *testing.T) { + // The knob is only useful if it survives the config file. The field lives in + // internal/dbconfig, so a wrong json tag there would leave the accessor + // returning the default however the operator set it. + path := filepath.Join(t.TempDir(), "config.json") + if err := os.WriteFile(path, []byte(`{"db":{"analysisLimit":123}}`), 0o600); err != nil { + t.Fatal(err) + } + cfg, err := LoadConfig(path) + if err != nil { + t.Fatalf("LoadConfig: %v", err) + } + if got := cfg.AnalysisLimit(); got != 123 { + t.Errorf("analysisLimit did not survive the config file: got %d, want 123", got) + } +} + +func TestPostgresOwnerAnalyzesTelemetry(t *testing.T) { + if testBackend(t) != dbconfig.Postgres { + t.Skip("PostgreSQL planner matrix not selected") + } + s := newTestStore(t) + owner := testAdmin(t, s) + if _, err := owner.Exec(`INSERT INTO transmissions(raw_hex,hash,first_seen) SELECT 'aa', n::text,'2026-01-01T00:00:00Z' FROM generate_series(1,1000) n`); err != nil { + t.Fatal(err) + } + if _, err := owner.Exec(`ANALYZE transmissions`); err != nil { + t.Fatal(err) + } + var estimate float64 + if err := owner.QueryRow(`SELECT reltuples FROM pg_class WHERE oid='transmissions'::regclass`).Scan(&estimate); err != nil { + t.Fatal(err) + } + if estimate != 1000 { + t.Fatalf("planner row estimate=%v, want 1000", estimate) + } +} +func TestPostgresRuntimeCannotAlterPlannerTargets(t *testing.T) { + if testBackend(t) != dbconfig.Postgres { + t.Skip("PostgreSQL planner matrix not selected") + } + s := newTestStore(t) + if _, err := s.db.Exec(`ALTER TABLE transmissions ALTER COLUMN payload_type SET STATISTICS 1000`); err == nil { + t.Fatal("runtime role changed owner planner policy") + } +} + +func TestRefreshPlannerStatsWritesStatistics_Issue2058(t *testing.T) { + if testBackend(t) != dbconfig.SQLite { + t.Skip("SQLite planner matrix not selected") + } + + s := newTestStore(t) + defer s.Close() + + if hasStat1(t, s) { + t.Fatal("a fresh store already carries sqlite_stat1, so this test cannot tell whether the refresh did anything") + } + + if !s.RefreshPlannerStats(10000) { + t.Fatal("RefreshPlannerStats reported no refresh") + } + + if !hasStat1(t, s) { + t.Error("sqlite_stat1 was not created, so the planner still has no statistics") + } +} + +func TestRefreshPlannerStatsAppliesTheLimit_Issue2058(t *testing.T) { + if testBackend(t) != dbconfig.SQLite { + t.Skip("SQLite planner matrix not selected") + } + + s := newTestStore(t) + defer s.Close() + + s.RefreshPlannerStats(250) + + // analysis_limit is per connection. The store runs SetMaxOpenConns(1) + // (db.go:142), which is the only reason setting it through Exec is sound + // here: on a multi-connection pool the pragma could land on a connection + // the ANALYZE never uses, and the limit would silently not apply. + var limit int + if err := s.db.QueryRow("PRAGMA analysis_limit").Scan(&limit); err != nil { + t.Fatalf("read back analysis_limit: %v", err) + } + if limit != 250 { + t.Errorf("analysis_limit did not reach the connection: want 250, got %d", limit) + } +} + +func TestRefreshPlannerStatsNegativeLimitIsANoop_Issue2058(t *testing.T) { + if testBackend(t) != dbconfig.SQLite { + t.Skip("SQLite planner matrix not selected") + } + + s := newTestStore(t) + defer s.Close() + + if s.RefreshPlannerStats(-1) { + t.Error("a negative limit must not report a refresh") + } + if hasStat1(t, s) { + t.Error("a negative limit still built sqlite_stat1; the refresh is not actually disabled") + } +} + +func TestRefreshPlannerStatsIsRepeatable_Issue2058(t *testing.T) { + if testBackend(t) != dbconfig.SQLite { + t.Skip("SQLite planner matrix not selected") + } + + s := newTestStore(t) + defer s.Close() + + // The ticker calls this every 24h for the life of the process. A second + // call must not error or undo the first, which is the part a single-call + // test would not notice. + s.RefreshPlannerStats(10000) + if !s.RefreshPlannerStats(10000) { + t.Fatal("the second refresh reported failure") + } + if !hasStat1(t, s) { + t.Error("sqlite_stat1 disappeared across two refreshes") + } +} + +func TestEnsurePlannerStatsBuildsWhenAbsent_Issue2058(t *testing.T) { + if testBackend(t) != dbconfig.SQLite { + t.Skip("SQLite planner matrix not selected") + } + + s := newTestStore(t) + defer s.Close() + + if !s.EnsurePlannerStats(10000) { + t.Fatal("EnsurePlannerStats did not build statistics on a database that has none") + } + if !hasStat1(t, s) { + t.Error("sqlite_stat1 absent after EnsurePlannerStats") + } +} + +func TestEnsurePlannerStatsSkipsWhenPresent_Issue2058(t *testing.T) { + if testBackend(t) != dbconfig.SQLite { + t.Skip("SQLite planner matrix not selected") + } + + s := newTestStore(t) + defer s.Close() + + s.RefreshPlannerStats(10000) + // The point of the skip: on every restart after the first this must cost one + // sqlite_master query, not an ANALYZE. If it ever returns true here it is + // running the 2s refresh on every boot. + if s.EnsurePlannerStats(10000) { + t.Error("EnsurePlannerStats rebuilt statistics that were already there") + } +} + +func TestEnsurePlannerStatsRespectsDisabled_Issue2058(t *testing.T) { + if testBackend(t) != dbconfig.SQLite { + t.Skip("SQLite planner matrix not selected") + } + + s := newTestStore(t) + defer s.Close() + + if s.EnsurePlannerStats(-1) { + t.Error("a negative limit must not build statistics") + } + if hasStat1(t, s) { + t.Error("sqlite_stat1 built despite the refresh being disabled") + } +} + +func TestPlannerStatsSurviveReopen_Issue2058(t *testing.T) { + if testBackend(t) != dbconfig.SQLite { + t.Skip("SQLite planner matrix not selected") + } + + dir := t.TempDir() + dbPath := dir + "/reopen.db" + + first, err := OpenStore(dbPath) + if err != nil { + t.Fatal(err) + } + first.WaitForAsyncMigrations() + if !first.EnsurePlannerStats(10000) { + t.Fatal("first store did not build statistics") + } + first.Close() + + second, err := OpenStore(dbPath) + if err != nil { + t.Fatal(err) + } + defer second.Close() + second.WaitForAsyncMigrations() + + if !hasStat1(t, second) { + t.Fatal("statistics did not survive reopening the database") + } + if second.EnsurePlannerStats(10000) { + t.Error("the reopened store rebuilt statistics, so a restart would pay for an ANALYZE it does not need") + } +} + +func TestEnsurePlannerStatsWarnsBeforeBuilding_Issue2058(t *testing.T) { + if testBackend(t) != dbconfig.SQLite { + t.Skip("SQLite planner matrix not selected") + } + + s := newTestStore(t) + defer s.Close() + + var buf bytes.Buffer + orig := log.Writer() + log.SetOutput(&buf) + defer log.SetOutput(orig) + + s.EnsurePlannerStats(10000) + + out := buf.String() + for _, want := range []string{"no planner statistics", "write connection", "Once per database"} { + if !strings.Contains(out, want) { + t.Errorf("the first-build warning does not mention %q; an operator seeing ingest stall gets no explanation.\ngot: %s", want, out) + } + } + if i, j := strings.Index(out, "no planner statistics"), strings.Index(out, "statistics built in"); i == -1 || j == -1 || i > j { + t.Errorf("the warning must come before the completion line, so it is visible while the write path is held; got: %s", out) + } +} + +func TestEnsurePlannerStatsIsQuietWhenPresent_Issue2058(t *testing.T) { + if testBackend(t) != dbconfig.SQLite { + t.Skip("SQLite planner matrix not selected") + } + + s := newTestStore(t) + defer s.Close() + s.RefreshPlannerStats(10000) + + var buf bytes.Buffer + orig := log.Writer() + log.SetOutput(&buf) + defer log.SetOutput(orig) + + s.EnsurePlannerStats(10000) + + if out := buf.String(); strings.Contains(out, "no planner statistics") { + t.Errorf("warned about building on a database that already has statistics: %s", out) + } +} + +func hasStat1(t *testing.T, s *Store) bool { + t.Helper() + var n int + if err := s.db.QueryRow( + `SELECT count(*) FROM sqlite_master WHERE type='table' AND name='sqlite_stat1'`).Scan(&n); err != nil { + t.Fatalf("query sqlite_master: %v", err) + } + return n > 0 +} diff --git a/cmd/ingestor/region_keys.go b/cmd/ingestor/region_keys.go new file mode 100644 index 000000000..26b9c5f41 --- /dev/null +++ b/cmd/ingestor/region_keys.go @@ -0,0 +1,467 @@ +package main + +import ( + "crypto/sha256" + "fmt" + "log" + "sort" + "strings" + "sync/atomic" + "time" +) + +// declaredRegionStat is one region name as reported over RF, with the two +// facts the cap ranks on: how many distinct repeaters declare it, and how +// recently any of them last answered. +type declaredRegionStat struct { + Name string + Declarers int + LastSeen string // ISO, greatest observed_at across declarers +} + +// maxRegionNameLen bounds a derived region name. Firmware region names are +// short labels; anything longer is a malformed or hostile entry, and each +// accepted name costs an HMAC on every transport-scoped packet. +const maxRegionNameLen = 32 + +// regionNameAcceptable reports whether a declared name may become a derived +// region key, and returns the name in the form the key is derived from. +// +// The two declared-region sources spell a name differently, so this +// canonicalises before judging: nodes.configured_scope carries the leading +// "#" (normalizeScopeList puts it there, matching every other stored scope +// value), while a node_declared_regions row carries the bare name the OTA +// query returned. Both mean the same region, and loadRegionKeys already +// prefixes a missing "#" before hashing, so the "#" is stripped here and put +// back by the caller. +// +// The rules are structural, never about the name's meaning. A declared set +// contains entries that look like junk ("null", "sol3"), but a blocklist on +// string values is unmaintainable, and the cost of one bad name is a single +// slot out of maxDerived plus a 1-in-65536 collision chance. What IS rejected +// is anything that cannot be a region name: +// +// - "*" is the flood wildcard, not a region, and hashing it would invent a +// region nobody declared +// - a comma would split the name on the next round-trip through a +// comma-separated column +// - a second "#" cannot come from either source intact +// - non-ASCII or whitespace would make the key a hash over bytes nobody +// intended, silently mismatching the sender +// - a NUL is block-cipher padding a stale client failed to trim +func regionNameAcceptable(name string) (string, bool) { + name = strings.TrimPrefix(name, "#") + if name == "" || name == "*" || len(name) > maxRegionNameLen { + return "", false + } + for i := 0; i < len(name); i++ { + c := name[i] + if c <= ' ' || c >= 0x7F || c == ',' || c == '#' { + return "", false + } + } + return name, true +} + +// declaredRegionSources reads the declared region names from every source this +// instance has, newest answer per node, and counts how many distinct nodes +// declare each name. +// +// Two sources, mirroring what the server's AllCurrentDeclaredRegions already +// merges, so the derived tier sees exactly what the audit sees: +// +// - nodes.configured_scope, written by the observer /neighbors ingestion +// (#1865). Always present; the column is part of the schema. +// - node_declared_regions, an optional table a deployment may fill by other +// means. Absent on a stock install, so its absence is not an error: the +// probe below asks information_schema first rather than letting "no such table" +// abort a refresh that the first source could still answer. +// +// A node counts once per name however many times it declares it, so the +// ranking below reflects how widely a region is claimed rather than how +// chatty one node is. +func (s *Store) declaredRegionSources() ([]declaredRegionStat, error) { + agg := map[string]*declaredRegionStat{} + count := func(csv, at string) { + seenHere := map[string]bool{} // one node counts once per name + for _, name := range splitDeclaredRegionsCSV(csv) { + // '*' is the flood wildcard, not a region. regionNameAcceptable + // rejects it too, but skipping it here as well keeps it out of the + // declared-name count that the cap arithmetic and the refresh log + // are stated in: nearly every node declares it, so counting it + // would overstate both on every deployment. + if name == "*" || name == "#*" || seenHere[name] { + continue + } + seenHere[name] = true + st, ok := agg[name] + if !ok { + st = &declaredRegionStat{Name: name} + agg[name] = st + } + st.Declarers++ + if at > st.LastSeen { + st.LastSeen = at + } + } + } + + rows, err := s.db.Query(` + SELECT COALESCE(configured_scope, ''), COALESCE(configured_scope_at, '') + FROM nodes + WHERE configured_scope IS NOT NULL AND configured_scope != '' + `) + if err != nil { + return nil, fmt.Errorf("declared regions from configured_scope: %w", err) + } + for rows.Next() { + var csv, at string + if err := rows.Scan(&csv, &at); err != nil { + rows.Close() + return nil, fmt.Errorf("declared regions from configured_scope scan: %w", err) + } + count(csv, at) + } + if err := rows.Err(); err != nil { + rows.Close() + return nil, fmt.Errorf("declared regions from configured_scope rows: %w", err) + } + rows.Close() + + var present string + if err := s.db.QueryRow(s.nativeSQL("SELECT name FROM sqlite_master WHERE type='table' AND name='node_declared_regions'", `SELECT table_name FROM information_schema.tables WHERE table_schema=current_schema() AND table_name='node_declared_regions'`)).Scan(&present); err == nil && present != "" { + ndr, err := s.db.Query(` + WITH ranked AS ( + SELECT target, observed_at, regions_csv, + ROW_NUMBER() OVER (PARTITION BY target ORDER BY observed_at DESC) AS rn + FROM node_declared_regions + ) + SELECT observed_at, regions_csv FROM ranked WHERE rn = 1 + `) + if err != nil { + return nil, fmt.Errorf("declared regions from node_declared_regions: %w", err) + } + defer ndr.Close() + for ndr.Next() { + var at, csv string + if err := ndr.Scan(&at, &csv); err != nil { + return nil, fmt.Errorf("declared regions from node_declared_regions scan: %w", err) + } + count(csv, at) + } + if err := ndr.Err(); err != nil { + return nil, fmt.Errorf("declared regions from node_declared_regions rows: %w", err) + } + } + + out := make([]declaredRegionStat, 0, len(agg)) + for _, st := range agg { + out = append(out, *st) + } + return out, nil +} + +// rankDeclaredRegions filters stats through regionNameAcceptable and returns at +// most max names, most-worth-keeping first: by declarer count descending, then +// by recency, then by name. The name tie-break is what makes the result +// deterministic — without it the derived tier would churn between refreshes on +// equally-ranked names and the add/drop logging would be noise. +func rankDeclaredRegions(stats []declaredRegionStat, max int) []string { + kept := make([]declaredRegionStat, 0, len(stats)) + for _, s := range stats { + if canonical, ok := regionNameAcceptable(s.Name); ok { + s.Name = canonical + kept = append(kept, s) + } + } + sort.Slice(kept, func(i, j int) bool { + if kept[i].Declarers != kept[j].Declarers { + return kept[i].Declarers > kept[j].Declarers + } + if kept[i].LastSeen != kept[j].LastSeen { + return kept[i].LastSeen > kept[j].LastSeen + } + return kept[i].Name < kept[j].Name + }) + if max > 0 && len(kept) > max { + kept = kept[:max] + } + names := make([]string, 0, len(kept)) + for _, s := range kept { + names = append(names, s.Name) + } + return names +} + +// splitDeclaredRegionsCSV parses a regions_csv value into its entries. The +// ingestor writes this column with strings.Join(regions, ","), so this is its +// exact inverse. Mirrors splitRegionsCSV in cmd/server/scopes.go. +func splitDeclaredRegionsCSV(csv string) []string { + out := []string{} + if csv == "" { + return out + } + for _, part := range strings.Split(csv, ",") { + part = strings.TrimSpace(part) + if part != "" { + out = append(out, part) + } + } + return out +} + +// regionKeySnapshot is an immutable view of the region keys in force for one +// packet. `all` is the single map matching iterates - merging at build time +// rather than per packet keeps the hot path free of allocation. `explicit` +// carries membership only, and exists so the ambiguity tie-break can tell an +// operator-configured region from one derived off the air. +type regionKeySnapshot struct { + all map[string][]byte + explicit map[string]bool +} + +func (s *regionKeySnapshot) isExplicit(name string) bool { return s.explicit[name] } + +// regionKeySet holds the live snapshot. Readers take one atomic load; a +// refresh builds the replacement off to the side and swaps the pointer, so the +// ingest hot path never blocks on a rebuild (AGENTS.md rule 0). +type regionKeySet struct { + cur atomic.Pointer[regionKeySnapshot] + enabled bool + max int +} + +// newRegionKeySet builds the explicit tier from hashRegions. The derived tier +// starts empty; refreshDerived fills it, and does nothing at all when +// autoRegionKeys is off. +func newRegionKeySet(cfg *Config) *regionKeySet { + explicitKeys := loadRegionKeys(cfg) + explicitNames := make(map[string]bool, len(explicitKeys)) + all := make(map[string][]byte, len(explicitKeys)) + for name, key := range explicitKeys { + explicitNames[name] = true + all[name] = key + } + s := ®ionKeySet{ + enabled: cfg.AutoRegionKeysEnabled(), + max: cfg.AutoRegionKeysMaxDerived(), + } + s.cur.Store(®ionKeySnapshot{all: all, explicit: explicitNames}) + return s +} + +// regionKeySetFromKeys wraps an explicit key map in a set with no derived +// tier. Tests that predate the derived tier use it to keep expressing "these +// are the configured regions" without building a Config. +func regionKeySetFromKeys(keys map[string][]byte) *regionKeySet { + names := make(map[string]bool, len(keys)) + all := make(map[string][]byte, len(keys)) + for name, key := range keys { + names[name] = true + all[name] = key + } + s := ®ionKeySet{} + s.cur.Store(®ionKeySnapshot{all: all, explicit: names}) + return s +} + +// emptyRegionKeySnapshot backs the nil case below. Shared and never mutated: +// refreshDerived always builds a fresh map rather than writing into one. +var emptyRegionKeySnapshot = ®ionKeySnapshot{all: map[string][]byte{}, explicit: map[string]bool{}} + +// snapshot is nil-safe on purpose. A nil *regionKeySet means "no region keys", +// which is exactly what a nil map[string][]byte meant before M2 — the shape +// several call sites and a good many tests still pass. Panicking there would +// turn an absent key set into a crash on the ingest path, which is a far worse +// failure than naming nothing. +func (s *regionKeySet) snapshot() *regionKeySnapshot { + if s == nil { + return emptyRegionKeySnapshot + } + return s.cur.Load() +} + +// refreshDerived rebuilds the derived tier from names (already ranked and +// capped by the caller) and swaps in a new snapshot. It REPLACES the derived +// tier rather than merging into it, so a region that stops being declared +// leaves the key set and the cap keeps meaning something. +// +// A name that duplicates an explicit key is skipped, not re-added: the +// explicit tier must stay authoritative for the tie-break, and demoting a +// configured region because a repeater also declares it would invert the whole +// rule. +// +// Returns the names actually added, for the caller to log. +func (s *regionKeySet) refreshDerived(names []string) []string { + if !s.enabled { + return nil + } + old := s.cur.Load() + all := make(map[string][]byte, len(old.explicit)+len(names)) + for name := range old.explicit { + all[name] = old.all[name] + } + added := make([]string, 0, len(names)) + for _, raw := range names { + canonical, ok := regionNameAcceptable(raw) + if !ok { + continue + } + name := "#" + canonical + if old.explicit[name] { + continue + } + if _, exists := all[name]; exists { + continue + } + h := sha256.Sum256([]byte(name)) + all[name] = h[:16] + added = append(added, name) + } + s.cur.Store(®ionKeySnapshot{all: all, explicit: old.explicit}) + return added +} + +// scopeReason records how a scope match was decided, so the outcome is +// auditable in logs without a schema change. It is deliberately not stored: +// transmissions.scope_name keeps its existing three-state encoding. +type scopeReason string + +const ( + scopeReasonNone scopeReason = "none" // no key matched + scopeReasonUnique scopeReason = "unique" // exactly one key matched + scopeReasonExplicitOverDerived scopeReason = "explicit-over-derived" // several matched, one was operator config + scopeReasonAmbiguous scopeReason = "ambiguous" // several matched, no principled winner +) + +// scopeMatch is the result of naming one packet's region scope. +type scopeMatch struct { + Name string // empty when unresolved - the caller stores that as the unmatched state + Reason scopeReason + // Candidates is every name whose derived code equals the packet's code1. + // It is always populated, so a caller that needs the match COUNT (scope-repair + // does, to tell "the key set changed" from "several keys now collide") can read + // it from the same result that carries the verdict, instead of re-running the + // match and risking a second, divergent decision rule. + Candidates []string +} + +// match names the region scope of a transport-scoped packet, resolving a +// multi-key collision by evidence rather than by map order. +// +// Tiers, in order: +// +// 1. Exactly one key matched - name it. +// 2. Several matched but exactly one came from hashRegions - name that one. +// The operator's own configuration outranks a name picked up off the air, +// and this covers the bulk of the ambiguity auto-derivation introduces. +// 3. Otherwise abstain, returning an empty name. Two equally-sourced +// candidates offer no principled winner, and naming a packet wrongly is +// worse than leaving it unnamed - the rule #1609 established, unchanged. +// +// (The spec's tier-3 path-evidence tie-break sits between 2 and 3 and is +// deliberately not built here; see +// docs/specs/2026-09-07-auto-region-keys-design.md. The scopeReasonAmbiguous +// counter is what measures whether it is worth building.) +func (s *regionKeySnapshot) match(payloadType byte, payloadRaw []byte, code1 string) scopeMatch { + matched := matchingRegions(s.all, payloadType, payloadRaw, code1) + switch len(matched) { + case 0: + return scopeMatch{Reason: scopeReasonNone, Candidates: matched} + case 1: + return scopeMatch{Name: matched[0], Reason: scopeReasonUnique, Candidates: matched} + } + + var explicitMatches []string + for _, name := range matched { + if s.explicit[name] { + explicitMatches = append(explicitMatches, name) + } + } + if len(explicitMatches) == 1 { + return scopeMatch{Name: explicitMatches[0], Reason: scopeReasonExplicitOverDerived, Candidates: matched} + } + return scopeMatch{Reason: scopeReasonAmbiguous, Candidates: matched} +} + +// matchScopeName is what the ingest path calls: the tiered decision above, +// reduced to the string transmissions.scope_name stores, with the outcome +// tallied for the counters. A nil set (no region keys at all) names nothing, +// which is what an unconfigured instance did before this existed. +func (s *regionKeySet) matchScopeName(payloadType byte, payloadRaw []byte, code1 string) string { + m := s.snapshot().match(payloadType, payloadRaw, code1) + recordScopeMatch(m) + return m.Name +} + +// scopeMatchCounters tallies how each transport-scoped packet's region was +// decided. It exists to answer one question before more machinery is built: +// how often does an ambiguous collision actually happen? The spec gates the +// path-evidence tie-break (tier 3) on this number. +var scopeMatchCounters struct { + unique atomic.Int64 + explicitOverDerived atomic.Int64 + ambiguous atomic.Int64 + none atomic.Int64 + // sinceUnix is when this tally started counting, carried across + // restarts with the counters themselves (scope_match_tally.go). A + // ratio without its window is not a measurement. + sinceUnix atomic.Int64 +} + +// recordScopeMatch tallies one decision and logs the interesting ones. Unique +// and none are the overwhelming majority and are counted silently; the other +// two are rare by construction and worth a line each. +func recordScopeMatch(m scopeMatch) { + switch m.Reason { + case scopeReasonUnique: + scopeMatchCounters.unique.Add(1) + case scopeReasonNone: + scopeMatchCounters.none.Add(1) + case scopeReasonExplicitOverDerived: + scopeMatchCounters.explicitOverDerived.Add(1) + log.Printf("[regions] collision resolved to explicit %s over derived candidates %v", m.Name, m.Candidates) + case scopeReasonAmbiguous: + scopeMatchCounters.ambiguous.Add(1) + log.Printf("[regions] ambiguous collision between %v; storing unmatched", m.Candidates) + } +} + +// logScopeMatchCounters prints the running tally. Called from the refresh +// ticker so the numbers arrive on the same cadence as the key-set changes that +// move them. +func logScopeMatchCounters() { + log.Printf("[regions] scope matches since %s: unique=%d explicit-over-derived=%d ambiguous=%d none=%d", + time.Unix(scopeMatchCounters.sinceUnix.Load(), 0).UTC().Format(time.RFC3339), + scopeMatchCounters.unique.Load(), scopeMatchCounters.explicitOverDerived.Load(), + scopeMatchCounters.ambiguous.Load(), scopeMatchCounters.none.Load()) +} + +// refreshFromStore reads the declared region names, ranks and caps them, and +// swaps in a new snapshot. Shared by startup and the ticker so both apply +// identical rules. +// +// A DB error is logged and the CURRENT snapshot is kept. That matters: an +// empty key set would silently unname all traffic, which looks exactly like +// the bug this feature exists to fix. +func (s *regionKeySet) refreshFromStore(store *Store) { + if s == nil || !s.enabled { + return + } + stats, err := store.declaredRegionSources() + if err != nil { + log.Printf("[regions] derived-key refresh failed, keeping %d existing key(s): %v", len(s.snapshot().all), err) + return + } + ranked := rankDeclaredRegions(stats, s.max) + added := s.refreshDerived(ranked) + snap := s.snapshot() + log.Printf("[regions] derived-key refresh: %d name(s) declared, %d kept after filter+cap(%d), %d total key(s) in force", + len(stats), len(ranked), s.max, len(snap.all)) + if len(added) > 0 { + log.Printf("[regions] derived keys now active: %v", added) + } + if len(stats) > s.max { + log.Printf("[regions] NOTE: %d declared name(s) exceeded maxDerived=%d and were dropped, least-declared first", len(stats)-s.max, s.max) + } +} diff --git a/cmd/ingestor/region_keys_test.go b/cmd/ingestor/region_keys_test.go new file mode 100644 index 000000000..927c37050 --- /dev/null +++ b/cmd/ingestor/region_keys_test.go @@ -0,0 +1,633 @@ +package main + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "fmt" + "sort" + "strings" + "sync" + "testing" +) + +func TestRegionNameAcceptable(t *testing.T) { + cases := []struct { + name string + want string + ok bool + why string + }{ + {"be", "be", true, "a bare name from an OTA answer"}, + {"#be", "be", true, "the same region as nodes.configured_scope spells it — normalizeScopeList adds the '#'"}, + {"nl-li-nth", "nl-li-nth", true, "hyphens are ordinary in region names"}, + {"*", "", false, "the flood wildcard is not a region, and hashing it would invent one nobody declared"}, + {"#*", "", false, "the wildcard stays the wildcard however it is spelled"}, + {"", "", false, "empty"}, + {"#", "", false, "a bare prefix names nothing"}, + {"be,nl", "", false, "a comma would split the name on the next round-trip"}, + {"be#nl", "", false, "a second '#' cannot come from either source intact"}, + {"be nl", "", false, "whitespace would hash bytes nobody intended"}, + {"bé", "", false, "non-ASCII, same reason"}, + {"be" + string(rune(0)), "", false, "NUL is block-cipher padding a stale client failed to trim"}, + {"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "", false, "33 chars, past maxRegionNameLen"}, + } + for _, c := range cases { + got, ok := regionNameAcceptable(c.name) + if ok != c.ok || got != c.want { + t.Errorf("regionNameAcceptable(%q) = (%q, %v), want (%q, %v) — %s", c.name, got, ok, c.want, c.ok, c.why) + } + } +} + +func TestRankDeclaredRegionsPrefersWidelyDeclared(t *testing.T) { + // The cap must drop the long tail of one-off local names, never a region + // half the network declares. On live data "be" is declared by 127 + // repeaters and "behss" by 3. + stats := []declaredRegionStat{ + {Name: "behss", Declarers: 3, LastSeen: "2026-09-07T10:00:00Z"}, + {Name: "be", Declarers: 127, LastSeen: "2026-09-01T10:00:00Z"}, + {Name: "sol3", Declarers: 1, LastSeen: "2026-09-07T11:00:00Z"}, + } + got := rankDeclaredRegions(stats, 2) + want := []string{"be", "behss"} + if len(got) != len(want) { + t.Fatalf("rankDeclaredRegions = %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("rankDeclaredRegions = %v, want %v — declarer count must dominate recency", got, want) + } + } +} + +func TestRankDeclaredRegionsIsDeterministic(t *testing.T) { + // Equal declarer counts and equal timestamps must still produce a stable + // order, or the derived tier churns between refreshes and the add/drop + // logging becomes noise. + stats := []declaredRegionStat{ + {Name: "zz", Declarers: 2, LastSeen: "2026-09-07T10:00:00Z"}, + {Name: "aa", Declarers: 2, LastSeen: "2026-09-07T10:00:00Z"}, + } + for i := 0; i < 20; i++ { + got := rankDeclaredRegions(stats, 10) + if got[0] != "aa" || got[1] != "zz" { + t.Fatalf("run %d: rankDeclaredRegions = %v, want [aa zz]", i, got) + } + } +} + +func TestRankDeclaredRegionsBreaksTiesOnRecency(t *testing.T) { + stats := []declaredRegionStat{ + {Name: "old", Declarers: 2, LastSeen: "2026-01-01T00:00:00Z"}, + {Name: "new", Declarers: 2, LastSeen: "2026-09-07T00:00:00Z"}, + } + got := rankDeclaredRegions(stats, 1) + if len(got) != 1 || got[0] != "new" { + t.Fatalf("rankDeclaredRegions = %v, want [new] — equal declarers break on recency", got) + } +} + +func TestRankDeclaredRegionsDropsUnacceptableNames(t *testing.T) { + stats := []declaredRegionStat{ + {Name: "be", Declarers: 5, LastSeen: "2026-09-07T10:00:00Z"}, + {Name: "b,ad", Declarers: 99, LastSeen: "2026-09-07T10:00:00Z"}, + } + got := rankDeclaredRegions(stats, 10) + if len(got) != 1 || got[0] != "be" { + t.Fatalf("rankDeclaredRegions = %v, want [be] — a name with a comma cannot survive a comma-separated column, however widely declared", got) + } +} + +func TestSplitDeclaredRegionsCSV(t *testing.T) { + // Exact inverse of the strings.Join the ingestor writes the column with. + got := splitDeclaredRegionsCSV(" be , eu ,, nl ") + want := []string{"be", "eu", "nl"} + if len(got) != len(want) { + t.Fatalf("splitDeclaredRegionsCSV = %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("splitDeclaredRegionsCSV = %v, want %v", got, want) + } + } + if n := len(splitDeclaredRegionsCSV("")); n != 0 { + t.Errorf("empty csv produced %d entries, want 0", n) + } +} + +func TestRegionKeySetExplicitOnlyWhenDisabled(t *testing.T) { + // Derivation off: the snapshot must be exactly what loadRegionKeys built, + // and refreshDerived must be a no-op rather than a quiet opt-in. + cfg := &Config{HashRegions: []string{"#be"}} + set := newRegionKeySet(cfg) + set.refreshDerived([]string{"behss", "fm-112"}) + + snap := set.snapshot() + if len(snap.all) != 1 { + t.Fatalf("len(all) = %d, want 1 — refreshDerived must not add keys when disabled", len(snap.all)) + } + if _, ok := snap.all["#be"]; !ok { + t.Error("want the explicit #be key present") + } + if !snap.isExplicit("#be") { + t.Error("isExplicit(#be) = false, want true") + } +} + +func TestRegionKeySetMergesDerivedWhenEnabled(t *testing.T) { + cfg := &Config{ + HashRegions: []string{"#be"}, + AutoRegionKeys: &AutoRegionKeysConfig{Enabled: true}, + } + set := newRegionKeySet(cfg) + set.refreshDerived([]string{"behss", "be"}) // "be" duplicates the explicit key + + snap := set.snapshot() + if len(snap.all) != 2 { + t.Fatalf("len(all) = %d, want 2 (#be explicit + #behss derived), got keys %v", len(snap.all), keyNames(snap)) + } + if _, ok := snap.all["#behss"]; !ok { + t.Errorf("want the derived #behss key present, got %v", keyNames(snap)) + } + if snap.isExplicit("#behss") { + t.Error("isExplicit(#behss) = true, want false — a derived key is not operator config") + } + if !snap.isExplicit("#be") { + t.Error("isExplicit(#be) = false, want true — an explicit key must not be demoted by a duplicate declaration") + } +} + +func TestRegionKeySetRefreshReplacesRatherThanAccumulates(t *testing.T) { + // A region that stops being declared must leave the derived tier, or the + // key set only ever grows and the cap stops meaning anything. + cfg := &Config{AutoRegionKeys: &AutoRegionKeysConfig{Enabled: true}} + set := newRegionKeySet(cfg) + set.refreshDerived([]string{"aa"}) + set.refreshDerived([]string{"bb"}) + + snap := set.snapshot() + if _, ok := snap.all["#aa"]; ok { + t.Error("want #aa gone after a refresh that no longer lists it") + } + if _, ok := snap.all["#bb"]; !ok { + t.Error("want #bb present after the refresh that lists it") + } +} + +func TestRegionKeySetSnapshotIsStable(t *testing.T) { + // A snapshot handed to a packet must not change under it mid-match. + cfg := &Config{AutoRegionKeys: &AutoRegionKeysConfig{Enabled: true}} + set := newRegionKeySet(cfg) + set.refreshDerived([]string{"aa"}) + held := set.snapshot() + set.refreshDerived([]string{"bb"}) + + if _, ok := held.all["#aa"]; !ok { + t.Error("the held snapshot lost #aa — snapshots must be immutable, not aliases of live state") + } +} + +// keyNames is a test helper for readable failure messages. +func keyNames(s *regionKeySnapshot) []string { + out := make([]string, 0, len(s.all)) + for k := range s.all { + out = append(out, k) + } + sort.Strings(out) + return out +} + +// codeFor derives the on-wire code1 a sender in region `name` would emit for +// this payload - the same computation matchingRegions inverts. Used to build +// packets that genuinely belong to a region rather than asserting on a +// hardcoded string. +func codeFor(name string, payloadType byte, payload []byte) string { + if !strings.HasPrefix(name, "#") { + name = "#" + name + } + sum := sha256.Sum256([]byte(name)) + mac := hmac.New(sha256.New, sum[:16]) + mac.Write([]byte{payloadType}) + mac.Write(payload) + h := mac.Sum(nil) + code := uint16(h[0]) | uint16(h[1])<<8 + if code == 0 { + code = 1 + } else if code == 0xFFFF { + code = 0xFFFE + } + return strings.ToUpper(hex.EncodeToString([]byte{byte(code & 0xFF), byte(code >> 8)})) +} + +func TestScopeMatchUniqueNamesTheRegion(t *testing.T) { + payload := []byte{0xDE, 0xAD, 0xBE, 0xEF} + cfg := &Config{HashRegions: []string{"#be"}} + set := newRegionKeySet(cfg) + code := codeFor("#be", 5, payload) + + got := set.snapshot().match(5, payload, code) + if got.Name != "#be" { + t.Errorf("Name = %q, want %q", got.Name, "#be") + } + if got.Reason != scopeReasonUnique { + t.Errorf("Reason = %q, want %q", got.Reason, scopeReasonUnique) + } +} + +func TestScopeMatchNoKeyMatches(t *testing.T) { + cfg := &Config{HashRegions: []string{"#be"}} + set := newRegionKeySet(cfg) + + got := set.snapshot().match(5, []byte{1, 2, 3}, "0000") + if got.Name != "" || got.Reason != scopeReasonNone { + t.Errorf("got %+v, want an empty name with reason %q", got, scopeReasonNone) + } +} + +func TestScopeMatchExplicitBeatsDerived(t *testing.T) { + // The ambiguity this feature introduces: a derived key collides with an + // operator-configured one on this payload. Operator config wins - it is + // intent, the derived name is hearsay picked up over RF. + payload := []byte{0x01, 0x02, 0x03, 0x04} + cfg := &Config{HashRegions: []string{"#be"}, AutoRegionKeys: &AutoRegionKeysConfig{Enabled: true}} + set := newRegionKeySet(cfg) + code := codeFor("#be", 5, payload) + + // Force the collision rather than searching for a natural one: inject a + // derived key whose bytes are the explicit key's, so both match. + snap := set.snapshot() + collide := make(map[string][]byte, len(snap.all)+1) + for k, v := range snap.all { + collide[k] = v + } + collide["#collider"] = snap.all["#be"] + forced := ®ionKeySnapshot{all: collide, explicit: snap.explicit} + + got := forced.match(5, payload, code) + if got.Name != "#be" { + t.Errorf("Name = %q, want %q - the explicit key must win", got.Name, "#be") + } + if got.Reason != scopeReasonExplicitOverDerived { + t.Errorf("Reason = %q, want %q", got.Reason, scopeReasonExplicitOverDerived) + } + if len(got.Candidates) != 2 { + t.Errorf("Candidates = %v, want both names recorded for the log", got.Candidates) + } +} + +func TestScopeMatchTwoExplicitKeysStayAmbiguous(t *testing.T) { + // Two equally-sourced candidates: naming either would be a guess, and + // naming wrongly is worse than not naming. This is #1609's rule, unchanged. + payload := []byte{0x09, 0x08, 0x07} + cfg := &Config{HashRegions: []string{"#be", "#eu"}} + set := newRegionKeySet(cfg) + code := codeFor("#be", 5, payload) + + snap := set.snapshot() + collide := map[string][]byte{"#be": snap.all["#be"], "#eu": snap.all["#be"]} + forced := ®ionKeySnapshot{all: collide, explicit: snap.explicit} + + got := forced.match(5, payload, code) + if got.Name != "" { + t.Errorf("Name = %q, want empty - two explicit candidates must abstain", got.Name) + } + if got.Reason != scopeReasonAmbiguous { + t.Errorf("Reason = %q, want %q", got.Reason, scopeReasonAmbiguous) + } +} + +func TestScopeMatchTwoDerivedKeysStayAmbiguous(t *testing.T) { + // The tier-3 case, deliberately NOT resolved in M2. It must abstain rather + // than pick, and the reason must say ambiguous so the log can measure how + // often this happens before tier 3 is built. + payload := []byte{0x11, 0x22} + cfg := &Config{AutoRegionKeys: &AutoRegionKeysConfig{Enabled: true}} + set := newRegionKeySet(cfg) + set.refreshDerived([]string{"aa", "bb"}) + + snap := set.snapshot() + collide := map[string][]byte{"#aa": snap.all["#aa"], "#bb": snap.all["#aa"]} + forced := ®ionKeySnapshot{all: collide, explicit: snap.explicit} + code := codeFor("#aa", 5, payload) + + got := forced.match(5, payload, code) + if got.Name != "" || got.Reason != scopeReasonAmbiguous { + t.Errorf("got %+v, want an empty name with reason %q", got, scopeReasonAmbiguous) + } +} + +// regionSetFromKeys wraps a raw key map as a *regionKeySet with every key +// treated as explicit. Tests written before M2's two-tier type keep working +// unchanged this way, and "all keys explicit" is precisely what a hashRegions +// map meant back then - so the #1609 ambiguity semantics they assert are +// preserved exactly: two explicit candidates still abstain. +func regionSetFromKeys(keys map[string][]byte) *regionKeySet { + names := make(map[string]bool, len(keys)) + all := make(map[string][]byte, len(keys)) + for n, k := range keys { + names[n] = true + all[n] = k + } + s := ®ionKeySet{} + s.cur.Store(®ionKeySnapshot{all: all, explicit: names}) + return s +} + +// matchScopeName reproduces the removed matchScope's signature over the new +// type, so the tests written against it keep asserting the behaviour they were +// written for rather than being rewritten alongside the change they guard. +func matchScopeName(keys map[string][]byte, payloadType byte, payloadRaw []byte, code1 string) string { + return regionSetFromKeys(keys).snapshot().match(payloadType, payloadRaw, code1).Name +} + +func TestDeclaredRegionSourcesAggregatesLatestAnswerPerTarget(t *testing.T) { + store := newTestStore(t) + // Two answers from the same target: only the newer one counts, exactly as + // CurrentDeclaredRegions orders (by observed_at, never ingested_at - a + // drive buffered offline can arrive days late). + insertDeclaredRegionsRow(t, store, "aa"+strings.Repeat("11", 31), "2026-09-01T00:00:00Z", "be,old") + insertDeclaredRegionsRow(t, store, "aa"+strings.Repeat("11", 31), "2026-09-07T00:00:00Z", "be,new") + insertDeclaredRegionsRow(t, store, "bb"+strings.Repeat("22", 31), "2026-09-05T00:00:00Z", "be") + + stats, err := store.declaredRegionSources() + if err != nil { + t.Fatal(err) + } + byName := map[string]declaredRegionStat{} + for _, s := range stats { + byName[s.Name] = s + } + if got := byName["be"].Declarers; got != 2 { + t.Errorf("be declarers = %d, want 2", got) + } + if got := byName["be"].LastSeen; got != "2026-09-07T00:00:00Z" { + t.Errorf("be lastSeen = %q, want the greatest observed_at", got) + } + if _, ok := byName["old"]; ok { + t.Error("want the superseded answer's region gone - only the latest answer per target counts") + } + if got := byName["new"].Declarers; got != 1 { + t.Errorf("new declarers = %d, want 1", got) + } +} + +func TestDeclaredRegionSourcesIgnoresWildcard(t *testing.T) { + // '*' is the wildcard, not a region name. Deriving a key for it would add + // a permanent no-op entry to the cap on nearly every deployment. + store := newTestStore(t) + insertDeclaredRegionsRow(t, store, "cc"+strings.Repeat("33", 31), "2026-09-07T00:00:00Z", "*,be") + stats, err := store.declaredRegionSources() + if err != nil { + t.Fatal(err) + } + for _, s := range stats { + if s.Name == "*" { + t.Error("want '*' excluded - it is the wildcard, not a region") + } + } + if len(stats) != 1 { + t.Errorf("stats = %+v, want just be", stats) + } +} + +// insertDeclaredRegionsRow seeds one node_declared_regions answer, creating +// the table first. That table is optional: this ingestor never creates it, and +// declaredRegionSources probes sqlite_master before reading it. A test that +// wants the optional source has to bring it. +func insertDeclaredRegionsRow(t *testing.T, s *Store, target, observedAt, regionsCSV string) { + t.Helper() + if _, err := s.db.Exec( + `INSERT INTO node_declared_regions (target, rx_pubkey, observed_at, ingested_at, regions_csv, truncated) + VALUES ($1, 'rx', $2, $3, $4, 0)`, target, observedAt, observedAt, regionsCSV); err != nil { + t.Fatal(err) + } +} + +func TestRefreshFromStoreIsNoOpWhenDisabled(t *testing.T) { + store := newTestStore(t) + insertDeclaredRegionsRow(t, store, "aa"+strings.Repeat("11", 31), "2026-09-07T00:00:00Z", "behss") + + cfg := &Config{HashRegions: []string{"#be"}} // autoRegionKeys absent + set := newRegionKeySet(cfg) + before := len(set.snapshot().all) + set.refreshFromStore(store) + + if got := len(set.snapshot().all); got != before { + t.Errorf("key count %d -> %d with autoRegionKeys off, want unchanged", before, got) + } + if _, ok := set.snapshot().all["#behss"]; ok { + t.Error("a declared name became a key with the feature disabled - this is the safety property the default-off promise rests on") + } +} + +func TestRefreshFromStoreDerivesWhenEnabled(t *testing.T) { + store := newTestStore(t) + insertDeclaredRegionsRow(t, store, "aa"+strings.Repeat("11", 31), "2026-09-07T00:00:00Z", "behss,fm-112") + + cfg := &Config{HashRegions: []string{"#be"}, AutoRegionKeys: &AutoRegionKeysConfig{Enabled: true}} + set := newRegionKeySet(cfg) + set.refreshFromStore(store) + + snap := set.snapshot() + for _, want := range []string{"#be", "#behss", "#fm-112"} { + if _, ok := snap.all[want]; !ok { + t.Errorf("want %s in force, got %v", want, keyNames(snap)) + } + } + if !snap.isExplicit("#be") || snap.isExplicit("#behss") { + t.Error("tiers crossed: #be must stay explicit, #behss must be derived") + } +} + +func TestRefreshFromStoreHonoursTheCap(t *testing.T) { + store := newTestStore(t) + // Three names, one declared twice so the ranking is not a coin flip. + insertDeclaredRegionsRow(t, store, "aa"+strings.Repeat("11", 31), "2026-09-07T00:00:00Z", "wide,narrow1") + insertDeclaredRegionsRow(t, store, "bb"+strings.Repeat("22", 31), "2026-09-07T00:00:00Z", "wide,narrow2") + + cfg := &Config{AutoRegionKeys: &AutoRegionKeysConfig{Enabled: true, MaxDerived: 1}} + set := newRegionKeySet(cfg) + set.refreshFromStore(store) + + snap := set.snapshot() + if len(snap.all) != 1 { + t.Fatalf("keys = %v, want exactly 1 under maxDerived=1", keyNames(snap)) + } + if _, ok := snap.all["#wide"]; !ok { + t.Errorf("keys = %v, want the twice-declared name kept, not a one-off", keyNames(snap)) + } +} + +// BenchmarkScopeMatch sweeps key-set size because the cost is linear in it and +// cannot be reduced: code1 is an HMAC over the packet payload, so there is no +// payload-independent lookup key to index on. The sweep is the evidence for +// choosing maxDerived, not a single before/after number - the explicit tier's +// size is operator config and varies per deployment. +func BenchmarkScopeMatch(b *testing.B) { + payload := make([]byte, 51) // a typical GRP_TXT payload + for i := range payload { + payload[i] = byte(i) + } + for _, n := range []int{16, 58, 180, 314} { + b.Run(fmt.Sprintf("keys=%d", n), func(b *testing.B) { + all := make(map[string][]byte, n) + explicit := make(map[string]bool, n) + for i := 0; i < n; i++ { + name := fmt.Sprintf("#r%04d", i) + sum := sha256.Sum256([]byte(name)) + all[name] = sum[:16] + explicit[name] = true + } + snap := ®ionKeySnapshot{all: all, explicit: explicit} + code := codeFor("#r0000", 5, payload) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _ = snap.match(5, payload, code) + } + }) + } +} + +// TestRegionKeySetConcurrentRefreshAndMatch drives the shape the type exists +// for and that nothing else in this suite produces: ingest goroutines naming +// packets while the refresh ticker swaps the key set underneath them. +// +// It is written for `go test -race`. Run without it, it asserts only that the +// explicit tier keeps naming its packet across 200 swaps, which is worth +// little; run with it, the detector sees every unsynchronised read of the +// snapshot that a real refresh would expose. Until this existed, a race run of +// this package proved nothing about regionKeySet, because refreshDerived and +// match were never called concurrently anywhere in it — the absence of a +// finding was the absence of an experiment. +// +// The final assertion holds regardless of what the churn derives: a derived key +// colliding with #be on this payload resolves to the explicit key by tier 2, +// and #be is the only explicit key here, so there is no second explicit +// candidate that could make the answer ambiguous. +func TestRegionKeySetConcurrentRefreshAndMatch(t *testing.T) { + const explicitName = "#be" + payload := []byte{0x11, 0x22, 0x33, 0x44} + code1 := codeFor(explicitName, 5, payload) + + set := newRegionKeySet(&Config{ + HashRegions: []string{explicitName}, + AutoRegionKeys: &AutoRegionKeysConfig{Enabled: true}, + }) + + var wg sync.WaitGroup + stop := make(chan struct{}) + + for i := 0; i < 8; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for { + select { + case <-stop: + return + default: + } + // snapshot() then match() is exactly the ingest hot path: + // one atomic load, then reads of the maps it points at. + if got := set.snapshot().match(5, payload, code1); got.Name != explicitName && got.Reason != scopeReasonAmbiguous { + // Reported rather than fataled: t.Fatalf from a + // non-test goroutine is undefined behaviour. + t.Errorf("match during refresh returned %q (%s), want %q", got.Name, got.Reason, explicitName) + return + } + } + }() + } + + wg.Add(1) + go func() { + defer wg.Done() + defer close(stop) + for i := 0; i < 200; i++ { + // A different derived set each time, so every iteration really + // builds and swaps a new snapshot rather than reusing one. + set.refreshDerived([]string{fmt.Sprintf("r%03d", i%7), fmt.Sprintf("q%03d", i)}) + } + }() + + wg.Wait() + + if got := set.snapshot().match(5, payload, code1); got.Name != explicitName { + t.Errorf("after 200 refreshes match = %q (%s), want %q — the explicit tier must survive every swap", + got.Name, got.Reason, explicitName) + } +} + +// TestDeclaredRegionSourcesReadsConfiguredScope is the upstream-native source: +// nodes.configured_scope, written by the observer /neighbors ingestion. It is +// the only source a stock install has, so a derived tier that read only the +// optional table would be permanently empty here. +// +// The stored form carries the leading "#" (normalizeScopeList puts it there), +// which is exactly the spelling regionNameAcceptable has to accept. +func TestDeclaredRegionSourcesReadsConfiguredScope(t *testing.T) { + store := newTestStore(t) + seedNodeWithConfiguredScope(t, store, "aa"+strings.Repeat("11", 31), "#be,#nl-li,*", "2026-09-07T00:00:00Z") + seedNodeWithConfiguredScope(t, store, "bb"+strings.Repeat("22", 31), "#be", "2026-09-08T00:00:00Z") + + stats, err := store.declaredRegionSources() + if err != nil { + t.Fatal(err) + } + byName := map[string]declaredRegionStat{} + for _, s := range stats { + byName[s.Name] = s + } + if got := byName["#be"].Declarers; got != 2 { + t.Errorf("#be declarers = %d, want 2 — both nodes declare it", got) + } + if got := byName["#be"].LastSeen; got != "2026-09-08T00:00:00Z" { + t.Errorf("#be lastSeen = %q, want the newer of the two answers", got) + } + if got := byName["#nl-li"].Declarers; got != 1 { + t.Errorf("#nl-li declarers = %d, want 1", got) + } + if _, ok := byName["*"]; ok { + t.Error("want '*' excluded from the declared-name count — it is the wildcard, and nearly every node declares it") + } +} + +// TestDeclaredRegionSourcesMergesBothSources: an instance that has both the +// column and the optional table must see a name declared in either, and count +// a node once per name rather than once per source. +func TestDeclaredRegionSourcesMergesBothSources(t *testing.T) { + store := newTestStore(t) + pk := "cc" + strings.Repeat("33", 31) + seedNodeWithConfiguredScope(t, store, pk, "#be", "2026-09-07T00:00:00Z") + insertDeclaredRegionsRow(t, store, pk, "2026-09-08T00:00:00Z", "fm-112") + + stats, err := store.declaredRegionSources() + if err != nil { + t.Fatal(err) + } + names := map[string]int{} + for _, s := range stats { + names[s.Name] = s.Declarers + } + if names["#be"] != 1 { + t.Errorf("#be declarers = %d, want 1 (from configured_scope)", names["#be"]) + } + if names["fm-112"] != 1 { + t.Errorf("fm-112 declarers = %d, want 1 (from the optional table)", names["fm-112"]) + } + // Both spellings of the same region reach the same derived key, which is + // what regionNameAcceptable's "#"-stripping is for. Asserted there rather + // than here; this only pins that neither source is dropped. +} + +// seedNodeWithConfiguredScope writes one node with a confirmed scope list, the +// shape handleNeighborsReport produces. +func seedNodeWithConfiguredScope(t *testing.T, s *Store, pubkey, scopes, at string) { + t.Helper() + if _, err := s.db.Exec( + `INSERT INTO nodes (public_key, configured_scope, configured_scope_at) VALUES ($1, $2, $3)`, + pubkey, scopes, at); err != nil { + t.Fatal(err) + } +} diff --git a/cmd/ingestor/regions_gate_test.go b/cmd/ingestor/regions_gate_test.go new file mode 100644 index 000000000..b2e4fc10c --- /dev/null +++ b/cmd/ingestor/regions_gate_test.go @@ -0,0 +1,126 @@ +package main + +import "testing" + +// clientRegionsMsg builds a valid region-discovery answer on the dedicated +// topic meshcore/client//regions. +func clientRegionsMsg() *mockMessage { + payload := []byte(`{"type":"REGIONS","timestamp":"2026-06-09T12:00:00Z","target":"` + testTargetPK + `","regions":["be","be-vlg"],"truncated":false,"gps":{"lat":51.05,"lon":3.72,"acc_m":8.0}}`) + return &mockMessage{topic: "meshcore/client/" + testCompanionPK + "/regions", payload: payload} +} + +// clientRegionsMsgWithRaw is clientRegionsMsg plus a "raw" field (same advert +// hex as clientCoverageMsg). The real app never sends "raw" on /regions, but +// the fall-through guard must be structural — independent of payload shape — +// so this proves the dispatch itself (not payload luck) is what keeps a +// /regions message with an incidental "raw" field from being decoded as an +// ordinary observer packet were the gate ever to fall through. +func clientRegionsMsgWithRaw() *mockMessage { + advertHex := "11451000D818206D3AAC152C8A91F89957E6D30CA51F36E28790228971C473B755F244F718754CF5EE4A2FD58D944466E42CDED140C66D0CC590183E32BAF40F112BE8F3F2BDF6012B4B2793C52F1D36F69EE054D9A05593286F78453E56C0EC4A3EB95DDA2A7543FCCC00B939CACC009278603902FC12BCF84B706120526F6F6620536F6C6172" + payload := []byte(`{"raw":"` + advertHex + `","direction":"rx","type":"REGIONS","timestamp":"2026-06-09T12:00:00Z","origin":"MyMob","target":"` + testTargetPK + `","regions":["be"],"truncated":false,"gps":{"lat":51.05,"lon":3.72,"acc_m":8.0}}`) + return &mockMessage{topic: "meshcore/client/" + testCompanionPK + "/regions", payload: payload} +} + +// TestClientRegionsEnabledDefault verifies the gate helper defaults OFF for +// nil/absent config and is only true when explicitly enabled. +func TestClientRegionsEnabledDefault(t *testing.T) { + if (&Config{}).ClientRegionsEnabled() { + t.Fatal("nil ClientRegions must report disabled") + } + if (&Config{ClientRegions: &ClientRegionsConfig{Enabled: false}}).ClientRegionsEnabled() { + t.Fatal("Enabled:false must report disabled") + } + if !(&Config{ClientRegions: &ClientRegionsConfig{Enabled: true}}).ClientRegionsEnabled() { + t.Fatal("Enabled:true must report enabled") + } +} + +// TestClientRegionsGateOn drives handleMessage with the feature ON: the +// /regions topic message must be dispatched and write exactly one row. +func TestClientRegionsGateOn(t *testing.T) { + store := newTestStore(t) + source := MQTTSource{Name: "test"} + cfg := &Config{ClientRegions: &ClientRegionsConfig{Enabled: true}} + + handleMessage(store, "test", source, clientRegionsMsg(), nil, nil, cfg) + + if n := declaredRegionsCount(t, store); n != 1 { + t.Fatalf("feature ON: expected 1 node_declared_regions row, got %d", n) + } +} + +// TestClientRegionsGateOffDoesNotFallThroughToObserverPath is the +// dispatch-shape regression test for the /regions topic, mirroring +// TestClientRfSamplesGateOffDoesNotFallThroughToObserverPath. With the +// feature OFF, a meshcore/client//regions message must be dropped +// outright — never fall through to the observer packet path, which would +// take parts[1] ("client") as a region and parts[2] (the companion pubkey) +// as an observer id. +func TestClientRegionsGateOffDoesNotFallThroughToObserverPath(t *testing.T) { + store := newTestStore(t) + source := MQTTSource{Name: "test"} + cfg := &Config{} // ClientRegions nil ⇒ disabled + + handleMessage(store, "test", source, clientRegionsMsgWithRaw(), nil, nil, cfg) + + if n := declaredRegionsCount(t, store); n != 0 { + t.Fatalf("feature OFF: expected 0 node_declared_regions rows, got %d", n) + } + var observerRows int + if err := store.db.QueryRow(`SELECT COUNT(*) FROM observers WHERE id = $1`, testCompanionPK).Scan(&observerRows); err != nil { + t.Fatal(err) + } + if observerRows != 0 { + t.Fatalf("feature OFF: the companion pubkey must not be registered as an observer, got %d rows", observerRows) + } + var clientRegionRows int + if err := store.db.QueryRow(`SELECT COUNT(*) FROM observers WHERE iata = 'client'`).Scan(&clientRegionRows); err != nil { + t.Fatal(err) + } + if clientRegionRows != 0 { + t.Fatalf("feature OFF: no observer should be registered under a bogus 'client' region, got %d rows", clientRegionRows) + } + var txRows int + if err := store.db.QueryRow(`SELECT COUNT(*) FROM transmissions`).Scan(&txRows); err != nil { + t.Fatal(err) + } + if txRows != 0 { + t.Fatalf("feature OFF: the client-topic message must not be ingested as an ordinary observer packet, got %d transmissions rows", txRows) + } +} + +// TestClientRegionsBlacklistedDropped verifies a blacklisted operator cannot +// skirt the observer blacklist via the /regions topic, mirroring +// TestClientRfSamplesBlacklistedDropped. +func TestClientRegionsBlacklistedDropped(t *testing.T) { + store := newTestStore(t) + source := MQTTSource{Name: "test"} + cfg := &Config{ + ClientRegions: &ClientRegionsConfig{Enabled: true}, + ObserverBlacklist: []string{testCompanionPK}, + } + + handleMessage(store, "test", source, clientRegionsMsg(), nil, nil, cfg) + + if n := declaredRegionsCount(t, store); n != 0 { + t.Fatalf("blacklisted companion: expected 0 node_declared_regions rows, got %d", n) + } +} + +// TestClientOtherGatesUnaffectedByRegionsGate verifies the client sub-topic +// gates are independent: /regions disabled must not block /packets when +// clientRxCoverage is enabled. +func TestClientOtherGatesUnaffectedByRegionsGate(t *testing.T) { + store := newTestStore(t) + source := MQTTSource{Name: "test"} + cfg := &Config{ + ClientRxCoverage: &ClientRxCoverageConfig{Enabled: true}, + ClientRegions: &ClientRegionsConfig{Enabled: false}, + } + + handleMessage(store, "test", source, clientCoverageMsg(), nil, nil, cfg) + + if n := clientReceptionCount(t, store); n != 1 { + t.Fatalf("expected 1 client_receptions row with clientRxCoverage on / clientRegions off, got %d", n) + } +} diff --git a/cmd/ingestor/relay_touch_test.go b/cmd/ingestor/relay_touch_test.go index 53fa23480..7bbf8869f 100644 --- a/cmd/ingestor/relay_touch_test.go +++ b/cmd/ingestor/relay_touch_test.go @@ -29,7 +29,7 @@ func seedRelayNode(t *testing.T, s *Store, pubkey, name, lastSeen string) { func nodeLastSeen(t *testing.T, s *Store, pubkey string) string { t.Helper() var ls string - if err := s.db.QueryRow(`SELECT COALESCE(last_seen,'') FROM nodes WHERE public_key=?`, pubkey).Scan(&ls); err != nil { + if err := s.db.QueryRow(`SELECT COALESCE(last_seen,'') FROM nodes WHERE public_key=$1`, pubkey).Scan(&ls); err != nil { t.Fatalf("read last_seen for %s: %v", pubkey, err) } return ls diff --git a/cmd/ingestor/resolved_path_test.go b/cmd/ingestor/resolved_path_test.go index 308868506..cbc07da79 100644 --- a/cmd/ingestor/resolved_path_test.go +++ b/cmd/ingestor/resolved_path_test.go @@ -95,7 +95,7 @@ func TestInsertTransmissionWritesResolvedPath(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "ingest.db") - store, err := OpenStore(dbPath) + store, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore: %v", err) } @@ -103,7 +103,7 @@ func TestInsertTransmissionWritesResolvedPath(t *testing.T) { // Seed nodes with unique 1-byte prefixes. if _, err := store.db.Exec( - `INSERT INTO nodes (public_key, name) VALUES (?, ?), (?, ?)`, + `INSERT INTO nodes (public_key, name) VALUES ($1, $2), ($3, $4)`, "aaaaaaaaaa", "from-node", "bbbbbbbbbb", "first-hop", ); err != nil { @@ -138,7 +138,7 @@ func TestInsertTransmissionWritesResolvedPath(t *testing.T) { var rp sql.NullString if err := store.db.QueryRow( - `SELECT resolved_path FROM observations WHERE transmission_id = (SELECT id FROM transmissions WHERE hash = ?)`, + `SELECT resolved_path FROM observations WHERE transmission_id = (SELECT id FROM transmissions WHERE hash = $1)`, "h-1547", ).Scan(&rp); err != nil { t.Fatalf("query: %v", err) @@ -374,14 +374,14 @@ func TestInsertTransmissionDoesNotClobberResolvedPathOnAllNil(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "ingest.db") - store, err := OpenStore(dbPath) + store, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore: %v", err) } defer store.Close() if _, err := store.db.Exec( - `INSERT INTO nodes (public_key, name) VALUES (?, ?), (?, ?)`, + `INSERT INTO nodes (public_key, name) VALUES ($1, $2), ($3, $4)`, "aaaaaaaaaa", "from-node", "bbbbbbbbbb", "first-hop", ); err != nil { @@ -412,7 +412,7 @@ func TestInsertTransmissionDoesNotClobberResolvedPathOnAllNil(t *testing.T) { // Sanity: first write populated resolved_path. var first sql.NullString if err := store.db.QueryRow( - `SELECT resolved_path FROM observations WHERE transmission_id = (SELECT id FROM transmissions WHERE hash = ?)`, + `SELECT resolved_path FROM observations WHERE transmission_id = (SELECT id FROM transmissions WHERE hash = $1)`, "h-clobber", ).Scan(&first); err != nil { t.Fatalf("first query: %v", err) @@ -432,7 +432,7 @@ func TestInsertTransmissionDoesNotClobberResolvedPathOnAllNil(t *testing.T) { var after sql.NullString if err := store.db.QueryRow( - `SELECT resolved_path FROM observations WHERE transmission_id = (SELECT id FROM transmissions WHERE hash = ?)`, + `SELECT resolved_path FROM observations WHERE transmission_id = (SELECT id FROM transmissions WHERE hash = $1)`, "h-clobber", ).Scan(&after); err != nil { t.Fatalf("post-reingest query: %v", err) diff --git a/cmd/ingestor/retained_status_test.go b/cmd/ingestor/retained_status_test.go index 64a7ffc47..5ad07a574 100644 --- a/cmd/ingestor/retained_status_test.go +++ b/cmd/ingestor/retained_status_test.go @@ -26,7 +26,7 @@ func seedObserver(t *testing.T, s *Store, id string, ageDays int) string { t.Fatal(err) } old := time.Now().UTC().AddDate(0, 0, -ageDays).Format(time.RFC3339) - if _, err := s.db.Exec(`UPDATE observers SET last_seen = ? WHERE id = ?`, old, id); err != nil { + if _, err := s.db.Exec(`UPDATE observers SET last_seen = $1 WHERE id = $2`, old, id); err != nil { t.Fatal(err) } return old @@ -35,7 +35,7 @@ func seedObserver(t *testing.T, s *Store, id string, ageDays int) string { func observerLastSeen(t *testing.T, s *Store, id string) string { t.Helper() var got string - if err := s.db.QueryRow(`SELECT last_seen FROM observers WHERE id = ?`, id).Scan(&got); err != nil { + if err := s.db.QueryRow(`SELECT last_seen FROM observers WHERE id = $1`, id).Scan(&got); err != nil { t.Fatal(err) } return got @@ -57,7 +57,7 @@ func TestRetainedStatusDoesNotAdvanceLastSeen(t *testing.T) { func TestRetainedStatusDoesNotReactivateInactiveObserver(t *testing.T) { store := newTestStore(t) seedObserver(t, store, "obs-retained", 60) - if _, err := store.db.Exec(`UPDATE observers SET inactive = 1 WHERE id = ?`, "obs-retained"); err != nil { + if _, err := store.db.Exec(`UPDATE observers SET inactive = 1 WHERE id = $1`, "obs-retained"); err != nil { t.Fatal(err) } @@ -66,7 +66,7 @@ func TestRetainedStatusDoesNotReactivateInactiveObserver(t *testing.T) { nil, nil, &Config{}) var inactive int - if err := store.db.QueryRow(`SELECT inactive FROM observers WHERE id = ?`, "obs-retained").Scan(&inactive); err != nil { + if err := store.db.QueryRow(`SELECT inactive FROM observers WHERE id = $1`, "obs-retained").Scan(&inactive); err != nil { t.Fatal(err) } if inactive != 1 { @@ -85,7 +85,7 @@ func TestRetainedStatusDoesNotCreateUnknownObserver(t *testing.T) { nil, nil, &Config{}) var count int - if err := store.db.QueryRow(`SELECT COUNT(*) FROM observers WHERE id = ?`, "obs-never-seen").Scan(&count); err != nil { + if err := store.db.QueryRow(`SELECT COUNT(*) FROM observers WHERE id = $1`, "obs-never-seen").Scan(&count); err != nil { t.Fatal(err) } if count != 0 { @@ -105,7 +105,7 @@ func TestRetainedStatusDoesNotInsertMetricsSample(t *testing.T) { var count int if err := store.db.QueryRow( - `SELECT COUNT(*) FROM observer_metrics WHERE observer_id = ?`, "obs-retained", + `SELECT COUNT(*) FROM observer_metrics WHERE observer_id = $1`, "obs-retained", ).Scan(&count); err != nil { t.Fatal(err) } @@ -127,7 +127,7 @@ func TestRetainedStatusStillUpdatesMetadata(t *testing.T) { var firmware string var noiseFloor float64 if err := store.db.QueryRow( - `SELECT firmware, noise_floor FROM observers WHERE id = ?`, "obs-retained", + `SELECT firmware, noise_floor FROM observers WHERE id = $1`, "obs-retained", ).Scan(&firmware, &noiseFloor); err != nil { t.Fatal(err) } @@ -162,7 +162,7 @@ func TestLiveStatusStillCreatesUnknownObserver(t *testing.T) { nil, nil, &Config{}) var count int - if err := store.db.QueryRow(`SELECT COUNT(*) FROM observers WHERE id = ?`, "obs-fresh").Scan(&count); err != nil { + if err := store.db.QueryRow(`SELECT COUNT(*) FROM observers WHERE id = $1`, "obs-fresh").Scan(&count); err != nil { t.Fatal(err) } if count != 1 { diff --git a/cmd/ingestor/retention_yield_test.go b/cmd/ingestor/retention_yield_test.go new file mode 100644 index 000000000..3ad45d8ca --- /dev/null +++ b/cmd/ingestor/retention_yield_test.go @@ -0,0 +1,103 @@ +package main + +import ( + "context" + "fmt" + "testing" + "time" +) + +func TestPruneYieldsAfterCommittedFullBatches(t *testing.T) { + s := openPruneStore(t, "yield.db") + const aged = pruneBatchTransmissions*2 + 7 + seedRetentionFanout(t, s, aged, aged) + yields := 0 + removed, err := s.pruneOldPackets(5, func() { + yields++ + if !writerMu.TryLock() { + t.Error("scheduling action still holds writerMu") + return + } + writerMu.Unlock() + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + conn, err := s.db.Conn(ctx) + if err != nil { + t.Error("scheduling action still occupies the single DB connection", err) + return + } + var remaining int + err = conn.QueryRowContext(ctx, `SELECT count(*) FROM transmissions WHERE first_seen < '2021-01-01'`).Scan(&remaining) + conn.Close() + if err != nil || remaining != aged-yields*pruneBatchTransmissions { + t.Error("scheduling action preceded a committed full batch", remaining, err) + return + } + done := make(chan error, 1) + sequence := yields + go func() { + _, err := s.InsertTransmission(&PacketData{ + RawHex: "0A00D69FD7A5A7475DB07337749AE61FA53A4788E976", + Hash: fmt.Sprint("scheduled-live-", sequence), Timestamp: time.Now().UTC().Format(time.RFC3339), + RouteType: 2, PayloadType: 2, PathJSON: "[]", DecodedJSON: "{}", + }) + done <- err + }() + select { + case err := <-done: + if err != nil { + t.Error(err) + } + case <-ctx.Done(): + t.Error("foreground ingestion could not finish between committed batches") + } + }) + if err != nil || removed != aged { + t.Fatalf("removed=%d err=%v", removed, err) + } + if yields != 2 { + t.Errorf("scheduling action called %d times; want once after each of two full committed batches", yields) + } + if countRows(t, s, "transmissions") != 2 || countRows(t, s, "observations") != 2 { + t.Error("foreground rows were not ingested and preserved between batches") + } +} + +func seedRetentionFanout(tb testing.TB, s *Store, transmissions, observations int) { + tb.Helper() + s.WaitForAsyncMigrations() + tx, err := s.db.Begin() + if err != nil { + tb.Fatal(err) + } + defer tx.Rollback() + parents, err := tx.Prepare(`INSERT INTO transmissions(id,raw_hex,hash,first_seen,last_seen) VALUES($1,'00',$2,'2020-01-01T00:00:00Z',1)`) + if err != nil { + tb.Fatal(err) + } + defer parents.Close() + children, err := tx.Prepare(`INSERT INTO observations(transmission_id,observer_idx,path_json,timestamp) VALUES($1,$2,'[]',1)`) + if err != nil { + tb.Fatal(err) + } + defer children.Close() + for i := 1; i <= transmissions; i++ { + if _, err := parents.Exec(i, fmt.Sprint("retention-", i)); err != nil { + tb.Fatal(err) + } + } + for i := 0; i < observations; i++ { + if _, err := children.Exec(i%transmissions+1, i/transmissions); err != nil { + tb.Fatal(err) + } + } + + if err := tx.Commit(); err != nil { + tb.Fatal(err) + } + if s.Backend() == "postgres" { + if _, err := testAdmin(tb, s).Exec(`SELECT setval('transmissions_id_seq',$1,true)`, transmissions); err != nil { + tb.Fatal(err) + } + } +} diff --git a/cmd/ingestor/rf_gate_test.go b/cmd/ingestor/rf_gate_test.go index 4b6f8dee7..8f05afe6f 100644 --- a/cmd/ingestor/rf_gate_test.go +++ b/cmd/ingestor/rf_gate_test.go @@ -76,7 +76,7 @@ func TestClientRfSamplesGateOffDoesNotFallThroughToObserverPath(t *testing.T) { t.Fatalf("feature OFF: expected 0 client_rf_samples rows, got %d", n) } var observerRows int - if err := store.db.QueryRow(`SELECT COUNT(*) FROM observers WHERE id = ?`, testCompanionPK).Scan(&observerRows); err != nil { + if err := store.db.QueryRow(`SELECT COUNT(*) FROM observers WHERE id = $1`, testCompanionPK).Scan(&observerRows); err != nil { t.Fatal(err) } if observerRows != 0 { diff --git a/cmd/ingestor/scope_match_tally.go b/cmd/ingestor/scope_match_tally.go new file mode 100644 index 000000000..813cbf029 --- /dev/null +++ b/cmd/ingestor/scope_match_tally.go @@ -0,0 +1,73 @@ +package main + +import ( + "database/sql" + "errors" + "time" +) + +// The scope-match tally lives in scopeMatchCounters, four atomics in this +// process and nowhere else. Every restart — a deploy, a crash, an operator +// bouncing the container — sets them back to zero and takes the container's +// log with it, so the previous count is not recoverable afterwards. That is +// how a finished 24h ambiguity measurement was lost. +// +// The numbers only say something over days: "ambiguous is 0.4% of matches" +// needs a denominator large enough to be stable, and the decision they gate +// (whether a collision tie-break is worth building) is not one to take on an +// hour of traffic. So they have to outlive the process doing the counting. +// +// scope_match_totals is one row. The ingestor reads it into the counters at +// startup and writes them back periodically and on shutdown, which makes the +// tally cumulative since `since_unix` instead of since boot. It is deliberately +// included in the canonical telemetry schema owned by the migration command. + +// LoadScopeMatchTotals seeds the in-process counters from the stored row so +// counting continues where the previous process stopped. On a DB that has no +// row yet it anchors `since` at now and stores the zeroes, so the very first +// window has a start time too — without it, a tally read after one restart +// would have no honest denominator. +func (s *Store) LoadScopeMatchTotals() error { + var since, uniq, explicit, ambiguous, none int64 + err := s.db.QueryRow(` + SELECT since_unix, unique_matches, explicit_over_derived, ambiguous, none_matches + FROM scope_match_totals WHERE id = 1`).Scan(&since, &uniq, &explicit, &ambiguous, &none) + if errors.Is(err, sql.ErrNoRows) { + scopeMatchCounters.sinceUnix.Store(time.Now().Unix()) + return s.SaveScopeMatchTotals() + } + if err != nil { + return err + } + scopeMatchCounters.sinceUnix.Store(since) + scopeMatchCounters.unique.Store(uniq) + scopeMatchCounters.explicitOverDerived.Store(explicit) + scopeMatchCounters.ambiguous.Store(ambiguous) + scopeMatchCounters.none.Store(none) + return nil +} + +// SaveScopeMatchTotals writes the current counters to the single row. Each +// atomic is loaded independently, so a save taken under live traffic can miss +// a match that lands mid-write; the next save picks it up. The tally is a +// long-window ratio, not an audit trail. +func (s *Store) SaveScopeMatchTotals() error { + _, err := s.db.Exec(` + INSERT INTO scope_match_totals + (id, since_unix, unique_matches, explicit_over_derived, ambiguous, none_matches, updated_unix) + VALUES (1, `+s.parameter(1)+`, `+s.parameter(2)+`, `+s.parameter(3)+`, `+s.parameter(4)+`, `+s.parameter(5)+`, `+s.parameter(6)+`) + ON CONFLICT(id) DO UPDATE SET + since_unix = excluded.since_unix, + unique_matches = excluded.unique_matches, + explicit_over_derived = excluded.explicit_over_derived, + ambiguous = excluded.ambiguous, + none_matches = excluded.none_matches, + updated_unix = excluded.updated_unix`, + scopeMatchCounters.sinceUnix.Load(), + scopeMatchCounters.unique.Load(), + scopeMatchCounters.explicitOverDerived.Load(), + scopeMatchCounters.ambiguous.Load(), + scopeMatchCounters.none.Load(), + time.Now().Unix()) + return err +} diff --git a/cmd/ingestor/scope_match_tally_test.go b/cmd/ingestor/scope_match_tally_test.go new file mode 100644 index 000000000..3332797dd --- /dev/null +++ b/cmd/ingestor/scope_match_tally_test.go @@ -0,0 +1,177 @@ +package main + +import ( + "testing" + "time" +) + +// resetScopeMatchCounters zeroes the package-level tally and puts it back +// afterwards. The counters are global and OpenStore restores them from the +// DB, so every test here has to start from a known state; no test in this +// package calls t.Parallel, which is what makes that safe. +func resetScopeMatchCounters(t *testing.T) { + t.Helper() + prevUnique := scopeMatchCounters.unique.Load() + prevExplicit := scopeMatchCounters.explicitOverDerived.Load() + prevAmbiguous := scopeMatchCounters.ambiguous.Load() + prevNone := scopeMatchCounters.none.Load() + prevSince := scopeMatchCounters.sinceUnix.Load() + t.Cleanup(func() { + scopeMatchCounters.unique.Store(prevUnique) + scopeMatchCounters.explicitOverDerived.Store(prevExplicit) + scopeMatchCounters.ambiguous.Store(prevAmbiguous) + scopeMatchCounters.none.Store(prevNone) + scopeMatchCounters.sinceUnix.Store(prevSince) + }) + scopeMatchCounters.unique.Store(0) + scopeMatchCounters.explicitOverDerived.Store(0) + scopeMatchCounters.ambiguous.Store(0) + scopeMatchCounters.none.Store(0) + scopeMatchCounters.sinceUnix.Store(0) +} + +// TestScopeMatchTallySurvivesReopen is the whole point of the table: a +// restart used to reset the tally to zero and delete the container log that +// held the previous number, so a measurement in progress was unrecoverable. +func TestScopeMatchTallySurvivesReopen(t *testing.T) { + resetScopeMatchCounters(t) + dbPath := t.TempDir() + "/tally.db" + + s, err := openPostgresTestStore(t, dbPath) + if err != nil { + t.Fatal(err) + } + since := scopeMatchCounters.sinceUnix.Load() + scopeMatchCounters.unique.Store(19692) + scopeMatchCounters.ambiguous.Store(80) + scopeMatchCounters.explicitOverDerived.Store(3) + scopeMatchCounters.none.Store(41) + if err := s.SaveScopeMatchTotals(); err != nil { + t.Fatal(err) + } + s.Close() + + // A new process starts with zeroed counters. + scopeMatchCounters.unique.Store(0) + scopeMatchCounters.ambiguous.Store(0) + scopeMatchCounters.explicitOverDerived.Store(0) + scopeMatchCounters.none.Store(0) + scopeMatchCounters.sinceUnix.Store(0) + + s2, err := openPostgresTestStore(t, dbPath) + if err != nil { + t.Fatal(err) + } + defer s2.Close() + + if got := scopeMatchCounters.unique.Load(); got != 19692 { + t.Errorf("unique = %d, want 19692 restored from the previous run", got) + } + if got := scopeMatchCounters.ambiguous.Load(); got != 80 { + t.Errorf("ambiguous = %d, want 80 restored from the previous run", got) + } + if got := scopeMatchCounters.explicitOverDerived.Load(); got != 3 { + t.Errorf("explicitOverDerived = %d, want 3", got) + } + if got := scopeMatchCounters.none.Load(); got != 41 { + t.Errorf("none = %d, want 41", got) + } + if got := scopeMatchCounters.sinceUnix.Load(); got != since { + t.Errorf("sinceUnix = %d, want %d — the window start must carry across the restart, or the ratio has no denominator", got, since) + } +} + +// TestScopeMatchTallyAnchorsSinceOnFirstOpen pins that a DB with no row gets +// one immediately. Without it the first window's start would only be written +// once something had already been counted, and a tally read after one restart +// would report a rate over an unknown period. +func TestScopeMatchTallyAnchorsSinceOnFirstOpen(t *testing.T) { + resetScopeMatchCounters(t) + before := time.Now().Unix() + + s, err := openPostgresTestStore(t, t.TempDir()+"/tally.db") + if err != nil { + t.Fatal(err) + } + defer s.Close() + + since := scopeMatchCounters.sinceUnix.Load() + if since < before || since > time.Now().Unix() { + t.Errorf("sinceUnix = %d, want a stamp between %d and now", since, before) + } + var rows int + if err := s.db.QueryRow(`SELECT COUNT(*) FROM scope_match_totals`).Scan(&rows); err != nil { + t.Fatal(err) + } + if rows != 1 { + t.Errorf("scope_match_totals rows = %d, want exactly 1 — the table holds a single running total", rows) + } +} + +// TestScopeMatchTallyContinuesAfterRestore checks the restored values are a +// starting point and not a ceiling: recording after a restart adds to the +// carried total rather than counting from zero again. +func TestScopeMatchTallyContinuesAfterRestore(t *testing.T) { + resetScopeMatchCounters(t) + dbPath := t.TempDir() + "/tally.db" + + s, err := openPostgresTestStore(t, dbPath) + if err != nil { + t.Fatal(err) + } + scopeMatchCounters.unique.Store(10) + scopeMatchCounters.ambiguous.Store(2) + if err := s.SaveScopeMatchTotals(); err != nil { + t.Fatal(err) + } + s.Close() + + s2, err := openPostgresTestStore(t, dbPath) + if err != nil { + t.Fatal(err) + } + defer s2.Close() + + recordScopeMatch(scopeMatch{Reason: scopeReasonUnique, Name: "#be"}) + recordScopeMatch(scopeMatch{Reason: scopeReasonAmbiguous, Candidates: []string{"#be", "#nl"}}) + if err := s2.SaveScopeMatchTotals(); err != nil { + t.Fatal(err) + } + + var uniq, ambiguous int64 + if err := s2.db.QueryRow(`SELECT unique_matches, ambiguous FROM scope_match_totals WHERE id = 1`).Scan(&uniq, &ambiguous); err != nil { + t.Fatal(err) + } + if uniq != 11 || ambiguous != 3 { + t.Errorf("stored unique=%d ambiguous=%d, want 11 and 3 — counting must resume on top of the restored totals", uniq, ambiguous) + } +} + +// TestScopeMatchTallySaveIsIdempotent pins the single-row upsert: repeated +// saves must update the row, never accumulate rows. The CHECK(id = 1) makes a +// second row impossible, and this fails loudly if the INSERT is ever changed +// to one without the conflict clause. +func TestScopeMatchTallySaveIsIdempotent(t *testing.T) { + resetScopeMatchCounters(t) + + s, err := openPostgresTestStore(t, t.TempDir()+"/tally.db") + if err != nil { + t.Fatal(err) + } + defer s.Close() + + for i := 0; i < 3; i++ { + scopeMatchCounters.unique.Add(1) + if err := s.SaveScopeMatchTotals(); err != nil { + t.Fatalf("save %d: %v", i, err) + } + } + + var rows, uniq int64 + if err := s.db.QueryRow(`SELECT COUNT(*), MAX(unique_matches) FROM scope_match_totals`).Scan(&rows, &uniq); err != nil { + t.Fatal(err) + } + if rows != 1 || uniq != 3 { + t.Errorf("rows=%d unique=%d, want 1 row holding 3", rows, uniq) + } +} diff --git a/cmd/ingestor/sig_validate_ingest_test.go b/cmd/ingestor/sig_validate_ingest_test.go index ab09e1fb3..2443606e4 100644 --- a/cmd/ingestor/sig_validate_ingest_test.go +++ b/cmd/ingestor/sig_validate_ingest_test.go @@ -4,8 +4,10 @@ import ( "crypto/ed25519" "encoding/binary" "encoding/hex" + "encoding/json" "strings" "testing" + "unicode/utf8" ) // buildAdvertHex constructs a full ADVERT packet hex string. @@ -47,7 +49,7 @@ func makeAppdata(name string) []byte { func TestSigValidation_ValidAdvertStored(t *testing.T) { dbPath := t.TempDir() + "/test.db" - store, err := OpenStoreWithInterval(dbPath, 300) + store, err := openPostgresTestStoreInterval(t, dbPath, 300) if err != nil { t.Fatal(err) } @@ -73,7 +75,7 @@ func TestSigValidation_ValidAdvertStored(t *testing.T) { func TestSigValidation_TamperedSignatureDropped(t *testing.T) { dbPath := t.TempDir() + "/test.db" - store, err := OpenStoreWithInterval(dbPath, 300) + store, err := openPostgresTestStoreInterval(t, dbPath, 300) if err != nil { t.Fatal(err) } @@ -138,7 +140,7 @@ func TestSigValidation_TamperedSignatureDropped(t *testing.T) { func TestSigValidation_TruncatedAppdataDropped(t *testing.T) { dbPath := t.TempDir() + "/test.db" - store, err := OpenStoreWithInterval(dbPath, 300) + store, err := openPostgresTestStoreInterval(t, dbPath, 300) if err != nil { t.Fatal(err) } @@ -166,9 +168,118 @@ func TestSigValidation_TruncatedAppdataDropped(t *testing.T) { } } +func TestSigValidation_DroppedPacketText(t *testing.T) { + cases := []struct { + name, nodeName, observerName, wantNode, wantObserver string + }{ + {"valid multibyte", strings.Repeat("é", 16), "Observer", strings.Repeat("é", 16), "Observer"}, + {"split two-byte rune", strings.Repeat("x", 31) + "é", "Observer", strings.Repeat("x", 31), "Observer"}, + {"split replacement rune", strings.Repeat("x", 30) + "\xff", "Observer", strings.Repeat("x", 30), "Observer"}, + {"split four-byte rune", strings.Repeat("x", 31) + "🌍", "Observer", strings.Repeat("x", 31), "Observer"}, + {"malformed name", "No\xffde", "Observer", "No\ufffdde", "Observer"}, + {"observer NUL", "Node", "Obs\x00erver", "Node", "Observer"}, + {"long observer", "Node", strings.Repeat("é", 129), "Node", strings.Repeat("é", maxObserverTextLen)}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + store := newTestStore(t) + pub, priv, err := ed25519.GenerateKey(nil) + if err != nil { + t.Fatal(err) + } + rawHex := buildAdvertHex(pub, priv, 1700000000, makeAppdata(tc.nodeName)) + raw, err := hex.DecodeString(rawHex) + if err != nil { + t.Fatal(err) + } + raw[38] ^= 1 // Invalidate the signature without changing the name bytes. + rawHex = hex.EncodeToString(raw) + payload, err := json.Marshal(struct { + Raw string `json:"raw"` + Origin string `json:"origin"` + }{rawHex, tc.observerName}) + if err != nil { + t.Fatal(err) + } + handleMessage(store, "test", MQTTSource{Name: "test"}, newMockMsg("meshcore/US/obs1/packet", string(payload)), nil, nil, &Config{}) + + var hash, storedRaw, nodeName, observerName string + if err := store.db.QueryRow("SELECT hash, raw_hex, node_name, observer_name FROM dropped_packets").Scan(&hash, &storedRaw, &nodeName, &observerName); err != nil { + t.Fatalf("rejected packet audit missing: %v (write errors=%d)", err, store.Stats.WriteErrors.Load()) + } + if storedRaw != rawHex || hash != ComputeContentHash(rawHex) { + t.Error("audit changed raw packet evidence or its hash") + } + if nodeName != tc.wantNode || !utf8.ValidString(nodeName) || len(nodeName) > 32 { + t.Errorf("node name=%q (%d bytes), want %q and valid UTF-8 within 32 bytes", nodeName, len(nodeName), tc.wantNode) + } + if observerName != tc.wantObserver || !utf8.ValidString(observerName) || utf8.RuneCountInString(observerName) > maxObserverTextLen { + t.Errorf("observer name=%q, want %q within observer text limit", observerName, tc.wantObserver) + } + if drops, errors := store.Stats.SignatureDrops.Load(), store.Stats.WriteErrors.Load(); drops != 1 || errors != 0 { + t.Errorf("signature drops=%d write errors=%d, want 1 and 0", drops, errors) + } + var transmissions int + if err := store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&transmissions); err != nil || transmissions != 0 { + t.Fatalf("invalid signature stored as transmission: count=%d err=%v", transmissions, err) + } + }) + } +} + +func TestSigValidation_AcceptedNodeText(t *testing.T) { + pub, priv, err := ed25519.GenerateKey(nil) + if err != nil { + t.Fatal(err) + } + cases := []struct { + name, nodeName, want string + update bool + }{ + {"insert split two-byte rune", strings.Repeat("x", 31) + "é", strings.Repeat("x", 31), false}, + {"update split replacement rune", strings.Repeat("x", 30) + "\xff", strings.Repeat("x", 30), true}, + {"update split four-byte rune", strings.Repeat("x", 31) + "🌍", strings.Repeat("x", 31), true}, + {"insert valid multibyte", strings.Repeat("é", 16), strings.Repeat("é", 16), false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + store := newTestStore(t) + wantUpserts := int64(1) + if tc.update { + seedHex := buildAdvertHex(pub, priv, 1700000000, makeAppdata("Original")) + handleMessage(store, "test", MQTTSource{Name: "test"}, newMockMsg("meshcore/US/obs1/packet", `{"raw":"`+seedHex+`","origin":"Observer"}`), nil, nil, &Config{}) + if store.Stats.NodeUpserts.Load() != 1 { + t.Fatal("seed node not inserted") + } + wantUpserts++ + } + rawHex := buildAdvertHex(pub, priv, 1700000001, makeAppdata(tc.nodeName)) + msg := newMockMsg("meshcore/US/obs1/packet", `{"raw":"`+rawHex+`","origin":"Observer"}`) + handleMessage(store, "test", MQTTSource{Name: "test"}, msg, nil, nil, &Config{}) + + var nodeName, storedRaw string + if err := store.db.QueryRow("SELECT name FROM nodes WHERE public_key = "+store.parameter(1), hex.EncodeToString(pub)).Scan(&nodeName); err != nil { + t.Fatalf("accepted node missing: %v (write errors=%d)", err, store.Stats.WriteErrors.Load()) + } + if nodeName != tc.want || !utf8.ValidString(nodeName) || len(nodeName) > 32 { + t.Errorf("node name=%q (%d bytes), want %q and valid UTF-8 within 32 bytes", nodeName, len(nodeName), tc.want) + } + if err := store.db.QueryRow("SELECT raw_hex FROM transmissions WHERE hash = "+store.parameter(1), ComputeContentHash(rawHex)).Scan(&storedRaw); err != nil || storedRaw != rawHex { + t.Fatalf("accepted packet evidence changed: err=%v", err) + } + if upserts, errors := store.Stats.NodeUpserts.Load(), store.Stats.WriteErrors.Load(); upserts != wantUpserts || errors != 0 { + t.Errorf("node upserts=%d write errors=%d, want %d and 0", upserts, errors, wantUpserts) + } + if drops := store.Stats.SignatureDrops.Load(); drops != 0 { + t.Errorf("valid signature dropped: %d", drops) + } + }) + } +} + func TestSigValidation_DisabledByConfig(t *testing.T) { dbPath := t.TempDir() + "/test.db" - store, err := OpenStoreWithInterval(dbPath, 300) + store, err := openPostgresTestStoreInterval(t, dbPath, 300) if err != nil { t.Fatal(err) } @@ -204,7 +315,7 @@ func TestSigValidation_DisabledByConfig(t *testing.T) { func TestSigValidation_DropCounterIncrements(t *testing.T) { dbPath := t.TempDir() + "/test.db" - store, err := OpenStoreWithInterval(dbPath, 300) + store, err := openPostgresTestStoreInterval(t, dbPath, 300) if err != nil { t.Fatal(err) } @@ -236,7 +347,7 @@ func TestSigValidation_DropCounterIncrements(t *testing.T) { func TestSigValidation_LogContainsFields(t *testing.T) { // This test verifies the dropped_packets row has all required fields dbPath := t.TempDir() + "/test.db" - store, err := OpenStoreWithInterval(dbPath, 300) + store, err := openPostgresTestStoreInterval(t, dbPath, 300) if err != nil { t.Fatal(err) } @@ -289,15 +400,15 @@ func TestSigValidation_LogContainsFields(t *testing.T) { func TestPruneDroppedPackets(t *testing.T) { dbPath := t.TempDir() + "/test.db" - store, err := OpenStoreWithInterval(dbPath, 300) + store, err := openPostgresTestStoreInterval(t, dbPath, 300) if err != nil { t.Fatal(err) } defer store.Close() // Insert an old dropped packet - store.db.Exec(`INSERT INTO dropped_packets (hash, reason, dropped_at) VALUES ('old', 'test', datetime('now', '-60 days'))`) - store.db.Exec(`INSERT INTO dropped_packets (hash, reason, dropped_at) VALUES ('new', 'test', datetime('now'))`) + store.db.Exec(testNativeSQL(`INSERT INTO dropped_packets (hash, reason, dropped_at) VALUES ('old', 'test', datetime('now','-60 days'))`, `INSERT INTO dropped_packets (hash, reason, dropped_at) VALUES ('old', 'test', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC' - interval '60 days','YYYY-MM-DD HH24:MI:SS'))`)) + store.db.Exec(testNativeSQL(`INSERT INTO dropped_packets (hash, reason, dropped_at) VALUES ('new', 'test', datetime('now'))`, `INSERT INTO dropped_packets (hash, reason, dropped_at) VALUES ('new', 'test', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC','YYYY-MM-DD HH24:MI:SS'))`)) n, err := store.PruneDroppedPackets(30) if err != nil { diff --git a/cmd/ingestor/sqlite_maintenance.go b/cmd/ingestor/sqlite_maintenance.go new file mode 100644 index 000000000..8a89e921c --- /dev/null +++ b/cmd/ingestor/sqlite_maintenance.go @@ -0,0 +1,141 @@ +package main + +import ( + "fmt" + "github.com/meshcore-analyzer/dbconfig" + "log" + "time" +) + +func (s *Store) CheckAutoVacuum(cfg *Config) { + if s.Backend() != dbconfig.SQLite { + return + } + + var autoVacuum int + if err := s.db.QueryRow("PRAGMA auto_vacuum").Scan(&autoVacuum); err != nil { + log.Printf("[db] warning: could not read auto_vacuum: %v", err) + return + } + + if autoVacuum == 2 { + log.Printf("[db] auto_vacuum=INCREMENTAL") + return + } + + modes := map[int]string{0: "NONE", 1: "FULL", 2: "INCREMENTAL"} + mode := modes[autoVacuum] + if mode == "" { + mode = fmt.Sprintf("UNKNOWN(%d)", autoVacuum) + } + + log.Printf("[db] auto_vacuum=%s — DB needs one-time VACUUM to enable incremental auto-vacuum. "+ + "Set db.vacuumOnStartup: true in config to migrate (will block startup for several minutes on large DBs). "+ + "See https://github.com/Kpa-clawbot/CoreScope/issues/919", mode) + + if cfg.DB != nil && cfg.DB.VacuumOnStartup { + // WARNING: Full VACUUM creates a temporary copy of the entire DB file. + // Requires ~2× the DB file size in free disk space or it will fail. + log.Printf("[db] vacuumOnStartup=true — starting one-time full VACUUM (ensure 2x DB size free disk space)...") + start := time.Now() + + if _, err := s.instrumentedExec("vacuum", "PRAGMA auto_vacuum = INCREMENTAL"); err != nil { + log.Printf("[db] VACUUM failed: could not set auto_vacuum: %v", err) + return + } + if _, err := s.instrumentedExec("vacuum", "VACUUM"); err != nil { + log.Printf("[db] VACUUM failed: %v", err) + return + } + + elapsed := time.Since(start) + log.Printf("[db] VACUUM complete in %v — auto_vacuum is now INCREMENTAL", elapsed.Round(time.Millisecond)) + } +} + +func (s *Store) RunIncrementalVacuum(pages int) { + if s.Backend() != dbconfig.SQLite { + return + } + + // Tagged for /api/perf writer-lock visibility (#1340). + if _, err := s.instrumentedExec("vacuum", fmt.Sprintf("PRAGMA incremental_vacuum(%d)", pages)); err != nil { + log.Printf("[vacuum] incremental_vacuum error: %v", err) + } +} + +func (s *Store) RefreshPlannerStats(analysisLimit int) bool { + if s.Backend() != dbconfig.SQLite { + return false + } + + if analysisLimit < 0 { + return false + } + first := !s.hasPlannerStats() + start := time.Now() + // Tagged for /api/perf writer-lock visibility (#1340). + if _, err := s.instrumentedExec("analyze", fmt.Sprintf("PRAGMA analysis_limit=%d", analysisLimit)); err != nil { + log.Printf("[analyze] could not set analysis_limit: %v", err) + return false + } + if _, err := s.instrumentedExec("analyze", "ANALYZE"); err != nil { + log.Printf("[analyze] ANALYZE failed: %v", err) + return false + } + elapsed := time.Since(start).Round(time.Millisecond) + if first { + log.Printf("[analyze] planner statistics built in %v (analysis_limit=%d, first run against this database)", elapsed, analysisLimit) + } else { + log.Printf("[analyze] planner statistics refreshed in %v (analysis_limit=%d)", elapsed, analysisLimit) + } + return true +} + +func (s *Store) EnsurePlannerStats(analysisLimit int) bool { + if s.Backend() != dbconfig.SQLite { + return false + } + + if s.hasPlannerStats() { + return false + } + log.Printf("[analyze] this database has no planner statistics; building them now. " + + "ANALYZE holds the single write connection until it finishes (3m43.9s measured on 9.4 GB, cold), " + + "so ingest will buffer and catch up. Once per database, not once per restart.") + return s.RefreshPlannerStats(analysisLimit) +} + +func (s *Store) hasPlannerStats() bool { + if s.Backend() != dbconfig.SQLite { + return false + } + + var n int + if err := s.db.QueryRow( + `SELECT count(*) FROM sqlite_master WHERE type='table' AND name='sqlite_stat1'`).Scan(&n); err != nil { + return false + } + return n > 0 +} + +func (s *Store) Checkpoint() int { + if s.Backend() != dbconfig.SQLite { + return 0 + } + waitStart := time.Now() + writerMu.Lock() + wait := time.Since(waitStart) + holdStart := time.Now() + defer func() { writerMu.Unlock(); recordWriterTiming("checkpoint", wait, time.Since(holdStart), "Checkpoint") }() + + var busy, walFrames, checkpointed int + if err := s.db.QueryRow("PRAGMA wal_checkpoint(TRUNCATE)").Scan(&busy, &walFrames, &checkpointed); err != nil { + log.Printf("[db] WAL checkpoint error: %v", err) + return 0 + } + if walFrames > 0 { + log.Printf("[db] WAL checkpoint: %d/%d frames checkpointed (blocked=%v)", checkpointed, walFrames, busy != 0) + } + return checkpointed +} diff --git a/cmd/ingestor/stats_file.go b/cmd/ingestor/stats_file.go index cb5ac9e41..67375ffb1 100644 --- a/cmd/ingestor/stats_file.go +++ b/cmd/ingestor/stats_file.go @@ -6,6 +6,8 @@ import ( "encoding/json" "log" "os" + "path/filepath" + "sync" "time" "github.com/meshcore-analyzer/perfio" @@ -43,7 +45,7 @@ type IngestorStatsSnapshot struct { // the server's /api/perf/io endpoint under .ingestor (#1120 — "Both // ingestor and server"). Optional; absent on non-Linux hosts. ProcIO *PerfIOSample `json:"procIO,omitempty"` - // WriterPerf is the per-component SQLite writer-lock latency + // WriterPerf is the per-component serialized writer-lock latency // snapshot (#1340) — wait_ms / hold_ms / contention_total tagged // by component (neighbor_builder, mqtt_handler, prune_packets, // prune_observers, prune_metrics, vacuum). Surfaced by the server @@ -105,15 +107,17 @@ type SourceLivenessSnapshot struct { // statsFilePath returns the writable path the ingestor will publish stats to. // Override via env CORESCOPE_INGESTOR_STATS for tests / non-default deploys. // -// SECURITY: the default lives in /tmp which is world-writable. The writer uses -// O_NOFOLLOW + 0o600 so a pre-planted symlink cannot be used to clobber an -// arbitrary file via this path. Operators who want stronger guarantees should -// point CORESCOPE_INGESTOR_STATS at a private directory (e.g. /var/lib/corescope/). +// The default lives in the private state directory. The writer retains +// O_NOFOLLOW and private permissions for an explicitly overridden path. func statsFilePath() string { if p := os.Getenv("CORESCOPE_INGESTOR_STATS"); p != "" { return p } - return "/tmp/corescope-ingestor-stats.json" + stateDir := os.Getenv("CORESCOPE_STATE_DIR") + if stateDir == "" { + stateDir = "data" + } + return filepath.Join(stateDir, "ingestor-stats.json") } // writeStatsAtomic writes b to path via a tmp-then-rename, refusing to follow @@ -240,14 +244,26 @@ func procIORate(prev, cur procIOSnapshot, stamp string) *PerfIOSample { // The stats file path is resolved via statsFilePath() once at writer-loop // start; the env var (CORESCOPE_INGESTOR_STATS) is only re-read on process // restart, not per tick. -func StartStatsFileWriter(s *Store, interval time.Duration) { +// The returned stop function ends the writer goroutine and returns once it has +// exited. Production ignores it and runs for the process lifetime; tests must +// call it, because this goroutine reads package-level hooks (readProcSelfIOFn) +// that a later test may replace, and a writer left running past its own test +// races with whoever runs next. It is safe to call more than once. +func StartStatsFileWriter(s *Store, interval time.Duration) (stop func()) { if interval <= 0 { interval = time.Second } + done := make(chan struct{}) + stopped := make(chan struct{}) + var once sync.Once go func() { + defer close(stopped) t := time.NewTicker(interval) defer t.Stop() path := statsFilePath() + if os.Getenv("CORESCOPE_INGESTOR_STATS") == "" && s.stateDir != "" { + path = filepath.Join(s.stateDir, "ingestor-stats.json") + } // Track previous procIO sample so we can compute per-second deltas // across ticks (#1120 follow-up: ingestor /proc/self/io exposure). prevIO := readProcSelfIOFn() @@ -257,7 +273,12 @@ func StartStatsFileWriter(s *Store, interval time.Duration) { // The buffer grows once and stays. var buf bytes.Buffer enc := json.NewEncoder(&buf) - for range t.C { + for { + select { + case <-done: + return + case <-t.C: + } // Capture time.Now() ONCE per tick (Carmack must-fix #5). // Both snapshot.SampledAt and procIO.SampledAt MUST share the // same string so the freshness guard isn't validating one @@ -305,4 +326,8 @@ func StartStatsFileWriter(s *Store, interval time.Duration) { } } }() + return func() { + once.Do(func() { close(done) }) + <-stopped + } } diff --git a/cmd/ingestor/stats_file_test.go b/cmd/ingestor/stats_file_test.go index 4a488f26a..431e104e6 100644 --- a/cmd/ingestor/stats_file_test.go +++ b/cmd/ingestor/stats_file_test.go @@ -2,8 +2,11 @@ package main import ( "encoding/json" + "errors" "os" "path/filepath" + "runtime" + "syscall" "testing" "time" ) @@ -50,13 +53,13 @@ func TestStatsFileWriter_PublishesProcIO(t *testing.T) { statsPath := filepath.Join(dir, "ingestor-stats.json") t.Setenv("CORESCOPE_INGESTOR_STATS", statsPath) - store, err := OpenStore(filepath.Join(dir, "test.db")) + store, err := openPostgresTestStore(t, filepath.Join(dir, "test.db")) if err != nil { t.Fatalf("OpenStore: %v", err) } defer store.Close() - StartStatsFileWriter(store, 50*time.Millisecond) + t.Cleanup(StartStatsFileWriter(store, 50*time.Millisecond)) // Wait for at least 2 ticks so the writer has had a chance to populate // procIO rates from a delta. @@ -129,6 +132,9 @@ func TestWriteStatsAtomic_SymlinkAtDestIsReplaced(t *testing.T) { // Pre-plant a symlink at the destination path. path := filepath.Join(dir, "stats.json") if err := os.Symlink(target, path); err != nil { + if runtime.GOOS == "windows" && errors.Is(err, syscall.Errno(1314)) { + t.Skip("Windows account cannot create symlinks; this security fixture must run on Linux CI") + } t.Fatalf("symlink: %v", err) } diff --git a/cmd/ingestor/stats_file_timestamp_test.go b/cmd/ingestor/stats_file_timestamp_test.go index 900e65606..abae95de8 100644 --- a/cmd/ingestor/stats_file_timestamp_test.go +++ b/cmd/ingestor/stats_file_timestamp_test.go @@ -38,7 +38,7 @@ func TestStatsFileWriter_SampledAtMatchesProcIOSampledAt(t *testing.T) { statsPath := filepath.Join(dir, "ingestor-stats.json") t.Setenv("CORESCOPE_INGESTOR_STATS", statsPath) - store, err := OpenStore(filepath.Join(dir, "test.db")) + store, err := openPostgresTestStore(t, filepath.Join(dir, "test.db")) if err != nil { t.Fatalf("OpenStore: %v", err) } @@ -69,7 +69,7 @@ func TestStatsFileWriter_SampledAtMatchesProcIOSampledAt(t *testing.T) { } } - StartStatsFileWriter(store, 50*time.Millisecond) + t.Cleanup(StartStatsFileWriter(store, 50*time.Millisecond)) // Wait for the file to land with a populated procIO block. deadline := time.Now().Add(3 * time.Second) diff --git a/cmd/ingestor/status_liveness_test.go b/cmd/ingestor/status_liveness_test.go index 6f96b07d1..a72b5aff2 100644 --- a/cmd/ingestor/status_liveness_test.go +++ b/cmd/ingestor/status_liveness_test.go @@ -55,7 +55,7 @@ func TestStaleStatusPayloadDoesNotCreateObserver(t *testing.T) { nil, nil, &Config{}) var count int - if err := store.db.QueryRow(`SELECT COUNT(*) FROM observers WHERE id = ?`, "obs-purged").Scan(&count); err != nil { + if err := store.db.QueryRow(`SELECT COUNT(*) FROM observers WHERE id = $1`, "obs-purged").Scan(&count); err != nil { t.Fatal(err) } if count != 0 { @@ -82,7 +82,7 @@ func TestOfflineStatusIsNotLiveness(t *testing.T) { func TestStaleStatusPayloadDoesNotReactivateInactiveObserver(t *testing.T) { store := newTestStore(t) seedObserver(t, store, "obs-zombie", 60) - if _, err := store.db.Exec(`UPDATE observers SET inactive = 1 WHERE id = ?`, "obs-zombie"); err != nil { + if _, err := store.db.Exec(`UPDATE observers SET inactive = 1 WHERE id = $1`, "obs-zombie"); err != nil { t.Fatal(err) } @@ -92,7 +92,7 @@ func TestStaleStatusPayloadDoesNotReactivateInactiveObserver(t *testing.T) { nil, nil, &Config{}) var inactive int - if err := store.db.QueryRow(`SELECT inactive FROM observers WHERE id = ?`, "obs-zombie").Scan(&inactive); err != nil { + if err := store.db.QueryRow(`SELECT inactive FROM observers WHERE id = $1`, "obs-zombie").Scan(&inactive); err != nil { t.Fatal(err) } if inactive != 1 { @@ -172,7 +172,7 @@ func TestStaleStatusPayloadStillUpdatesMetadata(t *testing.T) { nil, nil, &Config{}) var firmware string - if err := store.db.QueryRow(`SELECT firmware FROM observers WHERE id = ?`, "obs-zombie").Scan(&firmware); err != nil { + if err := store.db.QueryRow(`SELECT firmware FROM observers WHERE id = $1`, "obs-zombie").Scan(&firmware); err != nil { t.Fatal(err) } if firmware != "v1.14.1" { @@ -193,7 +193,7 @@ func TestStaleStatusPayloadDoesNotInsertMetricsSample(t *testing.T) { var count int if err := store.db.QueryRow( - `SELECT COUNT(*) FROM observer_metrics WHERE observer_id = ?`, "obs-zombie", + `SELECT COUNT(*) FROM observer_metrics WHERE observer_id = $1`, "obs-zombie", ).Scan(&count); err != nil { t.Fatal(err) } diff --git a/cmd/ingestor/storage_adoption.go b/cmd/ingestor/storage_adoption.go new file mode 100644 index 000000000..9145ee227 --- /dev/null +++ b/cmd/ingestor/storage_adoption.go @@ -0,0 +1,147 @@ +package main + +import ( + "context" + "database/sql" + "errors" + "net/url" + "os" + "path/filepath" + "time" + + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/dbschema/legacy" + "github.com/meshcore-analyzer/pgutil" +) + +// Adoption never initializes a missing telemetry file. Setup creates a fresh +// installation; this path records only an existing, validated native target. +func adoptLegacyStorage(raw dbconfig.StorageInputs) error { + selectionPath, err := storageSelectionPath(raw) + if err != nil { + return err + } + if _, err := dbconfig.ReadSelection(selectionPath); err == nil { + return nil + } else if !errors.Is(err, dbconfig.ErrSelectionMissing) { + return err + } + if raw.Backend == "" && raw.EnvBackend == "" && raw.DBPath == "" && raw.DatabaseURL == "" && raw.ReaderDatabaseURL == "" && raw.WriterDatabaseURL == "" && raw.UsersDatabaseURL == "" && raw.ApprovedChannelsDatabaseURL == "" { + raw.DBPath = filepath.Join(raw.BaseDir, "data", "meshcore.db") + } + raw.StateDir = filepath.Dir(selectionPath) + storage, err := dbconfig.ResolveStorage(raw) + if err != nil { + return err + } + selected, err := dbconfig.NewSelection(storage) + if err != nil { + return err + } + if storage.Backend == dbconfig.SQLite { + db, err := openExistingSQLite(storage.DBPath) + if err != nil { + return err + } + err = dbconfig.AssertSQLiteImportComplete(db) + if err == nil { + err = legacy.CheckLegacySource(db) + } + db.Close() + if err != nil { + return err + } + if _, err := os.Stat(storage.UsersDBPath); err == nil { + accounts, err := openExistingSQLite(storage.UsersDBPath) + if err != nil { + return err + } + var version int + if err = dbconfig.AssertSQLiteImportComplete(accounts); err != nil { + accounts.Close() + return err + } + err = accounts.QueryRow(`SELECT MAX(version) FROM schema_version`).Scan(&version) + accounts.Close() + if err != nil || version < 1 || version > 6 { + return errors.New("existing SQLite accounts schema is unsupported; use setup or offline recovery") + } + } else if errors.Is(err, os.ErrNotExist) { + // Absent legacy accounts are uninitialized, not an initialized empty store. + // Only explicit setup may create them and fill this selection target. + selected.Accounts = nil + } else { + return err + } + } else { + if storage.UsersDatabaseURL == "" && storage.ApprovedChannelsDatabaseURL == "" { + legacyAccounts := raw.UsersDBPath + if legacyAccounts == "" { + legacyAccounts = filepath.Join(storage.StateDir, "users.db") + } + if _, err := os.Stat(legacyAccounts); err == nil { + return errors.New("existing SQLite accounts require offline conversion before adopting PostgreSQL") + } else if !errors.Is(err, os.ErrNotExist) { + return err + } + } + db, err := pgutil.Open(storage.WriterDatabaseURL, false) + if err != nil { + return err + } + err = dbschema.AssertPostgresReady(db) + if err == nil { + err = dbschema.AssertWriter(db) + } + db.Close() + if err != nil { + return err + } + accountURL := storage.UsersDatabaseURL + if accountURL == "" { + accountURL = storage.ApprovedChannelsDatabaseURL + } + if accountURL != "" { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + same, err := pgutil.SameDatabase(ctx, storage.WriterDatabaseURL, accountURL) + cancel() + if err != nil { + return err + } + if same { + return errors.New("accounts must not use the telemetry database") + } + accounts, err := pgutil.Open(accountURL, true) + if err != nil { + return err + } + var ready bool + err = accounts.QueryRow(`SELECT ready FROM corescope_schema WHERE kind='accounts' AND version=1`).Scan(&ready) + accounts.Close() + if err != nil || !ready { + return errors.New("PostgreSQL accounts import/bootstrap is incomplete") + } + } + } + + _, err = dbconfig.AdoptSelection(selectionPath, selected) + return err +} + +func openExistingSQLite(path string) (*sql.DB, error) { + info, err := os.Stat(path) + if err != nil || !info.Mode().IsRegular() || info.Size() == 0 { + return nil, errors.New("existing SQLite storage is missing or empty; run explicit setup instead of creating a new runtime database") + } + dsn, err := dbconfig.SQLiteURI(path, url.Values{"mode": {"ro"}, "_query_only": {"on"}, "_busy_timeout": {"5000"}}) + if err != nil { + return nil, err + } + db, err := sql.Open("sqlite3", dsn) + if err != nil { + return nil, err + } + db.SetMaxOpenConns(1) + return db, nil +} diff --git a/cmd/ingestor/storage_adoption_test.go b/cmd/ingestor/storage_adoption_test.go new file mode 100644 index 000000000..ce9d882b4 --- /dev/null +++ b/cmd/ingestor/storage_adoption_test.go @@ -0,0 +1,125 @@ +package main + +import ( + "errors" + "os" + "path/filepath" + "testing" + + "github.com/meshcore-analyzer/dbconfig" +) + +func TestRuntimeMissingSQLiteCannotCreateReplacement(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "uninitialized", "meshcore.db") + raw := dbconfig.StorageInputs{BaseDir: dir, Backend: dbconfig.SQLite, DBPath: path} + if err := adoptLegacyStorage(raw); err == nil { + t.Fatal("missing source was treated as fresh") + } + if _, err := os.Stat(filepath.Dir(path)); !errors.Is(err, os.ErrNotExist) { + t.Fatal("runtime created replacement storage", err) + } +} + +func TestRuntimeAdoptsExistingSQLiteAndPinsRecordedChoice(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "meshcore.db") + seed, err := OpenStore(path) + if err != nil { + t.Fatal(err) + } + seed.Close() + raw := dbconfig.StorageInputs{BaseDir: dir, DBPath: path} + if err := adoptLegacyStorage(raw); err != nil { + t.Fatal(err) + } + raw.Backend = dbconfig.Postgres + raw.EnvBackend = dbconfig.Postgres + raw.DatabaseURL = "postgresql://unused@localhost/unselected" + storage, lease, err := resolveRuntimeStorage(raw) + if err != nil { + t.Fatal(err) + } + defer lease.Close() + if storage.Backend != dbconfig.SQLite || storage.DBPath != path || storage.UsersDBPath != "" { + t.Fatal("obsolete bootstrap choices overrode recorded installation") + } + selected, err := dbconfig.ReadSelection(dbconfig.SelectionPath(dir)) + if err != nil { + t.Fatal(err) + } + if selected.Accounts != nil { + t.Fatal("absent accounts marked initialized") + } + if _, err := dbconfig.BeginSelectionUpdate(dbconfig.SelectionPath(dir), selected.Generation); !errors.Is(err, dbconfig.ErrSelectionBusy) { + t.Fatal("runtime did not hold the selection lease", err) + } +} + +func TestRuntimeCannotAdoptStagedIncompleteSQLite(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "staged.db") + seed, err := OpenStore(path) + if err != nil { + t.Fatal(err) + } + if _, err = seed.db.Exec(`CREATE TABLE ` + dbconfig.SQLiteImportPendingTable + `(phase TEXT)`); err != nil { + t.Fatal(err) + } + seed.Close() + raw := dbconfig.StorageInputs{BaseDir: dir, StateDir: dir, DBPath: path} + if err := adoptLegacyStorage(raw); !errors.Is(err, dbconfig.ErrSQLiteImportIncomplete) { + t.Fatal("incomplete target accepted", err) + } + if _, err := dbconfig.ReadSelection(dbconfig.SelectionPath(dir)); !errors.Is(err, dbconfig.ErrSelectionMissing) { + t.Fatal("incomplete target recorded", err) + } +} + +func TestRecordedSQLiteCannotCreateReplacement(t *testing.T) { + for _, empty := range []bool{false, true} { + name := "missing" + if empty { + name = "empty" + } + t.Run(name, func(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "meshcore.db") + seed, err := OpenStore(path) + if err != nil { + t.Fatal(err) + } + seed.Close() + raw := dbconfig.StorageInputs{BaseDir: dir, DBPath: path} + if err := adoptLegacyStorage(raw); err != nil { + t.Fatal(err) + } + if err := os.Rename(path, path+".preserved"); err != nil { + t.Fatal(err) + } + if empty { + if err := os.WriteFile(path, nil, 0600); err != nil { + t.Fatal(err) + } + } + storage, lease, err := resolveRuntimeStorage(raw) + if err != nil { + t.Fatal(err) + } + defer lease.Close() + store, err := openStoreStorage(storage, nil, 300) + if err == nil { + store.Close() + t.Error("recorded runtime silently initialized replacement telemetry") + } + info, statErr := os.Stat(path) + if empty { + if statErr != nil || info.Size() != 0 { + t.Error("runtime modified the empty recorded target", statErr) + } + } else if !errors.Is(statErr, os.ErrNotExist) { + t.Error("runtime created a missing recorded target", statErr) + } + }) + } +} diff --git a/cmd/ingestor/storage_backend.go b/cmd/ingestor/storage_backend.go new file mode 100644 index 000000000..120e96fd5 --- /dev/null +++ b/cmd/ingestor/storage_backend.go @@ -0,0 +1,72 @@ +package main + +import ( + "database/sql" + "errors" + "net/url" + "os" + + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/dbschema/legacy" +) + +func (s *Store) Backend() dbconfig.Backend { + if s.backend == "" { + return dbconfig.SQLite + } + return s.backend +} +func (s *Store) parameter(n int) string { return s.Backend().Parameter(n) } +func (s *Store) nativeSQL(sqlite, postgres string) string { + if s.Backend() == dbconfig.SQLite { + return sqlite + } + return postgres +} + +// OpenStoreStorage is the runtime boundary after the shared selector resolves +// raw inputs. Compatibility constructors still accept paths or explicit URLs. +func OpenStoreStorage(storage dbconfig.Storage, cfg *DBConfig) (*Store, error) { + return openStoreStorage(storage, cfg, 300) +} + +// Retain the shared durability/connection options. Existing runtime targets use +// rw (never rwc); persistent pragmas run only after schema validation succeeds. +func sqliteTelemetryWriterDSN(path string, create bool) (string, error) { + if !create { + info, err := os.Stat(path) + if err != nil || !info.Mode().IsRegular() || info.Size() == 0 { + return "", errors.New("selected SQLite telemetry is missing or empty; restore the initialized store") + } + } + dsn, err := legacy.WriterDSN(path) + if err != nil { + return "", err + } + u, err := url.Parse(dsn) + if err != nil { + return "", err + } + q := u.Query() + q.Set("_txlock", "immediate") + if !create { + q.Set("mode", "rw") + q.Del("_journal_mode") + q.Del("_auto_vacuum") + } + u.RawQuery = q.Encode() + return u.String(), nil +} + +func validateExistingTelemetry(db *sql.DB) error { + if err := dbconfig.AssertSQLiteImportComplete(db); err != nil { + return err + } + if err := dbschema.AssertSQLiteReady(db); err == nil { + return nil + } + // A supported older v3 store may need the normal writer upgrade. This also + // rejects empty/header-only/unrelated databases through the opened handle. + return legacy.CheckLegacySource(db) +} diff --git a/cmd/ingestor/storage_backend_test.go b/cmd/ingestor/storage_backend_test.go new file mode 100644 index 000000000..51d823cfb --- /dev/null +++ b/cmd/ingestor/storage_backend_test.go @@ -0,0 +1,100 @@ +package main + +import ( + "database/sql" + "github.com/meshcore-analyzer/dbconfig" + "os" + "path/filepath" + "testing" +) + +func TestOpenStoreNativeSQLiteWithoutPostgres(t *testing.T) { + t.Setenv("CORESCOPE_TEST_POSTGRES_URL", "") + path := filepath.Join(t.TempDir(), "telemetry.db") + s, err := OpenStore(path) + if err != nil { + t.Fatalf("native SQLite writer must work without PostgreSQL: %v", err) + } + defer s.Close() + s.WaitForAsyncMigrations() + for pragma, want := range map[string]string{"journal_mode": "wal", "synchronous": "2", "foreign_keys": "1"} { + var actual string + if err := s.db.QueryRow("PRAGMA " + pragma).Scan(&actual); err != nil || actual != want { + t.Fatalf("%s=%q want %q: %v", pragma, actual, want, err) + } + } + if s.path != path { + t.Fatalf("legacy queue anchor changed: %q", s.path) + } +} + +func TestSQLiteObserverTimeIndexCompletesThroughAsyncLedger(t *testing.T) { + s, err := OpenStore(filepath.Join(t.TempDir(), "observer-index.db")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + s.WaitForAsyncMigrations() + var count int + if err = s.db.QueryRow(`SELECT count(*) FROM sqlite_master WHERE type='index' AND name='idx_observations_observer_idx_timestamp'`).Scan(&count); err != nil || count != 1 { + t.Fatalf("native observer/time index count=%d: %v", count, err) + } + if status, err := s.AsyncMigrationStatus("obs_observer_ts_idx_v1"); err != nil || status != "done" { + t.Fatal("native index migration did not complete", status, err) + } +} + +func TestSQLiteMissingObserverIndexIsNotHiddenByCompletionMarker(t *testing.T) { + path := filepath.Join(t.TempDir(), "recovered-index.db") + s, err := OpenStore(path) + if err != nil { + t.Fatal(err) + } + s.WaitForAsyncMigrations() + if _, err = s.db.Exec(`DROP INDEX idx_observations_observer_idx_timestamp`); err != nil { + t.Fatal(err) + } + s.Close() + s, err = OpenStore(path) + if err != nil { + t.Fatal(err) + } + defer s.Close() + s.WaitForAsyncMigrations() + var count int + if err = s.db.QueryRow(`SELECT count(*) FROM sqlite_master WHERE type='index' AND name='idx_observations_observer_idx_timestamp'`).Scan(&count); err != nil || count != 1 { + t.Fatal("missing index concealed by done marker", count, err) + } +} + +func testBackend(t testing.TB) dbconfig.Backend { + t.Helper() + v := os.Getenv("CORESCOPE_TEST_BACKEND") + if v != "" && v != "sqlite" && v != "postgres" { + t.Fatalf("invalid CORESCOPE_TEST_BACKEND") + } + if v == "postgres" { + return dbconfig.Postgres + } + return dbconfig.SQLite +} +func testBackendValue() dbconfig.Backend { + if os.Getenv("CORESCOPE_TEST_BACKEND") == "postgres" { + return dbconfig.Postgres + } + return dbconfig.SQLite +} +func testNativeSQL(sqlite, postgres string) string { + if testBackendValue() == dbconfig.Postgres { + return postgres + } + return sqlite +} + +func scanTestPlan(rows *sql.Rows, detail *string) error { + if testBackendValue() == dbconfig.SQLite { + var id, parent, unused int + return rows.Scan(&id, &parent, &unused, detail) + } + return rows.Scan(detail) +} diff --git a/cmd/ingestor/storage_config.go b/cmd/ingestor/storage_config.go new file mode 100644 index 000000000..021b1e3d9 --- /dev/null +++ b/cmd/ingestor/storage_config.go @@ -0,0 +1,120 @@ +package main + +import ( + "errors" + "io" + "path/filepath" + "strings" + + "github.com/meshcore-analyzer/dbconfig" +) + +type storageFlags struct { + Backend dbconfig.Backend + DBPath, DatabaseURL, StateDir string +} + +// storageInputs gathers bootstrap values without guessing an installation's +// state. The shared persisted selector applies before ResolveStorage runs. +func (c *Config) storageInputs(baseDir string, flags storageFlags, getenv func(string) string) (dbconfig.StorageInputs, error) { + base, err := filepath.Abs(baseDir) + if err != nil { + return dbconfig.StorageInputs{}, errors.New("resolve storage base directory") + } + in := dbconfig.StorageInputs{BaseDir: base, DBPath: c.DBPath, StateDir: envOrValue(getenv, "CORESCOPE_STATE_DIR", c.StateDir), + EnvBackend: dbconfig.Backend(getenv("CORESCOPE_DB_BACKEND")), + DatabaseURL: envOrValue(getenv, "CORESCOPE_DATABASE_URL", c.DatabaseURL), + ReaderDatabaseURL: getenv("CORESCOPE_READER_DATABASE_URL"), WriterDatabaseURL: getenv("CORESCOPE_WRITER_DATABASE_URL"), + UsersDatabaseURL: getenv("CORESCOPE_USERS_DATABASE_URL"), ApprovedChannelsDatabaseURL: getenv("CORESCOPE_APPROVED_CHANNELS_DATABASE_URL")} + if p := getenv("DB_PATH"); p != "" { + in.DBPath = p + } + if c.DB != nil { + in.Backend = c.DB.Backend + } + if c.UserManagement != nil { + in.UsersDBPath = strings.TrimSpace(c.UserManagement.DBPath) + in.UsersDatabaseURL = envOrValue(getenv, "CORESCOPE_USERS_DATABASE_URL", c.UserManagement.DatabaseURL) + in.ApprovedChannelsDatabaseURL = envOrValue(getenv, "CORESCOPE_APPROVED_CHANNELS_DATABASE_URL", c.UserManagement.ApprovedChannelsDatabaseURL) + } + if flags.Backend != "" { + in.Backend = flags.Backend + } + if flags.DBPath != "" { + in.DBPath = flags.DBPath + } + if flags.DatabaseURL != "" { + in.DatabaseURL = flags.DatabaseURL + in.WriterDatabaseURL = flags.DatabaseURL + } + if flags.StateDir != "" { + in.StateDir = flags.StateDir + } + // The ingestor receives the process working directory as its base; explicit + // legacy paths retain their historical working-directory interpretation. + for _, p := range []*string{&in.DBPath, &in.UsersDBPath} { + if *p == "" || strings.Contains(*p, "://") { + continue + } + if filepath.VolumeName(*p) != "" && !filepath.IsAbs(*p) { + return in, errors.New("drive-relative storage paths are ambiguous") + } + *p, err = filepath.Abs(*p) + if err != nil { + return in, errors.New("resolve SQLite storage path") + } + } + return in, nil +} + +func envOrValue(getenv func(string) string, key, fallback string) string { + if value := strings.TrimSpace(getenv(key)); value != "" { + return value + } + return fallback +} + +// Discovery uses the stable legacy anchor, including after a backend switch. +func storageSelectionPath(raw dbconfig.StorageInputs) (string, error) { + state := raw.StateDir + if state == "" { + if strings.Contains(raw.DBPath, "://") || strings.HasPrefix(raw.DBPath, "postgres:") || strings.HasPrefix(raw.DBPath, "postgresql:") { + return "", errors.New("a database URL cannot be used as the legacy state directory anchor") + } + path := raw.DBPath + if path == "" { + path = filepath.Join(raw.BaseDir, "data", "meshcore.db") + } + state = filepath.Dir(path) + } + if strings.Contains(state, "://") || strings.ContainsRune(state, 0) { + return "", errors.New("invalid storage state directory") + } + if !filepath.IsAbs(state) { + state = filepath.Join(raw.BaseDir, state) + } + return dbconfig.SelectionPath(filepath.Clean(state)), nil +} +func resolveRuntimeStorage(raw dbconfig.StorageInputs) (dbconfig.Storage, io.Closer, error) { + path, err := storageSelectionPath(raw) + if err != nil { + return dbconfig.Storage{}, nil, err + } + selected, lease, err := dbconfig.OpenSelection(path) + if err != nil { + return dbconfig.Storage{}, nil, err + } + fail := func(err error) (dbconfig.Storage, io.Closer, error) { + lease.Close() + return dbconfig.Storage{}, nil, err + } + raw, err = selected.ApplyTo(raw) + if err != nil { + return fail(err) + } + storage, err := dbconfig.ResolveStorage(raw) + if err != nil { + return fail(err) + } + return storage, lease, nil +} diff --git a/cmd/ingestor/storage_config_test.go b/cmd/ingestor/storage_config_test.go new file mode 100644 index 000000000..390e7d917 --- /dev/null +++ b/cmd/ingestor/storage_config_test.go @@ -0,0 +1,88 @@ +package main + +import ( + "path/filepath" + "testing" + + "github.com/meshcore-analyzer/dbconfig" +) + +func TestStorageInputsKeepBootstrapRawAndLegacyPrecedence(t *testing.T) { + base := t.TempDir() + cfg := &Config{DBPath: "configured.db", UserManagement: &UserManagementConfig{DBPath: "accounts.db"}} + env := func(k string) string { + if k == "DB_PATH" { + return "environment.db" + } + return "" + } + in, err := cfg.storageInputs(base, storageFlags{}, env) + if err != nil { + t.Fatal(err) + } + want, _ := filepath.Abs("environment.db") + if in.DBPath != want || in.StateDir != "" || in.Backend != "" || in.FreshInstall { + t.Fatalf("raw values/defaults changed: %+v", in) + } + in, err = cfg.storageInputs(base, storageFlags{DBPath: "cli.db"}, env) + if err != nil { + t.Fatal(err) + } + want, _ = filepath.Abs("cli.db") + if in.DBPath != want { + t.Fatal("CLI path did not win") + } + in, err = (&Config{}).storageInputs(base, storageFlags{}, func(string) string { return "" }) + if err != nil { + t.Fatal(err) + } + if _, err = dbconfig.ResolveStorage(in); err == nil { + t.Fatal("unknown existing install was guessed fresh") + } + in.FreshInstall = true + got, err := dbconfig.ResolveStorage(in) + if err != nil { + t.Fatal(err) + } + if got.Backend != dbconfig.SQLite || got.DBPath != filepath.Join(base, "data", "meshcore.db") { + t.Fatalf("fresh default: %+v", got) + } +} + +func TestStorageInputsWriterURLAndStatePrecedence(t *testing.T) { + base := t.TempDir() + cfg := &Config{DB: &DBConfig{Backend: dbconfig.Postgres}, DatabaseURL: "postgresql://reader@localhost/config", StateDir: "configured"} + env := func(k string) string { + switch k { + case "CORESCOPE_DATABASE_URL": + return "postgresql://reader@localhost/generic" + case "CORESCOPE_WRITER_DATABASE_URL": + return "postgresql://reader@localhost/reader" + case "CORESCOPE_STATE_DIR": + return "state" + } + return "" + } + in, err := cfg.storageInputs(base, storageFlags{}, env) + if err != nil { + t.Fatal(err) + } + got, err := dbconfig.ResolveStorage(in) + if err != nil { + t.Fatal(err) + } + if got.WriterDatabaseURL != env("CORESCOPE_WRITER_DATABASE_URL") || got.StateDir != filepath.Join(base, "state") { + t.Fatal("reader environment precedence changed") + } + in, err = cfg.storageInputs(base, storageFlags{DatabaseURL: "postgresql://reader@localhost/cli", StateDir: "cli-state"}, env) + if err != nil { + t.Fatal(err) + } + got, err = dbconfig.ResolveStorage(in) + if err != nil { + t.Fatal(err) + } + if got.ReaderDatabaseURL != "postgresql://reader@localhost/cli" || got.WriterDatabaseURL != "postgresql://reader@localhost/cli" || got.StateDir != filepath.Join(base, "cli-state") { + t.Fatal("CLI overrides lost") + } +} diff --git a/cmd/ingestor/storage_runtime_open_test.go b/cmd/ingestor/storage_runtime_open_test.go new file mode 100644 index 000000000..e182d164b --- /dev/null +++ b/cmd/ingestor/storage_runtime_open_test.go @@ -0,0 +1,121 @@ +package main + +import ( + "crypto/sha256" + "database/sql" + "errors" + "os" + "path/filepath" + "testing" + + "github.com/meshcore-analyzer/dbconfig" +) + +func TestSelectedTelemetryRejectsWrongOrUninitializedSchemaWithoutMutation(t *testing.T) { + for _, query := range []string{`PRAGMA user_version=1`, `CREATE TABLE unrelated(value TEXT); INSERT INTO unrelated VALUES('keep')`} { + path := filepath.Join(t.TempDir(), "existing.db") + db, err := sql.Open("sqlite3", path) + if err != nil { + t.Fatal(err) + } + if _, err = db.Exec(query); err != nil { + t.Fatal(err) + } + db.Close() + before, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if s, err := OpenStoreStorage(dbconfig.Storage{Backend: dbconfig.SQLite, DBPath: path}, nil); err == nil { + s.Close() + t.Fatal("unrelated or uninitialized store accepted") + } + after, err := os.ReadFile(path) + if err != nil || sha256.Sum256(before) != sha256.Sum256(after) { + t.Fatal("rejected telemetry store changed", err) + } + } +} + +func TestSelectedTelemetryCannotRecreateRemovedVolume(t *testing.T) { + dir := filepath.Join(t.TempDir(), "missing-volume") + if s, err := OpenStoreStorage(dbconfig.Storage{Backend: dbconfig.SQLite, DBPath: filepath.Join(dir, "telemetry.db")}, nil); err == nil { + s.Close() + t.Fatal("missing volume accepted") + } + if _, err := os.Stat(dir); !errors.Is(err, os.ErrNotExist) { + t.Fatal("runtime recreated missing volume", err) + } +} + +func TestSQLiteTelemetryTargetRemovedAfterDSNValidation(t *testing.T) { + for _, replaceEmpty := range []bool{false, true} { + path := filepath.Join(t.TempDir(), "telemetry.db") + seed, err := OpenStore(path) + if err != nil { + t.Fatal(err) + } + seed.Close() + dsn, err := sqliteTelemetryWriterDSN(path, false) + if err != nil { + t.Fatal(err) + } + if err = os.Rename(path, path+".preserved"); err != nil { + t.Fatal(err) + } + if replaceEmpty { + if err = os.WriteFile(path, nil, 0600); err != nil { + t.Fatal(err) + } + } + db, err := sql.Open("sqlite3", dsn) + if err != nil { + t.Fatal(err) + } + if err = validateExistingTelemetry(db); err == nil { + t.Fatal("raced missing/empty target accepted") + } + db.Close() + info, err := os.Stat(path) + if replaceEmpty { + if err != nil || info.Size() != 0 { + t.Fatal("raced empty file was initialized", err) + } + } else if !errors.Is(err, os.ErrNotExist) { + t.Fatal("raced missing file was recreated", err) + } + } +} + +func TestSelectedTelemetryRetainsRowsSequenceAndDurability(t *testing.T) { + path := filepath.Join(t.TempDir(), "telemetry.db") + seed, err := OpenStore(path) + if err != nil { + t.Fatal(err) + } + if _, err = seed.InsertTransmission(&PacketData{Hash: "preserved", RawHex: "AA", Timestamp: "2026-01-01T00:00:00Z", PathJSON: "[]"}); err != nil { + t.Fatal(err) + } + if _, err = seed.db.Exec(`INSERT INTO transmissions(id,raw_hex,hash,first_seen) VALUES(900,'AA','retired','2026-01-01T00:00:00Z'); DELETE FROM transmissions WHERE id=900`); err != nil { + t.Fatal(err) + } + seed.Close() + s, err := OpenStoreStorage(dbconfig.Storage{Backend: dbconfig.SQLite, DBPath: path}, nil) + if err != nil { + t.Fatal(err) + } + defer s.Close() + var count, sequence int + if err = s.db.QueryRow(`SELECT count(*) FROM transmissions WHERE hash='preserved'`).Scan(&count); err != nil || count != 1 { + t.Fatal("stored row lost", count, err) + } + if err = s.db.QueryRow(`SELECT seq FROM sqlite_sequence WHERE name='transmissions'`).Scan(&sequence); err != nil || sequence != 900 { + t.Fatal("allocator reset", sequence, err) + } + for pragma, want := range map[string]string{"journal_mode": "wal", "synchronous": "2", "foreign_keys": "1"} { + var got string + if err = s.db.QueryRow("PRAGMA " + pragma).Scan(&got); err != nil || got != want { + t.Fatalf("%s=%s: %v", pragma, got, err) + } + } +} diff --git a/cmd/migrate/account_setup.go b/cmd/migrate/account_setup.go new file mode 100644 index 000000000..b139b11ca --- /dev/null +++ b/cmd/migrate/account_setup.go @@ -0,0 +1,186 @@ +package main + +import ( + "context" + "errors" + "os" + "reflect" + + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/pgutil" + "github.com/meshcore-analyzer/users" +) + +// The existing journal identifies this one same-backend transition without a +// separate initialization flag or sidecar. Its telemetry target never changes. +func accountInitialization(current, next dbconfig.Selection) bool { + return current.Generation != "" && current.Accounts == nil && next.Accounts != nil && current.Backend == next.Backend && current.StateDir == next.StateDir && reflect.DeepEqual(current.Telemetry, next.Telemetry) +} + +func initializeAccounts(ctx context.Context, o storageOptions, current dbconfig.Selection, raw dbconfig.StorageInputs) error { + update, err := dbconfig.BeginSelectionUpdate(o.SelectionFile, current.Generation) + if err != nil { + return err + } + defer update.Close() + // Failed validation before staging has done no account work. A real process + // crash still leaves the journal for an explicit metadata-only abort. + defer func() { + if update.Target().Generation == "" { + _ = update.Abort() + } + }() + if err := validateRuntimeSelection(ctx, current, raw); err != nil { + return err + } + bound, err := current.ApplyTo(raw) + if err != nil { + return err + } + bound.ExistingBackend = "" // This explicit setup, alone, supplies a new account default. + if current.Backend == dbconfig.SQLite { + bound.UsersDBPath = raw.UsersDBPath + } else { + if o.UsersOwnerURL == "" || raw.UsersDatabaseURL == "" { + return errors.New("first PostgreSQL account setup requires both account owner and runtime writer URLs") + } + bound.UsersDatabaseURL = o.UsersOwnerURL + bound.ApprovedChannelsDatabaseURL = "" + } + storage, err := dbconfig.ResolveStorage(bound) + if err != nil { + return err + } + next, err := dbconfig.NewSelection(storage) + if err != nil { + return err + } + if !accountInitialization(current, next) { + return errors.New("account setup cannot change the selected telemetry target") + } + if current.Backend == dbconfig.Postgres { + if err := validateInitialOwnerTargets(ctx, next, storage); err != nil { + return err + } + } + if err := update.Stage(next); err != nil { + return err + } + return finishAccountSetup(ctx, o, update, raw, false) +} + +func finishAccountSetup(ctx context.Context, o storageOptions, update *dbconfig.SelectionUpdate, raw dbconfig.StorageInputs, resume bool) error { + current, next := update.Current(), update.Target() + if !accountInitialization(current, next) { + return errors.New("pending journal is not a supported first account initialization") + } + if err := validateRuntimeSelection(ctx, current, raw); err != nil { + return err + } + if next.Backend == dbconfig.SQLite { + path := next.Accounts.SQLitePath + telemetry, err := os.Stat(current.Telemetry.SQLitePath) + if err != nil { + return err + } + info, statErr := os.Stat(path) + if statErr == nil && os.SameFile(telemetry, info) { + return errors.New("account setup cannot use the telemetry file") + } + if statErr != nil && !errors.Is(statErr, os.ErrNotExist) { + return statErr + } + if resume { + if statErr != nil { + return errors.New("pending account target is missing; restore it or abort this uncommitted initialization before explicit setup; it was not recreated") + } + db, err := openSQLite(path, "ro") + if err != nil { + return err + } + err = assertSQLiteData(ctx, db, "accounts") + db.Close() + if err != nil { + return errors.New("account initialization did not reach a ready target; preserve it, abort the uncommitted update and rerun explicit setup") + } + } else if err := applySQLiteTarget(ctx, path, "accounts"); err != nil { + return err + } + db, err := openSQLite(path, "ro") + if err != nil { + return err + } + err = assertSQLiteData(ctx, db, "accounts") + db.Close() + if err != nil { + return err + } + } else { + if o.UsersOwnerURL == "" || raw.UsersDatabaseURL == "" { + return errors.New("first PostgreSQL account setup requires both account owner and runtime writer URLs") + } + ownerInputs := raw + ownerInputs.UsersDatabaseURL = o.UsersOwnerURL + ownerInputs.ApprovedChannelsDatabaseURL = "" + bound, err := next.ApplyTo(ownerInputs) + if err != nil { + return err + } + owners, err := dbconfig.ResolveStorage(bound) + if err != nil { + return err + } + same, err := pgutil.SameDatabase(ctx, owners.WriterDatabaseURL, owners.UsersDatabaseURL) + if err != nil { + return err + } + if same { + return errors.New("accounts require a separate PostgreSQL database") + } + db, err := pgutil.Open(owners.UsersDatabaseURL, false) + if err != nil { + return err + } + readyErr := users.AssertPostgresReady(db) + if readyErr != nil { + if resume { + db.Close() + return errors.New("account initialization did not reach a ready PostgreSQL target; preserve it, abort the uncommitted update and rerun setup") + } + var objects int + err = db.QueryRowContext(ctx, importDestinationObjects).Scan(&objects) + if err == nil && objects != 0 { + err = errors.New("account target is neither a ready store nor a fresh empty PostgreSQL schema") + } + if err == nil { + err = users.ApplyPostgres(db) + } + } + if err == nil { + tables, layoutErr := layouts("accounts") + if layoutErr != nil { + err = layoutErr + } else { + err = validatePostgresLayout(db, "accounts", tables) + } + } + db.Close() + if err != nil { + return err + } + owners.WriterDatabaseURL = "" // Existing telemetry permissions are not part of first account setup. + if err := grantRuntimeTargets(ctx, next, owners, raw); err != nil { + return err + } + } + if err := validateRuntimeSelection(ctx, next, raw); err != nil { + return err + } + if o.beforeCommit != nil { + if err := o.beforeCommit(); err != nil { + return err + } + } + _, err := update.Commit(next) + return err +} diff --git a/cmd/migrate/account_setup_test.go b/cmd/migrate/account_setup_test.go new file mode 100644 index 000000000..0aeba5a27 --- /dev/null +++ b/cmd/migrate/account_setup_test.go @@ -0,0 +1,244 @@ +package main + +import ( + "bytes" + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/users" +) + +func TestLegacyAccountAbsenceFirstSetupAndRecordedLoss(t *testing.T) { + for _, key := range []string{"CORESCOPE_DB_BACKEND", "DB_PATH", "CORESCOPE_DATABASE_URL", "CORESCOPE_USERS_OWNER_DATABASE_URL", "CORESCOPE_USERS_DATABASE_URL", "CORESCOPE_APPROVED_CHANNELS_DATABASE_URL"} { + t.Setenv(key, "") + } + source := telemetrySource(t) + accounts := filepath.Join(t.TempDir(), "first-accounts.sqlite") + dir := t.TempDir() + o := storageOptions{Action: "adopt", Backend: "sqlite", Offline: true, SelectionFile: filepath.Join(dir, "storage-selection.json"), SQLitePath: source, UsersSQLitePath: accounts, ConfigDir: t.TempDir()} + if err := runStorageAction(context.Background(), o, &bytes.Buffer{}); err != nil { + t.Fatal(err) + } + selected, err := dbconfig.ReadSelection(o.SelectionFile) + if err != nil || selected.Accounts != nil { + t.Fatal("legacy absence was initialized", err) + } + if _, err := os.Stat(accounts); !os.IsNotExist(err) { + t.Fatal("legacy adoption created account file", err) + } + _, lease, err := dbconfig.OpenSelection(o.SelectionFile) + if err != nil { + t.Fatal(err) + } + o.Action = "setup" + if err := runStorageAction(context.Background(), o, &bytes.Buffer{}); !errors.Is(err, dbconfig.ErrSelectionBusy) { + t.Fatal("first initialization lacked exclusive lease", err) + } + lease.Close() + if _, err := os.Stat(accounts); !os.IsNotExist(err) { + t.Fatal("busy setup created account file", err) + } + if err := runStorageAction(context.Background(), o, &bytes.Buffer{}); err != nil { + t.Fatal("first explicit setup", err) + } + initialized, err := dbconfig.ReadSelection(o.SelectionFile) + if err != nil || initialized.Accounts == nil || initialized.Accounts.SQLitePath != accounts || initialized.Generation == selected.Generation { + t.Fatal("initialized target not recorded", err) + } + db, err := openSQLite(accounts, "ro") + if err != nil { + t.Fatal(err) + } + if err := users.AssertSQLiteReady(db); err != nil { + t.Fatal(err) + } + db.Close() + if err := os.Remove(accounts); err != nil { + t.Fatal(err) + } + if err := runStorageAction(context.Background(), o, &bytes.Buffer{}); err == nil { + t.Fatal("recorded missing accounts recreated") + } + if _, err := os.Stat(accounts); !os.IsNotExist(err) { + t.Fatal("lost account file replaced", err) + } + after, err := dbconfig.ReadSelection(o.SelectionFile) + if err != nil || after.Generation != initialized.Generation { + t.Fatal("failed setup changed recorded target", err) + } +} + +func TestLegacyDisabledExistingAccountStoreIsRecorded(t *testing.T) { + source, accounts := telemetrySource(t), accountSource(t, 5) + dir := t.TempDir() + config := filepath.Join(dir, "config.json") + if err := os.WriteFile(config, []byte(`{"userManagement":{"enabled":false}}`), 0600); err != nil { + t.Fatal(err) + } + o := storageOptions{Action: "adopt", Backend: "sqlite", Offline: true, SelectionFile: filepath.Join(dir, "storage-selection.json"), SQLitePath: source, UsersSQLitePath: accounts, ConfigFile: config} + if err := runStorageAction(context.Background(), o, &bytes.Buffer{}); err != nil { + t.Fatal(err) + } + selected, err := dbconfig.ReadSelection(o.SelectionFile) + if err != nil || selected.Accounts == nil || selected.Accounts.SQLitePath != accounts { + t.Fatal("disabled initialized store was omitted", err) + } + db, err := openSQLite(accounts, "ro") + if err != nil { + t.Fatal(err) + } + defer db.Close() + var id int + if err := db.QueryRow(`SELECT id FROM mail_events WHERE mail_id=4`).Scan(&id); err != nil || id != 900 { + t.Fatal("existing account data changed", id, err) + } +} + +func TestSQLiteAccountSetupResumeUsesOnlyItsJournal(t *testing.T) { + for _, missing := range []bool{false, true} { + t.Run(map[bool]string{false: "complete", true: "lost-target"}[missing], func(t *testing.T) { + source := telemetrySource(t) + accounts := filepath.Join(t.TempDir(), "accounts.sqlite") + dir := t.TempDir() + o := storageOptions{Action: "adopt", Backend: "sqlite", Offline: true, SelectionFile: filepath.Join(dir, "storage-selection.json"), SQLitePath: source, UsersSQLitePath: accounts, ConfigDir: t.TempDir()} + if err := runStorageAction(context.Background(), o, &bytes.Buffer{}); err != nil { + t.Fatal(err) + } + o.Action = "setup" + o.beforeCommit = func() error { return errors.New("stop before account selection") } + if err := runStorageAction(context.Background(), o, &bytes.Buffer{}); err == nil { + t.Fatal("injected cutover failure passed") + } + status, err := dbconfig.InspectSelection(o.SelectionFile) + if err != nil || status.State != "pending" || status.SourceBackend != dbconfig.SQLite || status.TargetBackend != dbconfig.SQLite { + t.Fatal("same-backend journal status", status, err) + } + if missing { + if err := os.Remove(accounts); err != nil { + t.Fatal(err) + } + } + t.Setenv("CORESCOPE_DATABASE_URL", "must-not-connect") + t.Setenv("CORESCOPE_USERS_OWNER_DATABASE_URL", "must-not-connect") + t.Setenv("CORESCOPE_USERS_DATABASE_URL", "must-not-connect") + o.Action = "resume" + o.JobID = status.JobID + o.beforeCommit = nil + err = runStorageAction(context.Background(), o, &bytes.Buffer{}) + if missing { + if err == nil || !strings.Contains(err.Error(), "not recreated") { + t.Fatal("lost staged account target was recreated", err) + } + if _, err := os.Stat(accounts); !os.IsNotExist(err) { + t.Fatal("missing staged target replaced") + } + o.Action = "abort" + if err := runStorageAction(context.Background(), o, &bytes.Buffer{}); err != nil { + t.Fatal(err) + } + } else if err != nil { + t.Fatal("SQLite-only account resume", err) + } + }) + } +} + +func TestUnstagedJournalNeedsNoDatabaseForAbort(t *testing.T) { + source := telemetrySource(t) + if err := applySQLiteTarget(context.Background(), source, "telemetry"); err != nil { + t.Fatal(err) + } + dir := t.TempDir() + selected, err := dbconfig.NewSelection(dbconfig.Storage{Backend: dbconfig.SQLite, StateDir: dir, DBPath: source}) + if err != nil { + t.Fatal(err) + } + path := dbconfig.SelectionPath(dir) + if _, err := dbconfig.AdoptSelection(path, selected); err != nil { + t.Fatal(err) + } + update, err := dbconfig.BeginSelectionUpdate(path, selected.Generation) + if err != nil { + t.Fatal(err) + } + id := update.ID() + update.Close() + t.Setenv("CORESCOPE_DATABASE_URL", "must-not-connect") + o := storageOptions{Action: "resume", Offline: true, SelectionFile: path, JobID: id, ConfigDir: t.TempDir()} + if err := runStorageAction(context.Background(), o, &bytes.Buffer{}); err == nil || !strings.Contains(err.Error(), "no staged destination") { + t.Fatal("unstaged job resume guessed a target", err) + } + o.Action = "abort" + if err := runStorageAction(context.Background(), o, &bytes.Buffer{}); err != nil { + t.Fatal("metadata-only abort", err) + } +} + +func TestPostgresAccountFirstSetupRequiresCredentialsAndResumes(t *testing.T) { + owner, accountOwner := postgresDatabase(t), postgresDatabase(t) + db := openImportDB(t, owner) + if err := dbschema.ApplyPostgres(db, nil); err != nil { + t.Fatal(err) + } + reader, writer := restrictedRole(t, owner), restrictedRole(t, owner) + dir := t.TempDir() + selected, err := dbconfig.NewSelection(dbconfig.Storage{Backend: dbconfig.Postgres, StateDir: dir, WriterDatabaseURL: owner}) + if err != nil { + t.Fatal(err) + } + raw := dbconfig.StorageInputs{BaseDir: dir, ReaderDatabaseURL: reader, WriterDatabaseURL: writer} + if err := grantRuntimeTargets(context.Background(), selected, dbconfig.Storage{WriterDatabaseURL: owner}, raw); err != nil { + t.Fatal(err) + } + selection := dbconfig.SelectionPath(dir) + if _, err := dbconfig.AdoptSelection(selection, selected); err != nil { + t.Fatal(err) + } + t.Setenv("CORESCOPE_DATABASE_URL", owner) + t.Setenv("CORESCOPE_READER_DATABASE_URL", reader) + t.Setenv("CORESCOPE_WRITER_DATABASE_URL", writer) + t.Setenv("CORESCOPE_USERS_OWNER_DATABASE_URL", "") + t.Setenv("CORESCOPE_USERS_DATABASE_URL", "") + t.Setenv("CORESCOPE_APPROVED_CHANNELS_DATABASE_URL", "") + o := storageOptions{Action: "setup", Offline: true, SelectionFile: selection, ConfigDir: t.TempDir()} + if err := runStorageAction(context.Background(), o, &bytes.Buffer{}); err != nil { + t.Fatal("ordinary PG setup invented an account target", err) + } + unchanged, err := dbconfig.ReadSelection(selection) + if err != nil || unchanged.Accounts != nil || unchanged.Generation != selected.Generation { + t.Fatal("ordinary PG setup changed absence", err) + } + t.Setenv("CORESCOPE_USERS_OWNER_DATABASE_URL", accountOwner) + if err := runStorageAction(context.Background(), o, &bytes.Buffer{}); err == nil { + t.Fatal("account initialization lacked runtime credentials") + } + unchanged, err = dbconfig.ReadSelection(selection) + if err != nil || unchanged.Accounts != nil { + t.Fatal("failed preflight changed selection", err) + } + t.Setenv("CORESCOPE_USERS_DATABASE_URL", restrictedRole(t, accountOwner)) + o.beforeCommit = func() error { return errors.New("interrupt account commit") } + if err := runStorageAction(context.Background(), o, &bytes.Buffer{}); err == nil { + t.Fatal("injected account commit passed") + } + status, err := dbconfig.InspectSelection(selection) + if err != nil || status.SourceBackend != dbconfig.Postgres || status.TargetBackend != dbconfig.Postgres { + t.Fatal("PG account journal", status, err) + } + o.Action = "resume" + o.JobID = status.JobID + o.beforeCommit = nil + if err := runStorageAction(context.Background(), o, &bytes.Buffer{}); err != nil { + t.Fatal("PG account initialization resume", err) + } + after, err := dbconfig.ReadSelection(selection) + if err != nil || after.Accounts == nil || after.Generation == selected.Generation { + t.Fatal("account target not atomically recorded", err) + } +} diff --git a/cmd/migrate/backend_test.go b/cmd/migrate/backend_test.go new file mode 100644 index 000000000..40e5f7d09 --- /dev/null +++ b/cmd/migrate/backend_test.go @@ -0,0 +1,41 @@ +package main + +import ( + "fmt" + "github.com/meshcore-analyzer/pgutil/pgtest" + "os" + "testing" +) + +func TestMain(m *testing.M) { + backend := os.Getenv("CORESCOPE_TEST_BACKEND") + if backend != "" && backend != "sqlite" && backend != "postgres" { + fmt.Fprintln(os.Stderr, "unsupported CORESCOPE_TEST_BACKEND") + os.Exit(1) + } + if backend == "postgres" && os.Getenv("CORESCOPE_TEST_POSTGRES_URL") == "" { + fmt.Fprintln(os.Stderr, "explicit PostgreSQL matrix requires CORESCOPE_TEST_POSTGRES_URL") + os.Exit(1) + } + os.Exit(m.Run()) +} + +func requirePostgres(t testing.TB) { + t.Helper() + switch os.Getenv("CORESCOPE_TEST_BACKEND") { + case "", "sqlite": + t.Skip("select CORESCOPE_TEST_BACKEND=postgres for the required PostgreSQL matrix") + case "postgres": + if os.Getenv("CORESCOPE_TEST_POSTGRES_URL") == "" { + t.Fatal("explicit PostgreSQL matrix requires CORESCOPE_TEST_POSTGRES_URL") + } + default: + t.Fatal("unsupported CORESCOPE_TEST_BACKEND") + } +} +func postgresSchema(t testing.TB) string { t.Helper(); requirePostgres(t); return pgtest.NewSchema(t) } +func postgresDatabase(t testing.TB) string { + t.Helper() + requirePostgres(t) + return pgtest.NewDatabase(t) +} diff --git a/cmd/migrate/cli_test.go b/cmd/migrate/cli_test.go new file mode 100644 index 000000000..3e79c41c3 --- /dev/null +++ b/cmd/migrate/cli_test.go @@ -0,0 +1,105 @@ +package main + +import ( + "bytes" + "context" + "database/sql" + "strings" + "testing" + + "github.com/meshcore-analyzer/dbschema" +) + +func TestCheckReadyDoesNotBootstrap(t *testing.T) { + dsn := postgresSchema(t) + t.Setenv("CORESCOPE_DATABASE_URL", dsn) + t.Setenv("CORESCOPE_USERS_DATABASE_URL", "") + var output bytes.Buffer + if err := runCommand(context.Background(), []string{"-check-ready"}, &output); err == nil { + t.Fatal("empty schema reported ready") + } + var n int + if err := openImportDB(t, dsn).QueryRow(`SELECT count(*) FROM information_schema.tables WHERE table_schema=current_schema()`).Scan(&n); err != nil { + t.Fatal(err) + } + if n != 0 { + t.Fatal("readiness check created tables") + } + if err := runCommand(context.Background(), nil, &output); err != nil { + t.Fatal(err) + } + if err := runCommand(context.Background(), []string{"-check-ready"}, &output); err != nil { + t.Fatal(err) + } + if strings.Contains(output.String(), dsn) { + t.Fatal("CLI output leaked a database URL") + } +} + +func TestCLIImportRequiresOfflineAndDoesNotFinalizeOnAccountFailure(t *testing.T) { + source := telemetrySource(t) + telemetry, accounts := postgresDatabase(t), postgresDatabase(t) + t.Setenv("CORESCOPE_DATABASE_URL", telemetry) + t.Setenv("CORESCOPE_USERS_DATABASE_URL", accounts) + var output bytes.Buffer + args := []string{"-from-sqlite", source, "-state-dir", t.TempDir()} + if err := runCommand(context.Background(), args, &output); err == nil { + t.Fatal("import accepted without explicit offline confirmation") + } + args = append(args, "-offline", "-users-from-sqlite", "missing-account-source.db") + if err := runCommand(context.Background(), args, &output); err == nil { + t.Fatal("missing account source was ignored") + } + if err := dbschema.AssertReady(openImportDB(t, telemetry)); err == nil { + t.Fatal("telemetry became ready despite failed account import") + } +} + +func TestCheckImportMarkerBindsExactSource(t *testing.T) { + dsn, source := postgresSchema(t), telemetrySource(t) + t.Setenv("CORESCOPE_DATABASE_URL", dsn) + t.Setenv("CORESCOPE_USERS_DATABASE_URL", "") + var output bytes.Buffer + args := []string{"-check-import-kind=telemetry", "-from-sqlite", source} + if err := runCommand(context.Background(), args, &output); err == nil { + t.Fatal("empty database passed imported-source check") + } + if _, err := importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: dsn, StateDir: t.TempDir(), Kind: "telemetry"}); err != nil { + t.Fatal(err) + } + if err := runCommand(context.Background(), args, &output); err == nil { + t.Fatal("unfinalized import passed bootstrap guard") + } + if _, err := finalizeImport(context.Background(), dsn, "telemetry"); err != nil { + t.Fatal(err) + } + if err := runCommand(context.Background(), args, &output); err != nil { + t.Fatal(err) + } + args[len(args)-1] = accountSource(t, 1) + if err := runCommand(context.Background(), args, &output); err == nil { + t.Fatal("different legacy source passed bootstrap guard") + } +} + +func TestWALModeCleanShutdownImportAndGuard(t *testing.T) { + source := telemetrySource(t) + db, err := sql.Open("sqlite3", source) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`PRAGMA journal_mode=WAL`); err != nil { + t.Fatal(err) + } + db.Close() + dsn := postgresSchema(t) + t.Setenv("CORESCOPE_DATABASE_URL", dsn) + t.Setenv("CORESCOPE_USERS_DATABASE_URL", "") + var output bytes.Buffer + if err := runCommand(context.Background(), []string{"-offline", "-from-sqlite", source, "-state-dir", t.TempDir()}, &output); err != nil { + t.Fatal(err) + } + if err := runCommand(context.Background(), []string{"-check-import-kind=telemetry", "-from-sqlite", source}, &output); err != nil { + t.Fatal(err) + } +} diff --git a/cmd/migrate/datetime_test.go b/cmd/migrate/datetime_test.go new file mode 100644 index 000000000..f1c2308bf --- /dev/null +++ b/cmd/migrate/datetime_test.go @@ -0,0 +1,152 @@ +package main + +import ( + "context" + "database/sql" + "errors" + "strings" + "testing" + + "github.com/meshcore-analyzer/dbschema" +) + +func dateSource(t *testing.T, declared string, values []any) string { + t.Helper() + source := telemetrySource(t) + db, err := openSQLite(source, "rw") + if err != nil { + t.Fatal(err) + } + defer db.Close() + if declared != "DATETIME" { + var ddl string + if err := db.QueryRow(`SELECT sql FROM sqlite_master WHERE name='dropped_packets'`).Scan(&ddl); err != nil { + t.Fatal(err) + } + if !strings.Contains(ddl, "dropped_at DATETIME") { + t.Fatal("legacy date declaration changed") + } + if _, err := db.Exec(`DROP TABLE dropped_packets`); err != nil { + t.Fatal(err) + } + if _, err := db.Exec(strings.Replace(ddl, "dropped_at DATETIME", "dropped_at "+declared, 1)); err != nil { + t.Fatal(err) + } + } + for i, value := range values { + if _, err := db.Exec(`INSERT INTO dropped_packets(id,reason,dropped_at) VALUES(?,?,?)`, i, "synthetic", value); err != nil { + t.Fatal(err) + } + } + if err := db.Close(); err != nil { + t.Fatal(err) + } + return source +} + +func TestImportPreservesDeclaredDateTextAndResume(t *testing.T) { + values := []any{ + nil, + "2026-10-09T01:02:03.123456789-04:30", + "2026-10-09 01:02:03.1200+00:00", + "2026-10-09T01:02:03Z", + "2026-10-09", + "", + "not a timestamp", + } + for _, declared := range []string{"DATETIME", "DATE", "TIMESTAMP"} { + t.Run(declared, func(t *testing.T) { + source := dateSource(t, declared, values) + before, err := sourceFingerprint(source) + if err != nil { + t.Fatal(err) + } + dsn := postgresSchema(t) + target := openImportDB(t, dsn) + options := importOptions{Source: source, DatabaseURL: dsn, StateDir: t.TempDir(), Kind: "telemetry", BatchSize: 2} + options.afterBatch = func() error { + var rows int + if err := target.QueryRow(`SELECT count(*) FROM dropped_packets`).Scan(&rows); err != nil { + return err + } + if rows == 2 { + return context.Canceled + } + return nil + } + if _, err := importSQLite(context.Background(), options); !errors.Is(err, context.Canceled) { + t.Fatalf("expected interruption after preserving two date rows: %v", err) + } + if err := dbschema.AssertReady(target); err == nil { + t.Fatal("interrupted date import became ready") + } + options.Resume, options.afterBatch = true, nil + report, err := importSQLite(context.Background(), options) + if err != nil || !report.Verified { + t.Fatalf("resume date import: verified=%v error=%v", report.Verified, err) + } + if _, err := finalizeImport(context.Background(), dsn, "telemetry"); err != nil { + t.Fatal(err) + } + if err := dbschema.AssertReady(target); err != nil { + t.Fatal(err) + } + rows, err := target.Query(`SELECT id,dropped_at FROM dropped_packets ORDER BY id`) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + count := 0 + for rows.Next() { + var id int + var value sql.NullString + if err := rows.Scan(&id, &value); err != nil { + t.Fatal(err) + } + if id != count || id >= len(values) { + t.Fatalf("date row identity changed: %d", id) + } + if want, ok := values[id].(string); ok { + if !value.Valid || value.String != want { + t.Errorf("date text changed at row %d: got %q want %q", id, value.String, want) + } + } else if value.Valid { + t.Errorf("NULL date became non-NULL at row %d", id) + } + count++ + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + if count != len(values) { + t.Fatalf("date rows duplicated or lost: %d", count) + } + after, err := sourceFingerprint(source) + if err != nil || before != after { + t.Fatal("date import changed the original SQLite source", err) + } + }) + } +} + +func TestImportRefusesNumericStorageInDeclaredDateColumn(t *testing.T) { + for _, value := range []any{int64(1700000000), 2.5, "12345"} { + source := dateSource(t, "DATETIME", []any{value}) + before, err := sourceFingerprint(source) + if err != nil { + t.Fatal(err) + } + dsn := postgresSchema(t) + report, err := importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: dsn, StateDir: t.TempDir(), Kind: "telemetry"}) + if err == nil || !strings.Contains(err.Error(), "non-text value") || report.Verified { + t.Fatalf("non-text date storage was silently coerced: verified=%v error=%v", report.Verified, err) + } + if err := dbschema.AssertReady(openImportDB(t, dsn)); err == nil { + t.Fatal("invalid date source became ready") + } + after, err := sourceFingerprint(source) + if err != nil || before != after { + t.Fatal("refused date import changed the source", err) + } + } +} diff --git a/cmd/migrate/destination_empty_test.go b/cmd/migrate/destination_empty_test.go new file mode 100644 index 000000000..519e674c9 --- /dev/null +++ b/cmd/migrate/destination_empty_test.go @@ -0,0 +1,37 @@ +package main + +import ( + "context" + "strings" + "testing" +) + +func TestFreshImportRefusesStandaloneSequence(t *testing.T) { + dsn := postgresSchema(t) + db := openImportDB(t, dsn) + if _, err := db.Exec(`CREATE SEQUENCE retained_sequence START WITH 77`); err != nil { + t.Fatal(err) + } + source := accountSource(t, 5) + before, err := fingerprint(context.Background(), source) + if err != nil { + t.Fatal(err) + } + _, err = importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: dsn, StateDir: t.TempDir(), Kind: "accounts"}) + if err == nil || !strings.Contains(err.Error(), "not empty") { + t.Fatalf("fresh import accepted a sequence-only occupied target: %v", err) + } + var value int + var called bool + if err := db.QueryRow(`SELECT last_value,is_called FROM retained_sequence`).Scan(&value, &called); err != nil || value != 77 || called { + t.Fatal("fresh import refusal changed the existing sequence", err) + } + var tables int + if err := db.QueryRow(`SELECT count(*) FROM information_schema.tables WHERE table_schema=current_schema()`).Scan(&tables); err != nil || tables != 0 { + t.Fatal("fresh import refusal created destination tables", err) + } + after, err := fingerprint(context.Background(), source) + if err != nil || before != after { + t.Fatal("fresh import refusal changed the retained source", err) + } +} diff --git a/cmd/migrate/generated_columns_test.go b/cmd/migrate/generated_columns_test.go new file mode 100644 index 000000000..56ed32272 --- /dev/null +++ b/cmd/migrate/generated_columns_test.go @@ -0,0 +1,81 @@ +package main + +import ( + "context" + "database/sql" + "path/filepath" + "strings" + "testing" + + "github.com/meshcore-analyzer/users" +) + +func TestImportRejectsGeneratedAccountColumnsBeforeNormalization(t *testing.T) { + for _, storage := range []string{"STORED", "VIRTUAL"} { + for _, column := range []string{"unsupported_note", "last_login_at"} { + t.Run(storage+"/"+column, func(t *testing.T) { + source := filepath.Join(t.TempDir(), "users.db") + db, err := sql.Open("sqlite3", source) + if err != nil { + t.Fatal(err) + } + defer db.Close() + if _, err := db.Exec(`CREATE TABLE schema_version(version INTEGER NOT NULL); INSERT INTO schema_version VALUES(1)`); err != nil { + t.Fatal(err) + } + for _, stmt := range users.LegacyMigrations()[0] { + if strings.HasPrefix(stmt, "CREATE TABLE users (") { + if column == "last_login_at" { + stmt = strings.Replace(stmt, "last_login_at INTEGER,", "last_login_at INTEGER GENERATED ALWAYS AS (created_at + 1) "+storage+",", 1) + } else { + stmt = strings.TrimSuffix(strings.TrimSpace(stmt), ")") + ", unsupported_note TEXT GENERATED ALWAYS AS (display_name || ':retained') " + storage + ")" + } + } + if _, err := db.Exec(stmt); err != nil { + t.Fatal(err) + } + } + if _, err := db.Exec(`INSERT INTO users(id,email,display_name,password_hash,role,status,created_at) VALUES(42,'synthetic@example.invalid','synthetic','hash','user','active',1)`); err != nil { + t.Fatal(err) + } + var value string + if err := db.QueryRow(`SELECT ` + quote(column) + ` FROM users WHERE id=42`).Scan(&value); err != nil || value == "" { + t.Fatal("generated fixture value missing", err) + } + if err := db.Close(); err != nil { + t.Fatal(err) + } + before, err := sourceFingerprint(source) + if err != nil { + t.Fatal(err) + } + dsn, state := postgresSchema(t), t.TempDir() + report, err := importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: dsn, StateDir: state, Kind: "accounts"}) + if err == nil || !strings.Contains(err.Error(), "generated or hidden") { + t.Errorf("generated source must be explicitly refused before normalization: verified=%v error=%v", report.Verified, err) + } + target := openImportDB(t, dsn) + var tables int + if err := target.QueryRow(`SELECT COUNT(*) FROM information_schema.tables WHERE table_schema=current_schema()`).Scan(&tables); err != nil || tables != 0 { + t.Errorf("unsupported source changed the target: tables=%d error=%v", tables, err) + } + if err := users.AssertReady(target); err == nil { + t.Error("unsupported source became runtime-ready") + } + normalized, err := openSQLite(filepath.Join(state, "accounts", "normalized.sqlite"), "ro") + if err != nil { + t.Fatal(err) + } + defer normalized.Close() + var version int + if err := normalized.QueryRow(`SELECT version FROM schema_version`).Scan(&version); err != nil || version != 1 { + t.Errorf("unsupported source reached account normalization: version=%d error=%v", version, err) + } + after, err := sourceFingerprint(source) + if err != nil || before != after { + t.Fatal("original source changed", err) + } + }) + } + } +} diff --git a/cmd/migrate/go.mod b/cmd/migrate/go.mod index b380d96ca..abaa9c9b0 100644 --- a/cmd/migrate/go.mod +++ b/cmd/migrate/go.mod @@ -1,22 +1,48 @@ module github.com/corescope/migrate -go 1.22 +go 1.25.0 require ( + github.com/jackc/pgx/v5 v5.7.6 + github.com/mattn/go-sqlite3 v1.14.52 + github.com/meshcore-analyzer/dbconfig v0.0.0 github.com/meshcore-analyzer/dbschema v0.0.0 - modernc.org/sqlite v1.34.5 + github.com/meshcore-analyzer/pgutil v0.0.0 + github.com/meshcore-analyzer/sqliteutil v0.0.0 + github.com/meshcore-analyzer/users v0.0.0 ) -replace github.com/meshcore-analyzer/dbschema => ../../internal/dbschema - require ( github.com/dustin/go-humanize v1.0.1 // indirect github.com/google/uuid v1.6.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/ncruces/go-strftime v0.1.9 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect - golang.org/x/sys v0.22.0 // indirect modernc.org/libc v1.55.3 // indirect modernc.org/mathutil v1.6.0 // indirect modernc.org/memory v1.8.0 // indirect + modernc.org/sqlite v1.34.5 // indirect +) + +require ( + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/meshcore-analyzer/packetpath v0.0.0 + golang.org/x/crypto v0.37.0 // indirect + golang.org/x/sync v0.13.0 // indirect + golang.org/x/sys v0.32.0 // indirect + golang.org/x/text v0.24.0 // indirect ) + +replace github.com/meshcore-analyzer/dbschema => ../../internal/dbschema + +replace github.com/meshcore-analyzer/users => ../../internal/users + +replace github.com/meshcore-analyzer/pgutil => ../../internal/pgutil + +replace github.com/meshcore-analyzer/packetpath => ../../internal/packetpath + +replace github.com/meshcore-analyzer/dbconfig => ../../internal/dbconfig + +replace github.com/meshcore-analyzer/sqliteutil => ../../internal/sqliteutil diff --git a/cmd/migrate/go.sum b/cmd/migrate/go.sum index 5424fe41d..eb0d9a0d8 100644 --- a/cmd/migrate/go.sum +++ b/cmd/migrate/go.sum @@ -1,22 +1,52 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo= github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.7.6 h1:rWQc5FwZSPX58r1OQmkuaNicxdmExaEz5A2DO2hUuTk= +github.com/jackc/pgx/v5 v5.7.6/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-sqlite3 v1.14.52 h1:wVbm2Qnf4OXkqhBTSPuCRZDRnxfbVrrmiCEroVdog8U= +github.com/mattn/go-sqlite3 v1.14.52/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w= github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= -golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= +golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= +golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= -golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw= -golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc= +golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= +golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= +golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= modernc.org/cc/v4 v4.21.4 h1:3Be/Rdo1fpr8GrQ7IVw9OHtplU4gWbb+wNgeoBMmGLQ= modernc.org/cc/v4 v4.21.4/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ= modernc.org/ccgo/v4 v4.19.2 h1:lwQZgvboKD0jBwdaeVCTouxhxAyN6iawF3STraAal8Y= diff --git a/cmd/migrate/grants.go b/cmd/migrate/grants.go new file mode 100644 index 000000000..d0f14de14 --- /dev/null +++ b/cmd/migrate/grants.go @@ -0,0 +1,123 @@ +package main + +import ( + "context" + "errors" + "strings" + + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/pgutil" +) + +// Same narrow policy as docker/postgres-grants.sql, applied to the verified +// unselected destination before runtime validation and the atomic selection +// commit. Runtime role names come from their explicit credential URLs; no +// password or connection URL enters a SQL statement or child process argv. +func grantRuntimeTargets(ctx context.Context, selected dbconfig.Selection, owners dbconfig.Storage, raw dbconfig.StorageInputs) error { + bound, err := selected.ApplyTo(raw) + if err != nil { + return err + } + runtime, err := dbconfig.ResolveStorage(bound) + if err != nil { + return err + } + role := func(dsn string) (string, error) { + if dsn == "" { + return "", errors.New("prepare the runtime role credentials before backend selection") + } + config, err := pgutil.ParseConfig(dsn) + if err != nil { + return "", err + } + return quote(config.User), nil + } + reader, err := role(runtime.ReaderDatabaseURL) + if err != nil { + return err + } + writer, err := role(runtime.WriterDatabaseURL) + if err != nil { + return err + } + if reader == writer { + return errors.New("PostgreSQL telemetry reader and writer must use separate restricted roles") + } + type grants struct { + kind, owner, schema string + roles []string + queries []string + } + telemetry := grants{kind: "telemetry", owner: owners.WriterDatabaseURL, schema: quote(selected.Telemetry.Postgres.Schema), roles: []string{reader, writer}} + telemetry.queries = []string{ + `GRANT SELECT ON ALL TABLES IN SCHEMA ` + telemetry.schema + ` TO ` + reader, + `GRANT SELECT ON ALL SEQUENCES IN SCHEMA ` + telemetry.schema + ` TO ` + reader, + `GRANT SELECT ON corescope_schema,packets_v TO ` + writer, + `GRANT USAGE,SELECT ON ALL SEQUENCES IN SCHEMA ` + telemetry.schema + ` TO ` + writer, + } + tables, err := layouts("telemetry") + if err != nil { + return err + } + for _, table := range tables { + telemetry.queries = append(telemetry.queries, `GRANT SELECT,INSERT,UPDATE,DELETE ON TABLE `+quote(table.Name)+` TO `+writer) + } + stores := []grants{telemetry} + if selected.Accounts != nil { + account, err := role(runtime.UsersDatabaseURL) + if err != nil { + return err + } + entry := grants{kind: "accounts", owner: owners.UsersDatabaseURL, schema: quote(selected.Accounts.Postgres.Schema), roles: []string{account}} + entry.queries = []string{`GRANT SELECT ON ALL TABLES IN SCHEMA ` + entry.schema + ` TO ` + account, `GRANT USAGE,SELECT ON ALL SEQUENCES IN SCHEMA ` + entry.schema + ` TO ` + account} + tables, err := layouts("accounts") + if err != nil { + return err + } + for _, table := range tables { + entry.queries = append(entry.queries, `GRANT SELECT,INSERT,UPDATE,DELETE ON TABLE `+quote(table.Name)+` TO `+account) + } + if runtime.ApprovedChannelsDatabaseURL != "" { + channel, err := role(runtime.ApprovedChannelsDatabaseURL) + if err != nil { + return err + } + if channel == account { + return errors.New("account writer and channel projection reader must use separate restricted roles") + } + entry.roles = append(entry.roles, channel) + entry.queries = append(entry.queries, `GRANT SELECT ON approved_channels,corescope_schema TO `+channel) + } + stores = append(stores, entry) + } + for _, store := range stores { + if store.owner == "" { + continue + } // Account-only initialization leaves telemetry grants unchanged. + db, err := pgutil.Open(store.owner, false) + if err != nil { + return err + } + tx, err := db.BeginTx(ctx, nil) + if err != nil { + db.Close() + return err + } + queries := append([]string{`REVOKE CREATE ON SCHEMA ` + store.schema + ` FROM PUBLIC`, `GRANT USAGE ON SCHEMA ` + store.schema + ` TO ` + strings.Join(store.roles, ",")}, store.queries...) + for _, query := range queries { + if _, err = tx.ExecContext(ctx, query); err != nil { + break + } + } + if err == nil { + err = tx.Commit() + } else { + tx.Rollback() + } + db.Close() + if err != nil { + return err + } + } + return nil +} diff --git a/cmd/migrate/identity_layout_test.go b/cmd/migrate/identity_layout_test.go new file mode 100644 index 000000000..7eed801fb --- /dev/null +++ b/cmd/migrate/identity_layout_test.go @@ -0,0 +1,107 @@ +package main + +import ( + "context" + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/users" + "path/filepath" + "testing" +) + +func TestNewSQLiteIdentitySourceLayouts(t *testing.T) { + for _, kind := range []string{"telemetry", "accounts"} { + t.Run(kind, func(t *testing.T) { + source := filepath.Join(t.TempDir(), "identity # % é.sqlite") + db, err := openSQLite(source, "rwc") + if err != nil { + t.Fatal(err) + } + if kind == "telemetry" { + err = dbschema.ApplySQLite(db, nil) + } else { + err = users.ApplySQLite(db) + } + if err != nil { + db.Close() + t.Fatal(err) + } + if kind == "accounts" { + for _, q := range []string{ + `INSERT INTO users(id,email,display_name,password_hash,created_at) VALUES(5,'test@example.invalid','','hash',1)`, + `INSERT INTO mail_log(id,user_id,to_email,purpose,sent_at,last_event_at) VALUES(4,5,'test@example.invalid','test',1,1)`, + `INSERT INTO mail_events(id,mail_id,event,at) VALUES(-9,4,'zero',1),(0,4,'negative',2),(900,4,'delivered',3)`, + `UPDATE sqlite_sequence SET seq=9000 WHERE name='mail_events'`, + } { + if _, err := db.Exec(q); err != nil { + t.Fatal(err) + } + } + } + db.Close() + before, err := sourceFingerprint(source) + if err != nil { + t.Fatal(err) + } + tables, err := layouts(kind) + if err != nil { + t.Fatal(err) + } + _, normalized, err := prepareSource(context.Background(), importOptions{Source: source, StateDir: t.TempDir(), Kind: kind, targetDigest: "test-only"}, tables, true) + if err != nil { + t.Fatal(err) + } + after, err := sourceFingerprint(source) + if err != nil || after != before { + t.Fatal("source changed", err) + } + copyDB, err := openSQLite(normalized, "ro") + if err != nil { + t.Fatal(err) + } + defer copyDB.Close() + if kind == "accounts" { + var n, seq, version int + if err := copyDB.QueryRow(`SELECT count(*) FROM mail_events WHERE id IN (-9,0,900)`).Scan(&n); err != nil || n != 3 { + t.Fatal("explicit event IDs changed", n, err) + } + if err := copyDB.QueryRow(`SELECT seq FROM sqlite_sequence WHERE name='mail_events'`).Scan(&seq); err != nil || seq != 9000 { + t.Fatal("event sequence changed", seq, err) + } + if err := copyDB.QueryRow(`SELECT version FROM schema_version`).Scan(&version); err != nil || version != 6 { + t.Fatal("SQLite source version changed", version, err) + } + } + }) + } +} + +func TestForwardNewSQLiteAccountIDsAndSequence(t *testing.T) { + dsn := postgresSchema(t) + source := accountSource(t, 5) + db, err := openSQLite(source, "rw") + if err != nil { + t.Fatal(err) + } + if err := users.ApplySQLite(db); err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`UPDATE sqlite_sequence SET seq=9000 WHERE name='mail_events'`); err != nil { + t.Fatal(err) + } + db.Close() + report, err := importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: dsn, StateDir: t.TempDir(), Kind: "accounts"}) + if err != nil || !report.Verified { + t.Fatal("forward v6", err) + } + pg := openImportDB(t, dsn) + var id, version int + if err := pg.QueryRow(`SELECT id FROM mail_events WHERE mail_id=4`).Scan(&id); err != nil || id != 900 { + t.Fatal("mail event identity", id, err) + } + if err := pg.QueryRow(`SELECT version FROM schema_version`).Scan(&version); err != nil || version != 5 { + t.Fatal("PG version", version, err) + } + if err := pg.QueryRow(`INSERT INTO mail_events(mail_id,event,at) VALUES(4,'later',44) RETURNING id`).Scan(&id); err != nil || id != 9001 { + t.Fatal("mail sequence", id, err) + } +} diff --git a/cmd/migrate/import.go b/cmd/migrate/import.go new file mode 100644 index 000000000..ac8552580 --- /dev/null +++ b/cmd/migrate/import.go @@ -0,0 +1,879 @@ +package main + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "database/sql" + "encoding/binary" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "hash" + "math" + "os" + "path/filepath" + "slices" + "strings" + "unicode/utf8" + + "github.com/jackc/pgx/v5" + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/pgutil" + "github.com/meshcore-analyzer/users" +) + +type importOptions struct { + Source, DatabaseURL, StateDir, Kind string + BatchSize int + Resume bool + afterBatch func() error + targetDigest string +} +type tableReport struct { + Table string `json:"table"` + Rows int64 `json:"rows"` + SHA256 string `json:"sha256"` +} +type importReport struct { + Kind string `json:"kind"` + Verified bool `json:"verified"` + Tables []tableReport `json:"tables"` +} +type sourceManifest struct { + Version int `json:"version"` + RunID string `json:"run_id"` + Kind string `json:"kind"` + SourceDigest string `json:"source_digest"` + RawDigest string `json:"raw_digest"` + NormalizedDigest string `json:"normalized_digest"` + TargetDigest string `json:"target_digest"` +} + +const importerLock int64 = 7289041410 + +const importDestinationObjects = `SELECT count(*) FROM pg_class WHERE relnamespace=current_schema()::regnamespace AND relkind IN ('r','p','v','m','S','f')` + +func fingerprint(ctx context.Context, path string) (string, error) { + fp, err := sourceFingerprintContext(ctx, path) + if err != nil { + return "", err + } + return fmt.Sprintf("%x:%x", fp[0], fp[1]), nil +} + +func prepareSource(ctx context.Context, o importOptions, tables []importTable, allowNew bool) (sourceManifest, string, error) { + var manifest sourceManifest + if o.Source == "" || o.StateDir == "" { + return manifest, "", errors.New("source and state directory are required for import") + } + digest, err := fingerprint(ctx, o.Source) + if err != nil { + return manifest, "", err + } + dir := filepath.Join(o.StateDir, o.Kind) + raw := filepath.Join(dir, "recovery.sqlite") + normalized := filepath.Join(dir, "normalized.sqlite") + path := filepath.Join(dir, "manifest.json") + data, err := os.ReadFile(path) + if err == nil { + if !o.Resume { + return manifest, "", errors.New("migration state already exists; use -resume with the same source and destination") + } + if err := json.Unmarshal(data, &manifest); err != nil { + return manifest, "", errors.New("invalid migration state manifest") + } + if manifest.Version != 1 || manifest.Kind != o.Kind || manifest.SourceDigest != digest || manifest.TargetDigest != o.targetDigest { + return manifest, "", errors.New("migration resume source or destination does not match the immutable manifest") + } + for p, want := range map[string]string{raw: manifest.RawDigest, normalized: manifest.NormalizedDigest} { + got, err := fingerprint(ctx, p) + if err != nil { + return manifest, "", err + } + if got != want { + return manifest, "", errors.New("migration recovery or normalized snapshot changed; resume refused") + } + } + return manifest, normalized, nil + } + if !errors.Is(err, os.ErrNotExist) { + return manifest, "", err + } + entries, err := os.ReadDir(dir) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return manifest, "", err + } + if len(entries) != 0 { + return manifest, "", errors.New("preparation stopped before a complete manifest; preserve this state directory and the original SQLite files, then use a new state directory for this store only after confirming its PostgreSQL destination is empty") + } + if !allowNew { + return manifest, "", errors.New("no complete migration state manifest exists to resume") + } + if err := os.MkdirAll(dir, 0o700); err != nil { + return manifest, "", err + } + if err := snapshotSource(ctx, o.Source, raw); err != nil { + return manifest, "", err + } + if err := snapshotSource(ctx, raw, normalized); err != nil { + return manifest, "", err + } + db, err := openSQLite(normalized, "rw") + if err != nil { + return manifest, "", err + } + db.SetMaxOpenConns(1) + if _, err = db.ExecContext(ctx, `PRAGMA trusted_schema=OFF`); err == nil { + err = normalizeSource(db, o.Kind, raw, tables) + } + closeErr := db.Close() + if err != nil { + return manifest, "", err + } + if closeErr != nil { + return manifest, "", closeErr + } + manifest = sourceManifest{Version: 1, Kind: o.Kind, SourceDigest: digest, TargetDigest: o.targetDigest} + var random [16]byte + if _, err := rand.Read(random[:]); err != nil { + return manifest, "", err + } + manifest.RunID = hex.EncodeToString(random[:]) + if manifest.RawDigest, err = fingerprint(ctx, raw); err != nil { + return manifest, "", err + } + if manifest.NormalizedDigest, err = fingerprint(ctx, normalized); err != nil { + return manifest, "", err + } + after, err := fingerprint(ctx, o.Source) + if err != nil { + return manifest, "", err + } + if after != digest { + return manifest, "", errors.New("SQLite source changed during preparation; keep every instance writer stopped") + } + data, err = json.MarshalIndent(manifest, "", " ") + if err != nil { + return manifest, "", err + } + f, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if err != nil { + return manifest, "", err + } + _, writeErr := f.Write(data) + syncErr := f.Sync() + closeErr = f.Close() + if err := errors.Join(writeErr, syncErr, closeErr); err != nil { + return manifest, "", err + } + return manifest, normalized, nil +} + +func importSQLite(ctx context.Context, o importOptions) (importReport, error) { + report := importReport{Kind: o.Kind} + if err := ctx.Err(); err != nil { + return report, err + } + if o.BatchSize == 0 { + o.BatchSize = 1000 + } + if o.BatchSize < 1 || o.BatchSize > 10000 { + return report, errors.New("import batch size must be between 1 and 10000") + } + tables, err := layouts(o.Kind) + if err != nil { + return report, err + } + config, err := pgutil.ParseConfig(o.DatabaseURL) + if err != nil { + return report, err + } + owner, err := pgutil.Open(o.DatabaseURL, false) + if err != nil { + return report, err + } + defer owner.Close() + owner.SetMaxOpenConns(1) + var database, schema string + var databaseOID, schemaOID int64 + if err := owner.QueryRowContext(ctx, `SELECT current_database(),current_schema(), + (SELECT oid::bigint FROM pg_database WHERE datname=current_database()), + (SELECT oid::bigint FROM pg_namespace WHERE nspname=current_schema())`).Scan(&database, &schema, &databaseOID, &schemaOID); err != nil { + return report, err + } + // Container IPs can change while their persistent volume and configured DNS + // endpoint remain the same. Bind resume to that endpoint and logical OIDs; + // the destination's random import run ID is verified separately below. + identity, _ := json.Marshal([]any{database, schema, databaseOID, schemaOID, config.Host, config.Port}) + identityHash := sha256.Sum256(identity) + o.targetDigest = hex.EncodeToString(identityHash[:]) + conn, err := pgx.ConnectConfig(ctx, config) + if err != nil { + return report, errors.New("connect to PostgreSQL import destination failed") + } + defer conn.Close(context.Background()) + // One importer owns this database until completion/cancellation. Session lock + // works across committed COPY batches; closing the connection always releases it. + if err := lockImport(ctx, conn); err != nil { + return report, err + } + allowNew := !o.Resume + if o.Resume { + var hasImport bool + if err := conn.QueryRow(ctx, `SELECT to_regclass('corescope_import') IS NOT NULL`).Scan(&hasImport); err != nil { + return report, err + } + if !hasImport { + var occupied int + if err := conn.QueryRow(ctx, importDestinationObjects).Scan(&occupied); err != nil { + return report, err + } + if occupied != 0 { + return report, errors.New("resume destination has no import marker and is not empty; existing data was not changed") + } + allowNew = true + } + } + manifest, path, err := prepareSource(ctx, o, tables, allowNew) + if err != nil { + return report, err + } + source, err := openSQLite(path, "ro") + if err != nil { + return report, err + } + defer source.Close() + source.SetMaxOpenConns(1) + if _, err := source.ExecContext(ctx, `PRAGMA trusted_schema=OFF`); err != nil { + return report, err + } + + var exists bool + if err := conn.QueryRow(ctx, `SELECT to_regclass('corescope_import') IS NOT NULL`).Scan(&exists); err != nil { + return report, err + } + if !exists { + var occupied int + if err := conn.QueryRow(ctx, importDestinationObjects).Scan(&occupied); err != nil { + return report, err + } + if occupied != 0 { + return report, errors.New("import destination is not empty; existing data was not changed") + } + tx, err := owner.BeginTx(ctx, nil) + if err != nil { + return report, err + } + defer tx.Rollback() + if o.Kind == "telemetry" { + err = dbschema.ApplyForImportTx(tx, nil) + } else { + err = users.ApplyForImportTx(tx) + } + if err != nil { + return report, err + } + if _, err := tx.ExecContext(ctx, `CREATE TABLE corescope_import (id INTEGER PRIMARY KEY CHECK(id=1),run_id TEXT NOT NULL,kind TEXT NOT NULL,source_digest TEXT NOT NULL,normalized_digest TEXT NOT NULL,verified BOOLEAN NOT NULL DEFAULT false,report TEXT NOT NULL DEFAULT '') + ;CREATE TABLE corescope_import_progress (table_name TEXT PRIMARY KEY,rows_copied BIGINT NOT NULL,sha256 TEXT NOT NULL,complete BOOLEAN NOT NULL)`); err != nil { + return report, err + } + if _, err := tx.ExecContext(ctx, `INSERT INTO corescope_import(id,run_id,kind,source_digest,normalized_digest) VALUES(1,$1,$2,$3,$4)`, manifest.RunID, o.Kind, manifest.SourceDigest, manifest.NormalizedDigest); err != nil { + return report, err + } + if err := tx.Commit(); err != nil { + return report, err + } + } else if !o.Resume { + return report, errors.New("destination has an import in progress; use -resume") + } + var runID, kind, sourceDigest, normalizedDigest string + if err := conn.QueryRow(ctx, `SELECT run_id,kind,source_digest,normalized_digest FROM corescope_import WHERE id=1`).Scan(&runID, &kind, &sourceDigest, &normalizedDigest); err != nil { + return report, err + } + if runID != manifest.RunID || kind != o.Kind || sourceDigest != manifest.SourceDigest || normalizedDigest != manifest.NormalizedDigest { + return report, errors.New("destination import identity does not match this source/state directory") + } + var version int + var ready bool + if err := conn.QueryRow(ctx, `SELECT version,ready FROM corescope_schema WHERE kind=$1`, o.Kind).Scan(&version, &ready); err != nil { + return report, err + } + if version != 1 { + return report, errors.New("unsupported PostgreSQL schema version for import") + } + var recordedReport importReport + if ready { + // A process can stop between the two databases' readiness commits. Keep + // this completed store unchanged, but never trust readiness by itself. + tx, err := conn.Begin(ctx) + if err != nil { + return report, err + } + defer tx.Rollback(context.Background()) + lockTables := []string{"corescope_import", "corescope_import_progress", "corescope_schema"} + for _, table := range tables { + lockTables = append(lockTables, quote(table.Name)) + } + if _, err := tx.Exec(ctx, `LOCK TABLE `+strings.Join(lockTables, ",")+` IN SHARE MODE`); err != nil { + return report, err + } + var verified bool + var reportText string + if err := tx.QueryRow(ctx, `SELECT verified,report FROM corescope_import WHERE id=1`).Scan(&verified, &reportText); err != nil { + return report, err + } + if !verified || json.Unmarshal([]byte(reportText), &recordedReport) != nil || !recordedReport.Verified || recordedReport.Kind != o.Kind || len(recordedReport.Tables) != len(tables) { + return report, errors.New("ready destination has no complete verified import report; resume refused") + } + } + for _, table := range tables { + got, err := copyTable(ctx, source, owner, conn, table, o, ready) + if err != nil { + return report, fmt.Errorf("import %s: %w", table.Name, err) + } + report.Tables = append(report.Tables, got) + } + if err := reseed(ctx, source, conn, tables, o.Kind, ready); err != nil { + return report, err + } + // COPY does not populate planner statistics. Prepare only this store's + // imported tables before declaring verification complete; the cost belongs + // to migration timing, never to the steady-state ingestion benchmark. + if !ready { + for _, table := range tables { + if _, err := conn.Exec(ctx, `ANALYZE `+quote(table.Name)); err != nil { + return report, fmt.Errorf("analyze imported %s: %w", table.Name, err) + } + } + } + // Source stays frozen through verification, not merely while snapshotting. + current, err := fingerprint(ctx, o.Source) + if err != nil { + return report, err + } + if current != manifest.SourceDigest { + return report, errors.New("original SQLite source changed during import; keep services stopped") + } + normalized, err := fingerprint(ctx, path) + if err != nil { + return report, err + } + if normalized != manifest.NormalizedDigest { + return report, errors.New("normalized snapshot changed during import") + } + report.Verified = true + if ready { + if !slices.Equal(report.Tables, recordedReport.Tables) { + return report, errors.New("ready destination differs from its verified import report; resume refused") + } + return report, nil + } + data, err := json.Marshal(report) + if err != nil { + return report, err + } + if _, err := conn.Exec(ctx, `UPDATE corescope_import SET verified=true,report=$1 WHERE id=1`, string(data)); err != nil { + return report, err + } + return report, nil +} + +func columnNames(table importTable) ([]string, []string) { + names := make([]string, len(table.Columns)) + quoted := make([]string, len(names)) + for i, col := range table.Columns { + names[i] = col.Name + quoted[i] = quote(col.Name) + } + return names, quoted +} + +func orderBy(db *sql.DB, table importTable, postgres, hiddenMailID bool) (string, error) { + // Sort by logical primary key with identical byte collation in both engines. + rows, err := db.Query(`SELECT a.attname FROM pg_index i JOIN pg_attribute a ON a.attrelid=i.indrelid AND a.attnum=ANY(i.indkey) + WHERE i.indrelid=to_regclass($1) AND i.indisprimary ORDER BY array_position(i.indkey,a.attnum)`, table.Name) + if err != nil { + return "", err + } + defer rows.Close() + var keys []string + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return "", err + } + keys = append(keys, name) + } + if err := rows.Err(); err != nil { + return "", err + } + if len(keys) == 0 { + return "", errors.New("destination table lacks a stable primary key") + } + var out []string + for _, name := range keys { + colType := "" + for _, col := range table.Columns { + if col.Name == name { + colType = col.Type + } + } + expr := quote(name) + if table.Name == "mail_events" && name == "id" && hiddenMailID { + expr = "rowid" + } + if colType == "TEXT" { + if postgres { + expr += ` COLLATE "C"` + } else { + expr += ` COLLATE BINARY` + } + } + if postgres { + expr += " NULLS FIRST" + } + out = append(out, expr) + } + return strings.Join(out, ","), nil +} + +func canonical(value any, kind string) (any, error) { + if value == nil { + return nil, nil + } + switch kind { + case "TEXT": + var s string + switch v := value.(type) { + case string: + s = v + case []byte: + s = string(v) + default: + return nil, errors.New("non-text value in a text column") + } + if !utf8.ValidString(s) || strings.ContainsRune(s, 0) { + return nil, errors.New("text contains invalid UTF-8 or NUL, which PostgreSQL cannot preserve") + } + return s, nil + case "BIGINT": + switch v := value.(type) { + case int64: + return v, nil + case int32: + return int64(v), nil + case int: + return int64(v), nil + case float64: + if math.Trunc(v) == v && v >= math.MinInt64 && v < 9223372036854775808.0 { + return int64(v), nil + } + } + return nil, errors.New("non-integral or out-of-range value in an integer column") + case "DOUBLE PRECISION": + switch v := value.(type) { + case float64: + return v, nil + case int64: + if int64(float64(v)) != v { + return nil, errors.New("integer cannot be represented exactly by the destination floating-point column") + } + return float64(v), nil + case int32: + return float64(v), nil + } + return nil, errors.New("non-numeric value in a floating-point column") + } + return nil, fmt.Errorf("unsupported destination column type %s", kind) +} + +func hashRow(h hash.Hash, values []any) { + for _, value := range values { + switch v := value.(type) { + case nil: + h.Write([]byte{0}) + case int64: + h.Write([]byte{1}) + var b [8]byte + binary.BigEndian.PutUint64(b[:], uint64(v)) + h.Write(b[:]) + case float64: + h.Write([]byte{2}) + var b [8]byte + binary.BigEndian.PutUint64(b[:], math.Float64bits(v)) + h.Write(b[:]) + case string: + h.Write([]byte{3}) + var b [8]byte + binary.BigEndian.PutUint64(b[:], uint64(len(v))) + h.Write(b[:]) + h.Write([]byte(v)) + } + } + h.Write([]byte{255}) +} + +func digestTarget(ctx context.Context, db *sql.DB, table importTable) (tableReport, error) { + report := tableReport{Table: table.Name} + _, quoted := columnNames(table) + order, err := orderBy(db, table, true, false) + if err != nil { + return report, err + } + rows, err := db.QueryContext(ctx, `SELECT `+strings.Join(quoted, ",")+` FROM `+quote(table.Name)+` ORDER BY `+order) + if err != nil { + return report, err + } + defer rows.Close() + h := sha256.New() + for rows.Next() { + values := make([]any, len(quoted)) + dest := make([]any, len(quoted)) + for i := range dest { + dest[i] = &values[i] + } + if err := rows.Scan(dest...); err != nil { + return report, err + } + for i, col := range table.Columns { + values[i], err = canonical(values[i], col.Type) + if err != nil { + return report, err + } + } + hashRow(h, values) + report.Rows++ + } + report.SHA256 = hex.EncodeToString(h.Sum(nil)) + return report, rows.Err() +} + +func copyTable(ctx context.Context, source, owner *sql.DB, conn *pgx.Conn, table importTable, o importOptions, verifyOnly bool) (tableReport, error) { + report := tableReport{Table: table.Name} + var copied int64 + var storedDigest string + var complete bool + err := conn.QueryRow(ctx, `SELECT rows_copied,sha256,complete FROM corescope_import_progress WHERE table_name=$1`, table.Name).Scan(&copied, &storedDigest, &complete) + if err != nil && !errors.Is(err, pgx.ErrNoRows) { + return report, err + } + if verifyOnly && !complete { + return report, errors.New("ready destination lacks complete import progress; resume refused") + } + existing, err := digestTarget(ctx, owner, table) + if err != nil { + return report, err + } + emptyHash := sha256.Sum256(nil) + if storedDigest == "" { + storedDigest = hex.EncodeToString(emptyHash[:]) + } + if existing.Rows != copied || existing.SHA256 != storedDigest { + return report, errors.New("destination rows differ from committed import progress; resume refused") + } + var exists int + if err := source.QueryRowContext(ctx, `SELECT count(*) FROM sqlite_master WHERE type='table' AND name=?`, table.Name).Scan(&exists); err != nil { + return report, err + } + h := sha256.New() + names, quoted := columnNames(table) + var batch [][]any + var batchBytes int + flush := func(done bool) error { + if len(batch) == 0 && !done { + return nil + } + tx, err := conn.Begin(ctx) + if err != nil { + return err + } + defer tx.Rollback(context.Background()) + if len(batch) > 0 { + if n, err := tx.CopyFrom(ctx, pgx.Identifier{table.Name}, names, pgx.CopyFromRows(batch)); err != nil { + return err + } else if n != int64(len(batch)) { + return errors.New("COPY row count mismatch") + } + } + digest := hex.EncodeToString(h.Sum(nil)) + if _, err := tx.Exec(ctx, `INSERT INTO corescope_import_progress(table_name,rows_copied,sha256,complete) VALUES($1,$2,$3,$4) + ON CONFLICT(table_name) DO UPDATE SET rows_copied=excluded.rows_copied,sha256=excluded.sha256,complete=excluded.complete`, table.Name, report.Rows, digest, done); err != nil { + return err + } + if err := tx.Commit(ctx); err != nil { + return err + } + wrote := len(batch) > 0 + batch = nil + batchBytes = 0 + if wrote && o.afterBatch != nil { + return o.afterBatch() + } + return nil + } + if exists != 0 { + hiddenMailID := false + if o.Kind == "accounts" && table.Name == "mail_events" { + var explicit int + if err := source.QueryRow(`SELECT count(*) FROM pragma_table_info('mail_events') WHERE name='id'`).Scan(&explicit); err != nil { + return report, err + } + hiddenMailID = explicit == 0 + } + order, err := orderBy(owner, table, false, hiddenMailID) + if err != nil { + return report, err + } + expressions := append([]string(nil), quoted...) + if hiddenMailID { + expressions[0] = "rowid" + } + // SQLite's unary + preserves the stored value/type while removing declared + // column metadata. Otherwise the driver parses DATE/DATETIME/TIMESTAMP + // values into time.Time before copy/resume hashing can preserve their text. + for i, expression := range expressions { + expressions[i] = "+" + expression + } + rows, err := source.QueryContext(ctx, `SELECT `+strings.Join(expressions, ",")+` FROM `+quote(table.Name)+` ORDER BY `+order) + if err != nil { + return report, err + } + defer rows.Close() + for rows.Next() { + values := make([]any, len(names)) + dest := make([]any, len(names)) + for i := range dest { + dest[i] = &values[i] + } + if err := rows.Scan(dest...); err != nil { + return report, err + } + rowBytes := 0 + for i, col := range table.Columns { + values[i], err = canonical(values[i], col.Type) + if err != nil { + return report, fmt.Errorf("column %s: %w", col.Name, err) + } + if s, ok := values[i].(string); ok { + rowBytes += len(s) + } else { + rowBytes += 8 + } + } + if rowBytes > maxImportRowBytes { + return report, errors.New("a source row exceeds the 64 MiB migration safety limit") + } + hashRow(h, values) + report.Rows++ + if report.Rows <= copied { + if report.Rows == copied && hex.EncodeToString(h.Sum(nil)) != storedDigest { + return report, errors.New("source prefix differs from import progress") + } + continue + } + if complete { + return report, errors.New("completed source table contains unexpected rows") + } + batch = append(batch, values) + batchBytes += rowBytes + if len(batch) >= o.BatchSize || batchBytes >= 8*1024*1024 { + if err := flush(false); err != nil { + return report, err + } + } + } + if err := rows.Err(); err != nil { + return report, err + } + rows.Close() + } + if report.Rows < copied { + return report, errors.New("source table shorter than committed progress") + } + if !verifyOnly { + if err := flush(true); err != nil { + return report, err + } + } + report.SHA256 = hex.EncodeToString(h.Sum(nil)) + final, err := digestTarget(ctx, owner, table) + if err != nil { + return report, err + } + if final != report { + return report, errors.New("row count or logical digest verification failed") + } + return report, nil +} + +func reseed(ctx context.Context, source *sql.DB, conn *pgx.Conn, tables []importTable, kind string, verifyOnly bool) error { + highWater := map[string]int64{} + var hasSequence int + if err := source.QueryRow(`SELECT count(*) FROM sqlite_master WHERE name='sqlite_sequence'`).Scan(&hasSequence); err != nil { + return err + } + if hasSequence > 0 { + rows, err := source.Query(`SELECT name,MAX(seq),count(*) FROM sqlite_sequence GROUP BY name`) + if err != nil { + return err + } + for rows.Next() { + var name string + var n int64 + var count int + if err := rows.Scan(&name, &n, &count); err != nil { + rows.Close() + return err + } + if count != 1 || n < 0 { + rows.Close() + return errors.New("source SQLite sequence state is invalid or duplicated") + } + highWater[name] = n + } + if err := rows.Err(); err != nil { + rows.Close() + return err + } + rows.Close() + } + references := identityReferences(kind) + for _, table := range tables { + for _, col := range table.Columns { + if !col.Identity { + continue + } + high := highWater[table.Name] + var n int64 + if err := conn.QueryRow(ctx, `SELECT COALESCE(MAX(`+quote(col.Name)+`),0) FROM `+quote(table.Name)).Scan(&n); err != nil { + return err + } + if n > high { + high = n + } + for _, ref := range references[table.Name] { + if err := conn.QueryRow(ctx, `SELECT COALESCE(MAX(`+quote(ref[1])+`),0) FROM `+quote(ref[0])).Scan(&n); err != nil { + return err + } + if n > high { + high = n + } + } + called := high > 0 + if !called { + high = 1 + } + if verifyOnly { + var sequence string + if err := conn.QueryRow(ctx, `SELECT pg_get_serial_sequence($1,$2)`, table.Name, col.Name).Scan(&sequence); err != nil { + return err + } + var actual int64 + var wasCalled bool + // pg_get_serial_sequence returns a safely quoted qualified identifier. + if err := conn.QueryRow(ctx, `SELECT last_value,is_called FROM `+sequence).Scan(&actual, &wasCalled); err != nil { + return err + } + if actual != high || wasCalled != called { + return fmt.Errorf("ready destination sequence for %s changed since import; resume refused", table.Name) + } + } else if _, err := conn.Exec(ctx, `SELECT setval(pg_get_serial_sequence($1,$2),$3,$4)`, table.Name, col.Name, high, called); err != nil { + return err + } + } + } + return nil +} + +// finalizeImport reports whether this call committed a closed-to-ready transition. +// The caller may roll back only transitions it owns if another store fails. +func finalizeImport(ctx context.Context, databaseURL, kind string) (bool, error) { + tables, err := layouts(kind) + if err != nil { + return false, err + } + db, err := pgutil.Open(databaseURL, false) + if err != nil { + return false, err + } + defer db.Close() + config, err := pgutil.ParseConfig(databaseURL) + if err != nil { + return false, err + } + conn, err := pgx.ConnectConfig(ctx, config) + if err != nil { + return false, errors.New("connect to PostgreSQL finalization destination failed") + } + defer conn.Close(context.Background()) + if err := lockImport(ctx, conn); err != nil { + return false, err + } + var verified bool + var recordedKind, reportText string + if err := conn.QueryRow(ctx, `SELECT kind,verified,report FROM corescope_import WHERE id=1`).Scan(&recordedKind, &verified, &reportText); err != nil { + return false, err + } + if !verified || recordedKind != kind { + return false, errors.New("cannot finalize an unverified or mismatched import") + } + var report importReport + if err := json.Unmarshal([]byte(reportText), &report); err != nil { + return false, err + } + if !report.Verified || report.Kind != kind || len(report.Tables) != len(tables) { + return false, errors.New("incomplete verification manifest") + } + tx, err := conn.Begin(ctx) + if err != nil { + return false, err + } + defer tx.Rollback(context.Background()) + var lockTables []string + for _, table := range tables { + lockTables = append(lockTables, quote(table.Name)) + } + if _, err := tx.Exec(ctx, `LOCK TABLE `+strings.Join(lockTables, ",")+` IN SHARE MODE`); err != nil { + return false, err + } + for i, table := range tables { + got, err := digestTarget(ctx, db, table) + if err != nil { + return false, err + } + if got != report.Tables[i] { + return false, fmt.Errorf("final verification failed for %s; target remains unready", table.Name) + } + } + var ready bool + if err := tx.QueryRow(ctx, `SELECT ready FROM corescope_schema WHERE kind=$1 AND version=1`, kind).Scan(&ready); err != nil { + return false, errors.New("missing or unsupported readiness marker") + } + if ready { + return false, nil + } + result, err := tx.Exec(ctx, `UPDATE corescope_schema SET ready=true WHERE kind=$1 AND version=1`, kind) + if err != nil { + return false, err + } + if result.RowsAffected() != 1 { + return false, errors.New("missing or unsupported readiness marker") + } + if err := tx.Commit(ctx); err != nil { + return false, err + } + return true, nil +} + +func lockImport(ctx context.Context, conn *pgx.Conn) error { + var locked bool + if err := conn.QueryRow(ctx, `SELECT pg_try_advisory_lock(hashtextextended(current_schema(),$1))`, importerLock).Scan(&locked); err != nil { + return err + } + if !locked { + return errors.New("another importer owns this destination; wait for it to finish before retrying") + } + return nil +} diff --git a/cmd/migrate/import_safety_test.go b/cmd/migrate/import_safety_test.go new file mode 100644 index 000000000..7203fbbbb --- /dev/null +++ b/cmd/migrate/import_safety_test.go @@ -0,0 +1,293 @@ +package main + +import ( + "context" + "database/sql" + "errors" + "fmt" + "path/filepath" + "strings" + "testing" + + "github.com/meshcore-analyzer/dbschema" +) + +func TestResumeRefusesDifferentDestination(t *testing.T) { + o := importOptions{Source: telemetrySource(t), DatabaseURL: postgresSchema(t), StateDir: t.TempDir(), Kind: "telemetry", BatchSize: 1, afterBatch: func() error { return context.Canceled }} + if _, err := importSQLite(context.Background(), o); !errors.Is(err, context.Canceled) { + t.Fatal(err) + } + o.Resume = true + o.afterBatch = nil + o.DatabaseURL = postgresSchema(t) + if _, err := importSQLite(context.Background(), o); err == nil { + t.Fatal("resume accepted a different PostgreSQL destination") + } +} + +func TestResumeAndFinalizationRejectTargetCorruption(t *testing.T) { + for _, partial := range []bool{false, true} { + t.Run(map[bool]string{false: "finalization", true: "resume"}[partial], func(t *testing.T) { + o := importOptions{Source: telemetrySource(t), DatabaseURL: postgresSchema(t), StateDir: t.TempDir(), Kind: "telemetry", BatchSize: 1} + if partial { + o.afterBatch = func() error { return context.Canceled } + } + _, err := importSQLite(context.Background(), o) + if partial { + if !errors.Is(err, context.Canceled) { + t.Fatal(err) + } + } else if err != nil { + t.Fatal(err) + } + db := openImportDB(t, o.DatabaseURL) + if _, err := db.Exec(`UPDATE observers SET name='tampered'`); err != nil { + t.Fatal(err) + } + if partial { + o.Resume = true + o.afterBatch = nil + _, err = importSQLite(context.Background(), o) + } else { + _, err = finalizeImport(context.Background(), o.DatabaseURL, o.Kind) + } + if err == nil { + t.Fatal("corrupted target was accepted") + } + if err := dbschema.AssertReady(db); err == nil { + t.Fatal("corrupted target became runtime-ready") + } + }) + } +} + +func TestImportPreservesRawCaseJSONAndFractionalScore(t *testing.T) { + source := telemetrySource(t) + db, err := sql.Open("sqlite3", source) + if err != nil { + t.Fatal(err) + } + for _, stmt := range []string{`INSERT INTO nodes(public_key,name,advert_count) VALUES('AbCd','MiXeD',123)`, `UPDATE observations SET path_json='{malformed',score=0.75 WHERE id=90`, `UPDATE transmissions SET channel_hash='public',payload_type=5 WHERE id=80`} { + if _, err := db.Exec(stmt); err != nil { + db.Close() + t.Fatal(err) + } + } + db.Close() + dsn := postgresSchema(t) + if _, err := importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: dsn, StateDir: t.TempDir(), Kind: "telemetry"}); err != nil { + t.Fatal(err) + } + target := openImportDB(t, dsn) + var key, jsonText, channel string + var count int + var score float64 + if err := target.QueryRow(`SELECT public_key,advert_count FROM nodes`).Scan(&key, &count); err != nil || key != "AbCd" || count != 123 { + t.Fatal("raw key/counter changed", err) + } + if err := target.QueryRow(`SELECT path_json,score FROM observations WHERE id=90`).Scan(&jsonText, &score); err != nil || jsonText != "{malformed" || score != 0.75 { + t.Fatal("raw JSON or fractional score changed", err) + } + if err := target.QueryRow(`SELECT channel_hash FROM transmissions WHERE id=80`).Scan(&channel); err != nil || channel != "public" { + t.Fatal("raw channel hash changed", err) + } +} + +func TestImportRejectsUnexpectedViewsTriggersAndFutureAccounts(t *testing.T) { + for _, statement := range []string{`CREATE VIEW unexpected AS SELECT * FROM users`, `CREATE TRIGGER surprise AFTER INSERT ON users BEGIN DELETE FROM tokens; END`, `UPDATE schema_version SET version=999`} { + source := accountSource(t, 5) + db, err := sql.Open("sqlite3", source) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(statement); err != nil { + t.Fatal(err) + } + db.Close() + dsn := postgresSchema(t) + _, err = importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: dsn, StateDir: t.TempDir(), Kind: "accounts"}) + if err == nil { + t.Fatal("unsupported source accepted") + } + var n int + if err := openImportDB(t, dsn).QueryRow(`SELECT count(*) FROM information_schema.tables WHERE table_schema=current_schema()`).Scan(&n); err != nil || n != 0 { + t.Fatal("unsupported source changed target", err) + } + } +} + +func TestAccountsReseedDeletedHistoricalReferences(t *testing.T) { + source := accountSource(t, 5) + db, err := sql.Open("sqlite3", source) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`INSERT INTO audit_log(at,actor_user_id,target_user_id,action) VALUES(1,700000,900000,'deleted'); UPDATE users SET activated_by=1000000`); err != nil { + t.Fatal(err) + } + db.Close() + dsn := postgresSchema(t) + if _, err := importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: dsn, StateDir: t.TempDir(), Kind: "accounts"}); err != nil { + t.Fatal(err) + } + var id int64 + if err := openImportDB(t, dsn).QueryRow(`INSERT INTO users(email,display_name,password_hash,created_at) VALUES('next@example.invalid','next','hash',1) RETURNING id`).Scan(&id); err != nil || id <= 1000000 { + t.Fatalf("historical account identity reused: %d,%v", id, err) + } +} + +func TestSourceFingerprintCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := sourceFingerprintContext(ctx, telemetrySource(t)) + if !errors.Is(err, context.Canceled) { + t.Fatalf("canceled source hashing: %v", err) + } +} + +func TestImportRefusesLegacyV2WithUpgradeGuidance(t *testing.T) { + source := telemetrySource(t) + db, err := sql.Open("sqlite3", source) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`ALTER TABLE observations RENAME COLUMN observer_idx TO observer_id`); err != nil { + t.Fatal(err) + } + db.Close() + _, err = importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: postgresSchema(t), StateDir: t.TempDir(), Kind: "telemetry"}) + if err == nil || !strings.Contains(err.Error(), "upgrade") { + t.Fatal("v2 refusal lacks explicit upgrade guidance") + } +} + +func TestAccountVersionMatrixAndMissingDeclaredTable(t *testing.T) { + for version := 0; version <= 5; version++ { + t.Run(fmt.Sprint(version), func(t *testing.T) { + var source string + if version > 0 { + source = accountSource(t, version) + } else { + source = filepath.Join(t.TempDir(), "empty.db") + db, err := sql.Open("sqlite3", source) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`CREATE TABLE schema_version(version INTEGER NOT NULL);INSERT INTO schema_version VALUES(0)`); err != nil { + t.Fatal(err) + } + db.Close() + } + report, err := importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: postgresSchema(t), StateDir: t.TempDir(), Kind: "accounts"}) + if err != nil || !report.Verified { + t.Fatalf("account version %d: %v", version, err) + } + }) + } + t.Run("missing-table", func(t *testing.T) { + source := accountSource(t, 5) + db, err := sql.Open("sqlite3", source) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`DROP TABLE mail_events`); err != nil { + t.Fatal(err) + } + db.Close() + if _, err := importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: postgresSchema(t), StateDir: t.TempDir(), Kind: "accounts"}); err == nil { + t.Fatal("incomplete declared account schema was accepted") + } + }) +} + +func TestNormalizeDoesNotRewriteObservationRows(t *testing.T) { + raw := telemetrySource(t) + db, err := sql.Open("sqlite3", raw) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<1000) + INSERT INTO observations(id,transmission_id,observer_idx,timestamp) SELECT x+1000,80,NULL,x FROM n`); err != nil { + t.Fatal(err) + } + db.Close() + working := filepath.Join(t.TempDir(), "working.sqlite") + if err := snapshotSource(context.Background(), raw, working); err != nil { + t.Fatal(err) + } + db, err = openSQLite(working, "rw") + if err != nil { + t.Fatal(err) + } + defer db.Close() + db.SetMaxOpenConns(1) + tables, err := layouts("telemetry") + if err != nil { + t.Fatal(err) + } + var before, after int64 + if err := db.QueryRow(`SELECT total_changes()`).Scan(&before); err != nil { + t.Fatal(err) + } + if err := normalizeSource(db, "telemetry", raw, tables); err != nil { + t.Fatal(err) + } + if err := db.QueryRow(`SELECT total_changes()`).Scan(&after); err != nil { + t.Fatal(err) + } + if after-before >= 50 { + t.Fatalf("normalization rewrote unchanged bulk rows: %d changes for 1003 observations", after-before) + } + t.Logf("normalization wrote %d rows for 1003 unchanged observations", after-before) +} + +func TestImportPopulatesPlannerStatsBeforeReadiness(t *testing.T) { + dsn := postgresSchema(t) + if _, err := importSQLite(context.Background(), importOptions{Source: telemetrySource(t), DatabaseURL: dsn, StateDir: t.TempDir(), Kind: "telemetry"}); err != nil { + t.Fatal(err) + } + db := openImportDB(t, dsn) + var estimate float64 + if err := db.QueryRow(`SELECT reltuples FROM pg_class WHERE oid=to_regclass('observations')`).Scan(&estimate); err != nil { + t.Fatal(err) + } + if estimate != 3 { + t.Fatalf("planner estimate before readiness=%v; want 3 imported observations", estimate) + } + if err := dbschema.AssertReady(db); err == nil { + t.Fatal("statistics preparation opened readiness before finalization") + } +} + +func TestReservedNameDoesNotHideSourceTrigger(t *testing.T) { + source := accountSource(t, 5) + db, err := sql.Open("sqlite3", source) + if err != nil { + t.Fatal(err) + } + for _, stmt := range []string{ + `CREATE TRIGGER hidden_source AFTER INSERT ON users BEGIN DELETE FROM tokens; END`, + `PRAGMA writable_schema=ON`, + `UPDATE sqlite_master SET name='sqlite_hidden_source',sql=replace(sql,'hidden_source','sqlite_hidden_source') WHERE type='trigger' AND name='hidden_source'`, + `PRAGMA writable_schema=OFF`, + } { + if _, err := db.Exec(stmt); err != nil { + db.Close() + t.Fatal(err) + } + } + db.Close() + db, err = openSQLite(source, "ro") + if err != nil { + t.Fatal(err) + } + var count int + if err := db.QueryRow(`SELECT count(*) FROM sqlite_master WHERE type='trigger' AND name='sqlite_hidden_source'`).Scan(&count); err != nil || count != 1 { + db.Close() + t.Fatalf("tampered schema fixture: count=%d err=%v", count, err) + } + db.Close() + if _, err := importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: postgresSchema(t), StateDir: t.TempDir(), Kind: "accounts"}); err == nil { + t.Fatal("reserved-name trigger bypassed source validation") + } +} diff --git a/cmd/migrate/import_test.go b/cmd/migrate/import_test.go new file mode 100644 index 000000000..01b6f9bad --- /dev/null +++ b/cmd/migrate/import_test.go @@ -0,0 +1,255 @@ +package main + +import ( + "context" + "database/sql" + "errors" + "path/filepath" + "testing" + + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/dbschema/legacy" + "github.com/meshcore-analyzer/pgutil" + "github.com/meshcore-analyzer/users" +) + +func telemetrySource(t *testing.T) string { + t.Helper() + path := filepath.Join(t.TempDir(), "source.db") + db, err := sql.Open("sqlite3", path) + if err != nil { + t.Fatal(err) + } + defer db.Close() + if err := legacy.ApplyBase(db); err != nil { + t.Fatal(err) + } + if err := legacy.Apply(db, t.Logf); err != nil { + t.Fatal(err) + } + for _, query := range []string{ + `INSERT INTO observers(rowid,id,name) VALUES(0,'observer-zero',NULL),(50,'observer-gap','')`, + `INSERT INTO transmissions(id,raw_hex,hash,first_seen,last_seen) VALUES(0,'00','zero','2026-01-01 00:00:00',0),(80,'00','gap','2026-01-02T00:00:00Z',1),(99999,'00','deleted','2026-01-01',0)`, + `DELETE FROM transmissions WHERE id=99999`, + `INSERT INTO observations(id,transmission_id,observer_idx,path_json,timestamp) VALUES(0,0,0,NULL,0),(90,80,999,'',1),(91,80,NULL,'[]',2)`, + } { + if _, err := db.Exec(query); err != nil { + t.Fatal(err) + } + } + return path +} + +func openImportDB(t *testing.T, dsn string) *sql.DB { + t.Helper() + db, err := pgutil.Open(dsn, false) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { db.Close() }) + return db +} + +func TestImportTelemetryPreservesIDsNullsAndHighWater(t *testing.T) { + source := telemetrySource(t) + before, err := sourceFingerprint(source) + if err != nil { + t.Fatal(err) + } + dsn := postgresSchema(t) + report, err := importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: dsn, StateDir: t.TempDir(), Kind: "telemetry", BatchSize: 2}) + if err != nil { + t.Fatalf("import valid telemetry: %v", err) + } + if !report.Verified || len(report.Tables) != len(dbschema.Tables) { + t.Fatalf("missing verification: %+v", report) + } + db := openImportDB(t, dsn) + if err := dbschema.AssertReady(db); err == nil { + t.Fatal("import accepted before explicit cutover finalization") + } + if _, err := finalizeImport(context.Background(), dsn, "telemetry"); err != nil { + t.Fatal(err) + } + if err := dbschema.AssertReady(db); err != nil { + t.Fatal(err) + } + var n int + if err := db.QueryRow(`SELECT count(*) FROM observations WHERE (id=0 AND observer_idx=0 AND path_json IS NULL) OR (id=90 AND observer_idx=999 AND path_json='') OR (id=91 AND observer_idx IS NULL)`).Scan(&n); err != nil { + t.Fatal(err) + } + if n != 3 { + t.Fatalf("lost observer identity/null/orphan links: %d", n) + } + var id int64 + if err := db.QueryRow(`INSERT INTO observers(id) VALUES('next-observer') RETURNING rowid`).Scan(&id); err != nil { + t.Fatal(err) + } + if id <= 999 { + t.Fatalf("observer identity reused historical reference: %d", id) + } + if err := db.QueryRow(`INSERT INTO transmissions(raw_hex,hash,first_seen) VALUES('00','next','2026-01-03') RETURNING id`).Scan(&id); err != nil { + t.Fatal(err) + } + if id <= 99999 { + t.Fatalf("transmission identity ignored deleted high water: %d", id) + } + after, err := sourceFingerprint(source) + if err != nil { + t.Fatal(err) + } + if before != after { + t.Fatal("migration changed original SQLite source") + } +} + +func TestImportInterruptResumeAndWrongSource(t *testing.T) { + source := telemetrySource(t) + dsn, state := postgresSchema(t), t.TempDir() + options := importOptions{Source: source, DatabaseURL: dsn, StateDir: state, Kind: "telemetry", BatchSize: 1, afterBatch: func() error { return context.Canceled }} + if _, err := importSQLite(context.Background(), options); !errors.Is(err, context.Canceled) { + t.Fatalf("expected cancellation after committed batch, got %v", err) + } + db := openImportDB(t, dsn) + if err := dbschema.AssertReady(db); err == nil { + t.Fatal("partially imported database accepted") + } + options.afterBatch = nil + if _, err := importSQLite(context.Background(), options); err == nil { + t.Fatal("partial import resumed without explicit resume") + } + options.Resume = true + options.Source = telemetrySource(t) + changed, err := sql.Open("sqlite3", options.Source) + if err != nil { + t.Fatal(err) + } + if _, err := changed.Exec(`UPDATE transmissions SET raw_hex='ff' WHERE id=0`); err != nil { + t.Fatal(err) + } + changed.Close() + if _, err := importSQLite(context.Background(), options); err == nil { + t.Fatal("resumed with a different source") + } + options.Source = source + report, err := importSQLite(context.Background(), options) + if err != nil { + t.Fatal(err) + } + if !report.Verified { + t.Fatal("resumed import unverified") + } + var n int + if err := db.QueryRow(`SELECT COUNT(*) FROM observations`).Scan(&n); err != nil { + t.Fatal(err) + } + if n != 3 { + t.Fatalf("resume duplicated/lost observations: %d", n) + } + if _, err := importSQLite(context.Background(), options); err != nil { + t.Fatalf("verified retry must be idempotent: %v", err) + } +} + +func TestImportRefusesUnknownSourceAndOccupiedTarget(t *testing.T) { + for _, unknown := range []bool{false, true} { + source, dsn := telemetrySource(t), postgresSchema(t) + if unknown { + db, err := sql.Open("sqlite3", source) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`ALTER TABLE transmissions ADD COLUMN future_secret TEXT`); err != nil { + t.Fatal(err) + } + db.Close() + } else { + db := openImportDB(t, dsn) + if _, err := db.Exec(`CREATE TABLE keep_me(value TEXT); INSERT INTO keep_me VALUES('retained')`); err != nil { + t.Fatal(err) + } + } + if _, err := importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: dsn, StateDir: t.TempDir(), Kind: "telemetry"}); err == nil { + t.Fatal("unsupported source/occupied destination accepted") + } + if !unknown { + var value string + if err := openImportDB(t, dsn).QueryRow(`SELECT value FROM keep_me`).Scan(&value); err != nil { + t.Fatal(err) + } + if value != "retained" { + t.Fatal("existing target changed") + } + } + } +} + +func accountSource(t *testing.T, version int) string { + t.Helper() + path := filepath.Join(t.TempDir(), "users.db") + db, err := sql.Open("sqlite3", path) + if err != nil { + t.Fatal(err) + } + defer db.Close() + if _, err := db.Exec(`CREATE TABLE schema_version(version INTEGER NOT NULL)`); err != nil { + t.Fatal(err) + } + for _, migration := range users.LegacyMigrations()[:version] { + for _, stmt := range migration { + if _, err := db.Exec(stmt); err != nil { + t.Fatal(err) + } + } + } + if _, err := db.Exec(`INSERT INTO schema_version VALUES(?)`, version); err != nil { + t.Fatal(err) + } + for _, stmt := range []string{ + `INSERT INTO users(id,email,display_name,password_hash,role,status,created_at) VALUES(0,'user@example.invalid','test','preserved-hash','user','active',1)`, + `INSERT INTO sessions(id,token_hash,user_id,csrf_token,created_at,expires_at,last_seen_at) VALUES(90,'preserved-token',0,'preserved-csrf',1,9999999999,2)`, + `INSERT INTO tokens(token_hash,user_id,purpose,expires_at,used_at) VALUES('one-use',0,'reset',9999999999,NULL)`, + `INSERT INTO mail_log(id,to_email,purpose,sent_at,last_event_at) VALUES(4,'user@example.invalid','activate',1,1)`, + `INSERT INTO mail_events(rowid,mail_id,event,at) VALUES(900,4,'delivered',1)`, + } { + if _, err := db.Exec(stmt); err != nil { + t.Fatal(err) + } + } + return path +} + +func TestImportLegacyAccountsPreservesAuthenticationAndEventOrder(t *testing.T) { + source, dsn := accountSource(t, 1), postgresSchema(t) + report, err := importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: dsn, StateDir: t.TempDir(), Kind: "accounts", BatchSize: 1}) + if err != nil { + t.Fatal(err) + } + if !report.Verified { + t.Fatal("accounts not verified") + } + db := openImportDB(t, dsn) + if err := users.AssertReady(db); err == nil { + t.Fatal("accounts accepted before finalization") + } + if _, err := finalizeImport(context.Background(), dsn, "accounts"); err != nil { + t.Fatal(err) + } + if err := users.AssertReady(db); err != nil { + t.Fatal(err) + } + var value string + if err := db.QueryRow(`SELECT password_hash FROM users WHERE id=0`).Scan(&value); err != nil || value != "preserved-hash" { + t.Fatalf("password hash changed: %v", err) + } + if err := db.QueryRow(`SELECT token_hash FROM sessions WHERE id=90 AND user_id=0`).Scan(&value); err != nil || value != "preserved-token" { + t.Fatalf("session changed: %v", err) + } + var n int + if err := db.QueryRow(`SELECT COUNT(*) FROM tokens WHERE token_hash='one-use' AND used_at IS NULL`).Scan(&n); err != nil || n != 1 { + t.Fatalf("one-time token changed: %v", err) + } + if err := db.QueryRow(`SELECT id FROM mail_events WHERE mail_id=4`).Scan(&n); err != nil || n != 900 { + t.Fatalf("mail event identity changed: %d %v", n, err) + } +} diff --git a/cmd/migrate/layout.go b/cmd/migrate/layout.go new file mode 100644 index 000000000..74c8d1c49 --- /dev/null +++ b/cmd/migrate/layout.go @@ -0,0 +1,343 @@ +package main + +import ( + "database/sql" + "errors" + "fmt" + "strings" + + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/dbschema/legacy" + "github.com/meshcore-analyzer/users" +) + +type importColumn struct { + Name string + Type string + Identity bool +} +type importTable struct { + Name string + Columns []importColumn +} + +func quote(name string) string { return `"` + strings.ReplaceAll(name, `"`, `""`) + `"` } + +func layouts(kind string) ([]importTable, error) { + var out []importTable + if kind == "telemetry" { + for _, table := range dbschema.Tables { + t := importTable{Name: table.Name} + for _, col := range table.Columns { + t.Columns = append(t.Columns, importColumn{col.Name, col.Type, col.Identity}) + } + out = append(out, t) + } + return out, nil + } + if kind != "accounts" { + return nil, errors.New("import kind must be telemetry or accounts") + } + // SQLite is used only by this offline importer. Derive the legacy account + // layout from the immutable migration metadata instead of duplicating it. + db, err := sql.Open("sqlite3", ":memory:") + if err != nil { + return nil, err + } + defer db.Close() + db.SetMaxOpenConns(1) + for _, migration := range users.LegacyMigrations() { + for _, stmt := range migration { + if _, err := db.Exec(stmt); err != nil { + return nil, err + } + } + } + identities := map[string]bool{"users": true, "sessions": true, "audit_log": true, "mail_log": true, "proposals": true, "mail_events": true} + for _, name := range users.Tables() { + t := importTable{Name: name} + if name == "mail_events" { + t.Columns = append(t.Columns, importColumn{"id", "BIGINT", true}) + } + cols, err := sqliteColumns(db, name) + if err != nil { + return nil, err + } + for _, col := range cols { + kind := "TEXT" + if strings.Contains(strings.ToUpper(col.Type), "INT") { + kind = "BIGINT" + } + t.Columns = append(t.Columns, importColumn{col.Name, kind, col.Name == "id" && identities[name]}) + } + out = append(out, t) + } + return out, nil +} + +func sqliteColumns(db *sql.DB, table string) ([]importColumn, error) { + // table_info omits generated and hidden columns, which must be refused even + // when their names match an ordinary column in a supported legacy layout. + rows, err := db.Query(`PRAGMA table_xinfo(` + quote(table) + `)`) + if err != nil { + return nil, err + } + defer rows.Close() + var out []importColumn + for rows.Next() { + var cid, notnull, pk, hidden int + var name, kind string + var def sql.NullString + if err := rows.Scan(&cid, &name, &kind, ¬null, &def, &pk, &hidden); err != nil { + return nil, err + } + if hidden != 0 { + return nil, fmt.Errorf("unsupported generated or hidden SQLite column %s.%s", table, name) + } + out = append(out, importColumn{Name: name, Type: kind}) + } + return out, rows.Err() +} + +func validateSource(db *sql.DB, kind string, tables []importTable) error { + var accountVersion int + if kind == "accounts" { + var n int + if err := db.QueryRow(`SELECT COUNT(*),COALESCE(MAX(version),0) FROM schema_version`).Scan(&n, &accountVersion); err != nil { + return errors.New("account source must contain its supported schema_version") + } + if n != 1 || accountVersion < 0 || accountVersion > users.SQLiteSchemaVersion { + return fmt.Errorf("unsupported account schema version %d (supported 0 through %d)", accountVersion, users.SQLiteSchemaVersion) + } + } + allowed := map[string]map[string]bool{} + for _, table := range tables { + cols := map[string]bool{} + for _, col := range table.Columns { + cols[col.Name] = true + } + if kind == "accounts" && table.Name == "mail_events" && accountVersion < users.SQLiteSchemaVersion { + delete(cols, "id") + } + allowed[table.Name] = cols + } + if kind == "accounts" { + allowed["schema_version"] = map[string]bool{"version": true} + } + rows, err := db.Query(`SELECT name,type,COALESCE(sql,'') FROM sqlite_master WHERE type IN ('table','view','trigger')`) + if err != nil { + return err + } + type object struct{ name, kind, sql string } + internalTables := map[string]bool{"sqlite_sequence": true, "sqlite_stat1": true, "sqlite_stat2": true, "sqlite_stat3": true, "sqlite_stat4": true} + var objects []object + for rows.Next() { + var o object + if err := rows.Scan(&o.name, &o.kind, &o.sql); err != nil { + rows.Close() + return err + } + ordinary := strings.HasPrefix(strings.ToUpper(strings.Join(strings.Fields(o.sql), " ")), "CREATE TABLE ") + if o.kind == "table" && internalTables[o.name] && ordinary { + continue + } + objects = append(objects, o) + if len(objects) > len(allowed)+1 { + rows.Close() + return errors.New("source has unexpected schema objects") + } + } + if err := rows.Err(); err != nil { + rows.Close() + return err + } + rows.Close() + actual := map[string]map[string]bool{} + for _, o := range objects { + if o.kind == "table" && !strings.HasPrefix(strings.ToUpper(strings.Join(strings.Fields(o.sql), " ")), "CREATE TABLE ") { + return fmt.Errorf("unsupported nonordinary source table %q", o.name) + } + if o.kind == "view" && kind == "telemetry" && o.name == "packets_v" { + continue + } + cols, known := allowed[o.name] + if o.kind != "table" || !known { + return fmt.Errorf("unsupported SQLite %s %q; no source data was discarded", o.kind, o.name) + } + have, err := sqliteColumns(db, o.name) + if err != nil { + return err + } + actual[o.name] = map[string]bool{} + for _, col := range have { + actual[o.name][col.Name] = true + if !cols[col.Name] { + if kind == "telemetry" && o.name == "observations" && col.Name == "observer_id" { + return errors.New("legacy v2 observer_id layout requires an offline upgrade with the last SQLite release before importing") + } + return fmt.Errorf("unsupported SQLite column %s.%s", o.name, col.Name) + } + } + } + if kind == "telemetry" { + return legacy.CheckLegacySource(db) + } + if accountVersion == users.SQLiteSchemaVersion { + return users.AssertSQLiteReady(db) + } + version := accountVersion + // A declared version must contain its complete historical shape. Otherwise + // treating an absent table as empty could conceal a damaged account source. + expected, err := sql.Open("sqlite3", ":memory:") + if err != nil { + return err + } + defer expected.Close() + expected.SetMaxOpenConns(1) + for _, migration := range users.LegacyMigrations()[:version] { + for _, stmt := range migration { + if _, err := expected.Exec(stmt); err != nil { + return err + } + } + } + for _, table := range tables { + columns, err := sqliteColumns(expected, table.Name) + if err != nil { + return err + } + if len(columns) == 0 { + if _, exists := actual[table.Name]; exists { + return fmt.Errorf("account table %s is newer than its declared schema version", table.Name) + } + continue + } + for _, col := range columns { + if !actual[table.Name][col.Name] { + return fmt.Errorf("account schema version %d is missing %s.%s", version, table.Name, col.Name) + } + } + } + var integrity string + if err := db.QueryRow(`PRAGMA quick_check`).Scan(&integrity); err != nil { + return err + } + if integrity != "ok" { + return errors.New("account source failed SQLite integrity check") + } + return nil +} + +func normalizeSource(db *sql.DB, kind, rawPath string, tables []importTable) error { + if err := validateSource(db, kind, tables); err != nil { + return err + } + if kind == "accounts" { + var version int + if err := db.QueryRow(`SELECT version FROM schema_version`).Scan(&version); err != nil { + return err + } + if version == users.SQLiteSchemaVersion { + return nil + } + for i, migration := range users.LegacyMigrations()[version:] { + tx, err := db.Begin() + if err != nil { + return err + } + for _, stmt := range migration { + if _, err := tx.Exec(stmt); err != nil { + tx.Rollback() + return err + } + } + if _, err := tx.Exec(`UPDATE schema_version SET version=?`, version+i+1); err != nil { + tx.Rollback() + return err + } + if err := tx.Commit(); err != nil { + return err + } + } + return nil + } + if err := legacy.Normalize(db, nil); err != nil { + return err + } + // Mutation inventory of legacy.Normalize: only these columns can change + // existing rows. Duplicate/invalid-row deletion paths were refused above. + // Restore changed tuples by rowid, without rewriting the observation corpus. + recoveryURI, err := sqliteURL(rawPath, "ro") + if err != nil { + return err + } + if _, err := db.Exec(`ATTACH DATABASE ? AS recovery`, recoveryURI); err != nil { + return err + } + defer db.Exec(`DETACH DATABASE recovery`) + mutated := map[string]map[string]bool{ + "nodes": {"public_key": true, "advert_count": true}, + "observers": {"noise_floor": true, "last_packet_at": true}, + "transmissions": {"channel_hash": true}, + } + for _, table := range tables { + selected, changes := mutated[table.Name] + if !changes { + continue + } + var exists int + if err := db.QueryRow(`SELECT count(*) FROM recovery.sqlite_master WHERE type='table' AND name=?`, table.Name).Scan(&exists); err != nil { + return err + } + if exists == 0 { + continue + } + rows, err := db.Query(`PRAGMA recovery.table_info(` + quote(table.Name) + `)`) + if err != nil { + return err + } + var columns []string + for rows.Next() { + var cid, nn, pk int + var name, kind string + var def sql.NullString + if err := rows.Scan(&cid, &name, &kind, &nn, &def, &pk); err != nil { + rows.Close() + return err + } + if selected[name] { + columns = append(columns, name) + } + } + if err := rows.Err(); err != nil { + rows.Close() + return err + } + rows.Close() + var sourceCount, targetCount int64 + if err := db.QueryRow(`SELECT count(*) FROM recovery.` + quote(table.Name)).Scan(&sourceCount); err != nil { + return err + } + if err := db.QueryRow(`SELECT count(*) FROM main.` + quote(table.Name)).Scan(&targetCount); err != nil { + return err + } + if sourceCount != targetCount { + return fmt.Errorf("normalization changed row count in %s; source requires an explicit repair", table.Name) + } + if len(columns) == 0 { + continue + } + var targets, originals, differences []string + for _, name := range columns { + targets = append(targets, quote(name)) + originals = append(originals, "old."+quote(name)) + differences = append(differences, "current."+quote(name)+" IS NOT old."+quote(name)) + } + restore := `UPDATE main.` + quote(table.Name) + ` AS current SET (` + strings.Join(targets, ",") + `)=(SELECT ` + strings.Join(originals, ",") + ` FROM recovery.` + quote(table.Name) + ` old WHERE old.rowid=current.rowid) + WHERE EXISTS(SELECT 1 FROM recovery.` + quote(table.Name) + ` old WHERE old.rowid=current.rowid AND (` + strings.Join(differences, " OR ") + `))` + if _, err := db.Exec(restore); err != nil { + return fmt.Errorf("preserve original %s values: %w", table.Name, err) + } + } + return nil +} diff --git a/cmd/migrate/main.go b/cmd/migrate/main.go index c10f75a10..73db2f73e 100644 --- a/cmd/migrate/main.go +++ b/cmd/migrate/main.go @@ -1,55 +1,257 @@ -// Command migrate runs all dbschema migrations against a SQLite -// CoreScope database and exits. Used by CI / one-shot tooling to bring -// an unmigrated fixture (or a fresh DB) up to the schema shape the -// read-only server (cmd/server) requires via dbschema.AssertReady. -// -// In production the ingestor (cmd/ingestor) runs dbschema.Apply at -// startup before subscribing to MQTT — this binary exists so CI's E2E -// job can migrate the e2e-fixture.db without booting the full ingestor -// (which needs MQTT brokers). -// -// Usage: -// -// migrate -db path/to/file.db +// Command migrate bootstraps PostgreSQL or imports immutable offline SQLite +// recovery snapshots. Runtime services never apply DDL themselves. package main import ( - "database/sql" + "context" + "encoding/json" + "errors" "flag" - "log" + "fmt" + "io" + "os" + "os/signal" + "syscall" "github.com/meshcore-analyzer/dbschema" - _ "modernc.org/sqlite" + "github.com/meshcore-analyzer/pgutil" + "github.com/meshcore-analyzer/users" ) func main() { - dbPath := flag.String("db", "", "path to SQLite database to migrate (required)") - flag.Parse() - - if *dbPath == "" { - log.Fatalf("[migrate] -db is required") + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer cancel() + if err := runCommand(ctx, os.Args[1:], os.Stdout); err != nil { + fmt.Fprintln(os.Stderr, "[migrate]", err) + os.Exit(1) } +} - log.SetFlags(log.LstdFlags | log.Lmsgprefix) - log.SetPrefix("[migrate] ") +func runCommand(ctx context.Context, args []string, out io.Writer) error { + flags := flag.NewFlagSet("migrate", flag.ContinueOnError) + flags.SetOutput(io.Discard) + databaseURL := flags.String("database-url", "", "PostgreSQL telemetry owner URL (prefer CORESCOPE_DATABASE_URL)") + usersURL := flags.String("users-database-url", "", "PostgreSQL accounts owner URL (prefer CORESCOPE_USERS_DATABASE_URL)") + source := flags.String("from-sqlite", "", "offline telemetry SQLite source") + accountSource := flags.String("users-from-sqlite", "", "offline account SQLite source") + stateDir := flags.String("state-dir", "", "private migration snapshots and resume manifests (storage actions default beside selection; legacy imports use state/migration)") + offline := flags.Bool("offline", false, "confirm telemetry and account writers are stopped") + resume := flags.Bool("resume", false, "resume the same source/state/destination import") + checkReady := flags.Bool("check-ready", false, "check readiness without changing any schema or data") + checkImportKind := flags.String("check-import-kind", "", "read-only completed-import guard: telemetry or accounts, with its SQLite source path") + action := flags.String("storage-action", "", "installation status, setup, adopt, init, switch, resume or abort") + selectionFile := flags.String("selection-file", "", "absolute stable storage-selection.json path") + backend := flags.String("backend", "", "bootstrap backend, or explicit switch destination: sqlite or postgres") + sqlitePath := flags.String("sqlite-path", "", "bootstrap or new destination telemetry SQLite path") + usersSQLitePath := flags.String("users-sqlite-path", "", "bootstrap or new destination account SQLite path") + jobID := flags.String("job-id", "", "exact pending storage switch job to resume or abort") + configFile := flags.String("config", "", "explicit configuration file for storage setup") + configDir := flags.String("config-dir", ".", "configuration discovery directory; relative data paths still use the working directory") + if err := flags.Parse(args); err != nil { + if errors.Is(err, flag.ErrHelp) { + flags.SetOutput(out) + flags.PrintDefaults() + return nil + } + return errors.New("invalid migration arguments; use -help") + } + if flags.NArg() != 0 { + return errors.New("unexpected positional migration argument; use -help") + } + if *action != "" { + if *source != "" || *accountSource != "" || *resume || *checkReady || *checkImportKind != "" { + return errors.New("storage actions cannot be combined with legacy import flags") + } + return runStorageAction(ctx, storageOptions{Action: *action, SelectionFile: *selectionFile, Backend: *backend, SQLitePath: *sqlitePath, UsersSQLitePath: *usersSQLitePath, OwnerURL: *databaseURL, UsersOwnerURL: *usersURL, StateDir: *stateDir, JobID: *jobID, Offline: *offline, ConfigFile: *configFile, ConfigDir: *configDir}, out) + } + if *stateDir == "" { + *stateDir = "state/migration" + } + if *databaseURL == "" { + *databaseURL = os.Getenv("CORESCOPE_DATABASE_URL") + } + if *usersURL == "" { + *usersURL = os.Getenv("CORESCOPE_USERS_DATABASE_URL") + } + if *databaseURL == "" && *usersURL == "" { + return errors.New("provide CORESCOPE_DATABASE_URL and/or CORESCOPE_USERS_DATABASE_URL") + } + if *databaseURL != "" && *usersURL != "" { + same, err := pgutil.SameDatabase(ctx, *databaseURL, *usersURL) + if err != nil { + return err + } + if same { + return errors.New("telemetry and accounts require separate PostgreSQL databases") + } + } + type store struct { + kind, dsn, source string + } + stores := []store{{kind: "telemetry", dsn: *databaseURL, source: *source}, {kind: "accounts", dsn: *usersURL, source: *accountSource}} + if *checkImportKind != "" { + if *checkReady || *resume { + return errors.New("-check-import-kind cannot be combined with -check-ready or -resume") + } + for _, s := range stores { + if s.kind == *checkImportKind { + if s.dsn == "" || s.source == "" { + return errors.New("completed-import check requires the matching PostgreSQL URL and SQLite source path") + } + if err := checkImportedSource(ctx, s.dsn, s.kind, s.source); err != nil { + return err + } + fmt.Fprintf(out, "%s verified import matches the retained SQLite source\n", s.kind) + return nil + } + } + return errors.New("-check-import-kind must be telemetry or accounts") + } + importing := *source != "" || *accountSource != "" + if importing && !*offline { + return errors.New("SQLite import requires -offline after stopping all telemetry and account writers") + } + if *checkReady && (importing || *resume) { + return errors.New("-check-ready cannot be combined with import or resume") + } + if *resume && !importing { + return errors.New("-resume requires a SQLite source") + } + var imported []store + for _, s := range stores { + if s.dsn == "" { + if s.source != "" { + return fmt.Errorf("%s source requires its PostgreSQL owner URL", s.kind) + } + continue + } + if *checkReady { + db, err := pgutil.Open(s.dsn, false) + if err != nil { + return err + } + if s.kind == "telemetry" { + err = dbschema.AssertReady(db) + } else { + err = users.AssertReady(db) + } + db.Close() + if err != nil { + return err + } + fmt.Fprintf(out, "%s PostgreSQL schema is ready\n", s.kind) + continue + } + if s.source != "" { + report, err := importSQLite(ctx, importOptions{Source: s.source, DatabaseURL: s.dsn, StateDir: *stateDir, Kind: s.kind, Resume: *resume}) + if err != nil { + return err + } + if !report.Verified { + return errors.New("import returned without complete verification") + } + if err := json.NewEncoder(out).Encode(report); err != nil { + return err + } + imported = append(imported, s) + } + } + if *checkReady { + return nil + } + // Empty/absent optional stores are initialized only after every requested + // SQLite import has verified, so an account failure cannot cut over telemetry. + for _, s := range stores { + if s.dsn == "" || s.source != "" { + continue + } + db, err := pgutil.Open(s.dsn, false) + if err != nil { + return err + } + if s.kind == "telemetry" { + err = dbschema.Apply(db, nil) + } else { + err = users.Apply(db) + } + db.Close() + if err != nil { + return err + } + fmt.Fprintf(out, "%s PostgreSQL schema initialized\n", s.kind) + } + var opened []store + for _, s := range imported { + didOpen, err := finalizeImport(ctx, s.dsn, s.kind) + if err != nil { + // There is no cross-database transaction. Writers stay offline throughout + // cutover; close only gates this invocation actually opened. Another + // finalizer can open a gate after our earlier import verification. + for _, prior := range opened { + db, openErr := pgutil.Open(prior.dsn, false) + if openErr == nil { + _, resetErr := db.Exec(`UPDATE corescope_schema SET ready=false WHERE kind=$1`, prior.kind) + db.Close() + if resetErr != nil { + return errors.Join(err, errors.New("an earlier readiness gate could not be reset; keep services stopped")) + } + } else { + return errors.Join(err, errors.New("an earlier readiness gate could not be reset; keep services stopped")) + } + } + return err + } + if didOpen { + opened = append(opened, s) + } + } + for _, s := range imported { + fmt.Fprintf(out, "%s import verified and finalized; keep the recovery snapshot for rollback\n", s.kind) + } + return nil +} - db, err := sql.Open("sqlite", *dbPath) +func checkImportedSource(ctx context.Context, databaseURL, kind, source string) error { + db, err := pgutil.Open(databaseURL, true) if err != nil { - log.Fatalf("open %s: %v", *dbPath, err) + return err } defer db.Close() - - if err := db.Ping(); err != nil { - log.Fatalf("ping %s: %v", *dbPath, err) + if kind == "telemetry" { + err = dbschema.AssertReady(db) + } else { + err = users.AssertReady(db) } - - if err := dbschema.Apply(db, log.Printf); err != nil { - log.Fatalf("dbschema.Apply: %v", err) + if err != nil { + return err } - - if err := dbschema.AssertReady(db); err != nil { - log.Fatalf("dbschema.AssertReady after Apply: %v (this is a bug — Apply did not produce a ready schema)", err) + var runID, recordedKind, digest, normalized, reportText string + var verified bool + if err := db.QueryRowContext(ctx, `SELECT run_id,kind,source_digest,normalized_digest,verified,report FROM corescope_import WHERE id=1`).Scan(&runID, &recordedKind, &digest, &normalized, &verified, &reportText); err != nil { + return errors.New("PostgreSQL has no completed SQLite import; run the explicit offline upgrade before starting services") } - - log.Printf("OK: %s is migrated and ready", *dbPath) + var report importReport + if !verified || recordedKind != kind || runID == "" || normalized == "" || json.Unmarshal([]byte(reportText), &report) != nil || !report.Verified || report.Kind != kind { + return errors.New("PostgreSQL import verification marker is incomplete or belongs to another store") + } + tables, err := layouts(kind) + if err != nil { + return err + } + if len(report.Tables) != len(tables) { + return errors.New("PostgreSQL import report is incomplete") + } + for i, table := range tables { + if report.Tables[i].Table != table.Name || report.Tables[i].Rows < 0 || len(report.Tables[i].SHA256) != 64 { + return errors.New("PostgreSQL import report is invalid") + } + } + actual, err := fingerprint(ctx, source) + if err != nil { + return err + } + if actual != digest { + return errors.New("retained SQLite source does not match this PostgreSQL import; keep services stopped") + } + return nil } diff --git a/cmd/migrate/main_test.go b/cmd/migrate/main_test.go index 63956fbcf..b08121737 100644 --- a/cmd/migrate/main_test.go +++ b/cmd/migrate/main_test.go @@ -6,14 +6,20 @@ package main import ( + "context" "database/sql" + "fmt" "io" "os" + "os/exec" "path/filepath" + "strings" "testing" - "github.com/meshcore-analyzer/dbschema" - _ "modernc.org/sqlite" + _ "github.com/mattn/go-sqlite3" + postgresSchemaAPI "github.com/meshcore-analyzer/dbschema" + dbschema "github.com/meshcore-analyzer/dbschema/legacy" + "github.com/meshcore-analyzer/packetpath" ) // fixtureCandidates lists possible locations of the committed e2e @@ -31,10 +37,152 @@ func locateFixture(t *testing.T) string { return abs } } - t.Skipf("e2e fixture not found (looked in: %v)", fixtureCandidates) + t.Fatalf("committed e2e fixture not found (looked in: %v)", fixtureCandidates) return "" } +func TestCommittedFixtureRequiresExplicitDuplicateRepair(t *testing.T) { + source := locateFixture(t) + _, err := importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: postgresSchema(t), StateDir: t.TempDir(), Kind: "telemetry"}) + if err == nil || !strings.Contains(err.Error(), "duplicate observation identity") { + t.Fatalf("raw legacy fixture must require explicit duplicate repair: %v", err) + } +} + +func TestPostgresImportCommittedFixture(t *testing.T) { + original := locateFixture(t) + before, err := sourceFingerprint(original) + if err != nil { + t.Fatal(err) + } + source := filepath.Join(t.TempDir(), "reviewed-fixture.sqlite") + if err := snapshotSource(context.Background(), original, source); err != nil { + t.Fatal(err) + } + legacyDB, err := openSQLite(source, "rw") + if err != nil { + t.Fatal(err) + } + // This historical fixture has one duplicate identity group. Its documented + // legacy repair is explicit and confined to this disposable copy; the runtime + // importer itself rejects duplicates and never silently collapses records. + if err := dbschema.Apply(legacyDB, t.Logf); err != nil { + legacyDB.Close() + t.Fatal(err) + } + var wantTransmissions, wantObservations int + if err := legacyDB.QueryRow(`SELECT count(*) FROM transmissions`).Scan(&wantTransmissions); err != nil { + t.Fatal(err) + } + if err := legacyDB.QueryRow(`SELECT count(*) FROM observations`).Scan(&wantObservations); err != nil { + t.Fatal(err) + } + legacyDB.Close() + dsn := postgresSchema(t) + report, err := importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: dsn, StateDir: t.TempDir(), Kind: "telemetry", BatchSize: 64}) + if err != nil { + t.Fatal(err) + } + if !report.Verified { + t.Fatal("committed fixture import was not verified") + } + if _, err := finalizeImport(context.Background(), dsn, "telemetry"); err != nil { + t.Fatal(err) + } + db := openImportDB(t, dsn) + if err := postgresSchemaAPI.AssertReady(db); err != nil { + t.Fatal(err) + } + var transmissions, observations int + if err := db.QueryRow(`SELECT count(*) FROM transmissions`).Scan(&transmissions); err != nil { + t.Fatal(err) + } + if err := db.QueryRow(`SELECT count(*) FROM observations`).Scan(&observations); err != nil { + t.Fatal(err) + } + if transmissions != wantTransmissions || observations != wantObservations || transmissions == 0 { + t.Fatalf("reviewed fixture changed: %d/%d transmissions, %d/%d observations", transmissions, wantTransmissions, observations, wantObservations) + } + after, err := sourceFingerprint(original) + if err != nil { + t.Fatal(err) + } + if after != before { + t.Fatal("committed fixture changed during import") + } +} + +func TestFixturePreparationToolKeepsOriginalAndRefusesOverwrite(t *testing.T) { + source := locateFixture(t) + before, err := sourceFingerprint(source) + if err != nil { + t.Fatal(err) + } + destination := filepath.Join(t.TempDir(), "prepared.sqlite") + args := []string{"run", "../../scripts/prepare-postgres-fixture.go", "-source", source, "-destination", destination} + output, err := exec.Command("go", args...).CombinedOutput() + if err != nil { + t.Fatalf("fixture tool: %v: %s", err, output) + } + if !strings.Contains(string(output), "observations:") || !strings.Contains(string(output), "original fixture unchanged") { + t.Fatal("fixture tool did not report the repair counts") + } + if output, err := exec.Command("go", "run", "../../scripts/migrate-fixture-hashes.go", destination).CombinedOutput(); err != nil { + t.Fatalf("prepared fixture hash normalization: %v: %s", err, output) + } + db, err := openSQLite(destination, "ro") + if err != nil { + t.Fatal(err) + } + defer db.Close() + if err := dbschema.CheckLegacySource(db); err != nil { + t.Fatal(err) + } + if err := exec.Command("go", args...).Run(); err == nil { + t.Fatal("fixture tool overwrote an existing destination") + } + after, err := sourceFingerprint(source) + if err != nil || after != before { + t.Fatal("fixture tool changed original source", err) + } +} + +func TestFixtureHashToolUsesSharedRuntimeIdentity(t *testing.T) { + file := filepath.Join(t.TempDir(), "hashes.sqlite") + db, err := sql.Open("sqlite3", file) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`CREATE TABLE transmissions(id INTEGER PRIMARY KEY,raw_hex TEXT,hash TEXT UNIQUE)`); err != nil { + t.Fatal(err) + } + raws := []string{"14010203040001020304", "150001020305", "160001020306", "17010203040001020307", "250001020304", "1d0001020304"} + for i, raw := range raws { + if _, err := db.Exec(`INSERT INTO transmissions VALUES(?,?,?)`, i, raw, fmt.Sprint("old-", i)); err != nil { + t.Fatal(err) + } + } + db.Close() + output, err := exec.Command("go", "run", "../../scripts/migrate-fixture-hashes.go", file).CombinedOutput() + if err != nil { + t.Fatalf("hash fixture: %v: %s", err, output) + } + db, err = openSQLite(file, "ro") + if err != nil { + t.Fatal(err) + } + defer db.Close() + for i, raw := range raws { + var got string + if err := db.QueryRow(`SELECT hash FROM transmissions WHERE id=?`, i).Scan(&got); err != nil { + t.Fatal(err) + } + if want := packetpath.ContentHash(raw); got != want { + t.Fatalf("fixture route/type %s hash=%s, want runtime %s", raw, got, want) + } + } +} + func copyFile(t *testing.T, src, dst string) { t.Helper() in, err := os.Open(src) @@ -61,7 +209,7 @@ func TestMigrateBringsFixtureToReady(t *testing.T) { dst := filepath.Join(t.TempDir(), "fixture-copy.db") copyFile(t, src, dst) - db, err := sql.Open("sqlite", dst) + db, err := sql.Open("sqlite3", dst) if err != nil { t.Fatalf("open: %v", err) } diff --git a/cmd/migrate/operator_paths_test.go b/cmd/migrate/operator_paths_test.go new file mode 100644 index 000000000..9649ab2c7 --- /dev/null +++ b/cmd/migrate/operator_paths_test.go @@ -0,0 +1,81 @@ +package main + +import ( + "bytes" + "context" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestOperatorCustomSourcePathsAndOptionalAccounts(t *testing.T) { + for _, existingAccounts := range []bool{false, true} { + t.Run(map[bool]string{false: "no previous accounts", true: "preserve existing accounts"}[existingAccounts], func(t *testing.T) { + t.Setenv("CORESCOPE_DATABASE_URL", postgresDatabase(t)) + accounts := postgresDatabase(t) + t.Setenv("CORESCOPE_USERS_DATABASE_URL", accounts) + dir := filepath.Join(t.TempDir(), "custom state and sources") + if err := os.Mkdir(dir, 0o700); err != nil { + t.Fatal(err) + } + source := filepath.Join(dir, "telemetry retained.db") + if err := os.Rename(telemetrySource(t), source); err != nil { + t.Fatal(err) + } + args := []string{"-offline", "-from-sqlite", source, "-state-dir", filepath.Join(dir, "migration snapshots")} + paths := []string{source} + if existingAccounts { + users := filepath.Join(dir, "existing accounts.db") + if err := os.Rename(accountSource(t, 5), users); err != nil { + t.Fatal(err) + } + args = append(args, "-users-from-sqlite", users) + paths = append(paths, users) + } + before := make([]string, len(paths)) + for i, path := range paths { + var err error + before[i], err = fingerprint(context.Background(), path) + if err != nil { + t.Fatal(err) + } + } + var output bytes.Buffer + if err := runCommand(context.Background(), args, &output); err != nil { + t.Fatal(err) + } + if err := runCommand(context.Background(), []string{"-check-ready"}, &output); err != nil { + t.Fatal(err) + } + for i, path := range paths { + kind, flag := "telemetry", "-from-sqlite" + if i == 1 { + kind, flag = "accounts", "-users-from-sqlite" + } + if err := runCommand(context.Background(), []string{"-check-import-kind=" + kind, flag, path}, &output); err != nil { + t.Fatal(err) + } + after, err := fingerprint(context.Background(), path) + if err != nil || before[i] != after { + t.Fatal("operator import/check changed the retained source", err) + } + } + var n int + if err := openImportDB(t, accounts).QueryRow(`SELECT count(*) FROM users`).Scan(&n); err != nil { + t.Fatal(err) + } + if (existingAccounts && n != 1) || (!existingAccounts && n != 0) { + t.Fatal("optional account initialization lost or invented accounts") + } + for _, kind := range []string{"telemetry", "accounts"} { + if !strings.Contains(output.String(), kind+" PostgreSQL schema is ready") { + t.Fatal("documented readiness message is missing") + } + } + if !existingAccounts && !strings.Contains(output.String(), "accounts PostgreSQL schema initialized") { + t.Fatal("documented absent-account initialization message is missing") + } + }) + } +} diff --git a/cmd/migrate/postgres_layout.go b/cmd/migrate/postgres_layout.go new file mode 100644 index 000000000..58fb10f71 --- /dev/null +++ b/cmd/migrate/postgres_layout.go @@ -0,0 +1,125 @@ +package main + +import ( + "database/sql" + _ "embed" + "encoding/json" + "errors" + "slices" + "sort" + "strings" +) + +// Captured from the versioned native ApplyPostgres entrypoints. The regression +// recreates both canonical schemas and compares this manifest, so a physical +// schema change requires an explicit converter compatibility decision. +// +//go:embed postgres_layout.json +var postgresLayoutJSON []byte + +func postgresLayout(db *sql.DB, kind string, tables []importTable) ([]string, error) { + var schema string + if err := db.QueryRow(`SELECT current_schema()`).Scan(&schema); err != nil { + return nil, err + } + allowed := map[string]bool{"corescope_schema": true, "corescope_import": true, "corescope_import_progress": true} + if kind == "accounts" { + allowed["schema_version"] = true + } + var tableNames []string + for _, table := range tables { + tableNames = append(tableNames, `'`+strings.ReplaceAll(table.Name, `'`, `''`)+`'`) + allowed[table.Name] = true + } + view := "packets_v" + if kind == "accounts" { + view = "approved_channels" + } + allowed[view] = true + relationRows, err := db.Query(`SELECT c.relname,c.relkind::text FROM pg_class c WHERE c.relnamespace=current_schema()::regnamespace AND c.relkind IN ('r','p','v','m','S','f') ORDER BY c.relname`) + if err != nil { + return nil, err + } + var relations [][2]string + for relationRows.Next() { + var pair [2]string + if err := relationRows.Scan(&pair[0], &pair[1]); err != nil { + relationRows.Close() + return nil, err + } + relations = append(relations, pair) + } + err = relationRows.Err() + relationRows.Close() + if err != nil { + return nil, err + } + for _, table := range tables { + for _, col := range table.Columns { + if !col.Identity { + continue + } + var name string + if err := db.QueryRow(`SELECT c.relname FROM pg_class c WHERE c.oid=pg_get_serial_sequence($1,$2)::regclass`, table.Name, col.Name).Scan(&name); err != nil { + return nil, err + } + allowed[name] = true + } + } + for _, pair := range relations { + if !allowed[pair[0]] { + return nil, errors.New("unsupported PostgreSQL source object; no source data was discarded") + } + } + names := strings.Join(tableNames, ",") + queries := []string{ + `SELECT json_build_array('column',table_name,ordinal_position,column_name,data_type,is_nullable,column_default,collation_name,is_identity,identity_generation,is_generated,generation_expression)::text FROM information_schema.columns WHERE table_schema=current_schema() AND table_name IN (` + names + `)`, + `SELECT json_build_array('table',c.relname,c.relkind,c.relrowsecurity,c.relforcerowsecurity)::text FROM pg_class c WHERE c.relnamespace=current_schema()::regnamespace AND c.relname IN (` + names + `)`, + `SELECT json_build_array('constraint',c.relname,k.conname,k.convalidated,pg_get_constraintdef(k.oid))::text FROM pg_constraint k JOIN pg_class c ON c.oid=k.conrelid WHERE c.relnamespace=current_schema()::regnamespace AND c.relname IN (` + names + `)`, + `SELECT json_build_array('index',c.relname,pg_get_indexdef(i.indexrelid))::text FROM pg_index i JOIN pg_class c ON c.oid=i.indrelid WHERE c.relnamespace=current_schema()::regnamespace AND c.relname IN (` + names + `)`, + `SELECT json_build_array('trigger',c.relname,pg_get_triggerdef(t.oid))::text FROM pg_trigger t JOIN pg_class c ON c.oid=t.tgrelid WHERE NOT t.tgisinternal AND c.relnamespace=current_schema()::regnamespace AND c.relname IN (` + names + `)`, + `SELECT json_build_array('policy',c.relname,p.polname,pg_get_expr(p.polqual,p.polrelid),pg_get_expr(p.polwithcheck,p.polrelid))::text FROM pg_policy p JOIN pg_class c ON c.oid=p.polrelid WHERE c.relnamespace=current_schema()::regnamespace AND c.relname IN (` + names + `)`, + `SELECT json_build_array('view',c.relname,pg_get_viewdef(c.oid))::text FROM pg_class c WHERE c.relnamespace=current_schema()::regnamespace AND c.relkind='v' AND c.relname='` + view + `'`, + } + var out []string + for _, query := range queries { + rows, err := db.Query(query) + if err != nil { + return nil, err + } + for rows.Next() { + var value string + if err := rows.Scan(&value); err != nil { + rows.Close() + return nil, err + } + // Only catalog qualification varies between installations. User data is + // never normalized, decoded or rewritten by these schema comparisons. + value = strings.ReplaceAll(value, strings.ReplaceAll(quote(schema), `"`, `\"`)+".", "{schema}.") + value = strings.ReplaceAll(value, schema+".", "{schema}.") + out = append(out, value) + } + err = rows.Err() + rows.Close() + if err != nil { + return nil, err + } + } + sort.Strings(out) + return out, nil +} + +func validatePostgresLayout(db *sql.DB, kind string, tables []importTable) error { + var expected map[string][]string + if err := json.Unmarshal(postgresLayoutJSON, &expected); err != nil { + return errors.New("invalid compiled PostgreSQL conversion layout") + } + got, err := postgresLayout(db, kind, tables) + if err != nil { + return err + } + if !slices.Equal(got, expected[kind]) { + return errors.New("unsupported PostgreSQL columns, constraints, indexes, policies or views; conversion refused without changing the source") + } + return nil +} diff --git a/cmd/migrate/postgres_layout.json b/cmd/migrate/postgres_layout.json new file mode 100644 index 000000000..bb0c02ff9 --- /dev/null +++ b/cmd/migrate/postgres_layout.json @@ -0,0 +1,575 @@ +{ + "accounts": [ + "[\"column\", \"audit_log\", 1, \"id\", \"bigint\", \"NO\", null, null, \"YES\", \"BY DEFAULT\", \"NEVER\", null]", + "[\"column\", \"audit_log\", 2, \"at\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"audit_log\", 3, \"actor_user_id\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"audit_log\", 4, \"action\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"audit_log\", 5, \"target_user_id\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"audit_log\", 6, \"detail\", \"text\", \"NO\", \"'{}'::text\", \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"mail_events\", 1, \"id\", \"bigint\", \"NO\", null, null, \"YES\", \"BY DEFAULT\", \"NEVER\", null]", + "[\"column\", \"mail_events\", 2, \"mail_id\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"mail_events\", 3, \"event\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"mail_events\", 4, \"at\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"mail_events\", 5, \"reason\", \"text\", \"NO\", \"''::text\", \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"mail_log\", 1, \"id\", \"bigint\", \"NO\", null, null, \"YES\", \"BY DEFAULT\", \"NEVER\", null]", + "[\"column\", \"mail_log\", 2, \"user_id\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"mail_log\", 3, \"to_email\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"mail_log\", 4, \"purpose\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"mail_log\", 5, \"provider_message_id\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"mail_log\", 6, \"sent_at\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"mail_log\", 7, \"last_event\", \"text\", \"NO\", \"'sent'::text\", \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"mail_log\", 8, \"last_event_at\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"mail_log\", 9, \"last_reason\", \"text\", \"NO\", \"''::text\", \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"notification_prefs\", 1, \"user_id\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"notification_prefs\", 2, \"enabled\", \"bigint\", \"NO\", \"1\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"notification_prefs\", 3, \"events\", \"text\", \"NO\", \"''::text\", \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"notification_prefs\", 4, \"unsub_token\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"notification_prefs\", 5, \"updated_at\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"notification_state\", 1, \"user_id\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"notification_state\", 2, \"event\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"notification_state\", 3, \"subject\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"notification_state\", 4, \"state\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"notification_state\", 5, \"changed_at\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"notification_watches\", 1, \"user_id\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"notification_watches\", 2, \"pubkey\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"notification_watches\", 3, \"created_at\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"proposals\", 1, \"id\", \"bigint\", \"NO\", null, null, \"YES\", \"BY DEFAULT\", \"NEVER\", null]", + "[\"column\", \"proposals\", 2, \"kind\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"proposals\", 3, \"subject\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"proposals\", 4, \"status\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"proposals\", 5, \"proposer_id\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"proposals\", 6, \"reviewer_id\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"proposals\", 7, \"note\", \"text\", \"NO\", \"''::text\", \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"proposals\", 8, \"created_at\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"proposals\", 9, \"decided_at\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"sessions\", 1, \"id\", \"bigint\", \"NO\", null, null, \"YES\", \"BY DEFAULT\", \"NEVER\", null]", + "[\"column\", \"sessions\", 2, \"token_hash\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"sessions\", 3, \"user_id\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"sessions\", 4, \"csrf_token\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"sessions\", 5, \"created_at\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"sessions\", 6, \"expires_at\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"sessions\", 7, \"last_seen_at\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"sessions\", 8, \"user_agent\", \"text\", \"NO\", \"''::text\", \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"tokens\", 1, \"token_hash\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"tokens\", 2, \"user_id\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"tokens\", 3, \"purpose\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"tokens\", 4, \"new_email\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"tokens\", 5, \"expires_at\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"tokens\", 6, \"used_at\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"user_settings\", 1, \"user_id\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"user_settings\", 2, \"doc\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"user_settings\", 3, \"revision\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"user_settings\", 4, \"generation\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"user_settings\", 5, \"updated_at\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"users\", 1, \"id\", \"bigint\", \"NO\", null, null, \"YES\", \"BY DEFAULT\", \"NEVER\", null]", + "[\"column\", \"users\", 10, \"last_login_at\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"users\", 11, \"email_bouncing\", \"bigint\", \"NO\", \"0\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"users\", 2, \"email\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"users\", 3, \"display_name\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"users\", 4, \"password_hash\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"users\", 5, \"role\", \"text\", \"NO\", \"'user'::text\", \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"users\", 6, \"status\", \"text\", \"NO\", \"'pending'::text\", \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"users\", 7, \"created_at\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"users\", 8, \"activated_at\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"users\", 9, \"activated_by\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"constraint\", \"audit_log\", \"audit_log_action_not_null\", true, \"NOT NULL action\"]", + "[\"constraint\", \"audit_log\", \"audit_log_at_not_null\", true, \"NOT NULL at\"]", + "[\"constraint\", \"audit_log\", \"audit_log_detail_not_null\", true, \"NOT NULL detail\"]", + "[\"constraint\", \"audit_log\", \"audit_log_id_not_null\", true, \"NOT NULL id\"]", + "[\"constraint\", \"audit_log\", \"audit_log_pkey\", true, \"PRIMARY KEY (id)\"]", + "[\"constraint\", \"mail_events\", \"mail_events_at_not_null\", true, \"NOT NULL at\"]", + "[\"constraint\", \"mail_events\", \"mail_events_event_not_null\", true, \"NOT NULL event\"]", + "[\"constraint\", \"mail_events\", \"mail_events_id_not_null\", true, \"NOT NULL id\"]", + "[\"constraint\", \"mail_events\", \"mail_events_mail_id_event_at_key\", true, \"UNIQUE (mail_id, event, at)\"]", + "[\"constraint\", \"mail_events\", \"mail_events_mail_id_fkey\", true, \"FOREIGN KEY (mail_id) REFERENCES mail_log(id) ON DELETE CASCADE\"]", + "[\"constraint\", \"mail_events\", \"mail_events_mail_id_not_null\", true, \"NOT NULL mail_id\"]", + "[\"constraint\", \"mail_events\", \"mail_events_pkey\", true, \"PRIMARY KEY (id)\"]", + "[\"constraint\", \"mail_events\", \"mail_events_reason_not_null\", true, \"NOT NULL reason\"]", + "[\"constraint\", \"mail_log\", \"mail_log_id_not_null\", true, \"NOT NULL id\"]", + "[\"constraint\", \"mail_log\", \"mail_log_last_event_at_not_null\", true, \"NOT NULL last_event_at\"]", + "[\"constraint\", \"mail_log\", \"mail_log_last_event_not_null\", true, \"NOT NULL last_event\"]", + "[\"constraint\", \"mail_log\", \"mail_log_last_reason_not_null\", true, \"NOT NULL last_reason\"]", + "[\"constraint\", \"mail_log\", \"mail_log_pkey\", true, \"PRIMARY KEY (id)\"]", + "[\"constraint\", \"mail_log\", \"mail_log_purpose_not_null\", true, \"NOT NULL purpose\"]", + "[\"constraint\", \"mail_log\", \"mail_log_sent_at_not_null\", true, \"NOT NULL sent_at\"]", + "[\"constraint\", \"mail_log\", \"mail_log_to_email_not_null\", true, \"NOT NULL to_email\"]", + "[\"constraint\", \"mail_log\", \"mail_log_user_id_fkey\", true, \"FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE SET NULL\"]", + "[\"constraint\", \"notification_prefs\", \"notification_prefs_enabled_check\", true, \"CHECK ((enabled = ANY (ARRAY[(0)::bigint, (1)::bigint])))\"]", + "[\"constraint\", \"notification_prefs\", \"notification_prefs_enabled_not_null\", true, \"NOT NULL enabled\"]", + "[\"constraint\", \"notification_prefs\", \"notification_prefs_events_not_null\", true, \"NOT NULL events\"]", + "[\"constraint\", \"notification_prefs\", \"notification_prefs_pkey\", true, \"PRIMARY KEY (user_id)\"]", + "[\"constraint\", \"notification_prefs\", \"notification_prefs_unsub_token_key\", true, \"UNIQUE (unsub_token)\"]", + "[\"constraint\", \"notification_prefs\", \"notification_prefs_unsub_token_not_null\", true, \"NOT NULL unsub_token\"]", + "[\"constraint\", \"notification_prefs\", \"notification_prefs_updated_at_not_null\", true, \"NOT NULL updated_at\"]", + "[\"constraint\", \"notification_prefs\", \"notification_prefs_user_id_fkey\", true, \"FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE\"]", + "[\"constraint\", \"notification_prefs\", \"notification_prefs_user_id_not_null\", true, \"NOT NULL user_id\"]", + "[\"constraint\", \"notification_state\", \"notification_state_changed_at_not_null\", true, \"NOT NULL changed_at\"]", + "[\"constraint\", \"notification_state\", \"notification_state_event_not_null\", true, \"NOT NULL event\"]", + "[\"constraint\", \"notification_state\", \"notification_state_pkey\", true, \"PRIMARY KEY (user_id, event, subject)\"]", + "[\"constraint\", \"notification_state\", \"notification_state_state_not_null\", true, \"NOT NULL state\"]", + "[\"constraint\", \"notification_state\", \"notification_state_subject_not_null\", true, \"NOT NULL subject\"]", + "[\"constraint\", \"notification_state\", \"notification_state_user_id_fkey\", true, \"FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE\"]", + "[\"constraint\", \"notification_state\", \"notification_state_user_id_not_null\", true, \"NOT NULL user_id\"]", + "[\"constraint\", \"notification_watches\", \"notification_watches_created_at_not_null\", true, \"NOT NULL created_at\"]", + "[\"constraint\", \"notification_watches\", \"notification_watches_pkey\", true, \"PRIMARY KEY (user_id, pubkey)\"]", + "[\"constraint\", \"notification_watches\", \"notification_watches_pubkey_not_null\", true, \"NOT NULL pubkey\"]", + "[\"constraint\", \"notification_watches\", \"notification_watches_user_id_fkey\", true, \"FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE\"]", + "[\"constraint\", \"notification_watches\", \"notification_watches_user_id_not_null\", true, \"NOT NULL user_id\"]", + "[\"constraint\", \"proposals\", \"proposals_created_at_not_null\", true, \"NOT NULL created_at\"]", + "[\"constraint\", \"proposals\", \"proposals_id_not_null\", true, \"NOT NULL id\"]", + "[\"constraint\", \"proposals\", \"proposals_kind_not_null\", true, \"NOT NULL kind\"]", + "[\"constraint\", \"proposals\", \"proposals_note_not_null\", true, \"NOT NULL note\"]", + "[\"constraint\", \"proposals\", \"proposals_pkey\", true, \"PRIMARY KEY (id)\"]", + "[\"constraint\", \"proposals\", \"proposals_proposer_id_fkey\", true, \"FOREIGN KEY (proposer_id) REFERENCES users(id) ON DELETE SET NULL\"]", + "[\"constraint\", \"proposals\", \"proposals_reviewer_id_fkey\", true, \"FOREIGN KEY (reviewer_id) REFERENCES users(id) ON DELETE SET NULL\"]", + "[\"constraint\", \"proposals\", \"proposals_status_check\", true, \"CHECK ((status = ANY (ARRAY['pending'::text, 'approved'::text, 'rejected'::text, 'revoked'::text])))\"]", + "[\"constraint\", \"proposals\", \"proposals_status_not_null\", true, \"NOT NULL status\"]", + "[\"constraint\", \"proposals\", \"proposals_subject_not_null\", true, \"NOT NULL subject\"]", + "[\"constraint\", \"sessions\", \"sessions_created_at_not_null\", true, \"NOT NULL created_at\"]", + "[\"constraint\", \"sessions\", \"sessions_csrf_token_not_null\", true, \"NOT NULL csrf_token\"]", + "[\"constraint\", \"sessions\", \"sessions_expires_at_not_null\", true, \"NOT NULL expires_at\"]", + "[\"constraint\", \"sessions\", \"sessions_id_not_null\", true, \"NOT NULL id\"]", + "[\"constraint\", \"sessions\", \"sessions_last_seen_at_not_null\", true, \"NOT NULL last_seen_at\"]", + "[\"constraint\", \"sessions\", \"sessions_pkey\", true, \"PRIMARY KEY (id)\"]", + "[\"constraint\", \"sessions\", \"sessions_token_hash_key\", true, \"UNIQUE (token_hash)\"]", + "[\"constraint\", \"sessions\", \"sessions_token_hash_not_null\", true, \"NOT NULL token_hash\"]", + "[\"constraint\", \"sessions\", \"sessions_user_agent_not_null\", true, \"NOT NULL user_agent\"]", + "[\"constraint\", \"sessions\", \"sessions_user_id_fkey\", true, \"FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE\"]", + "[\"constraint\", \"sessions\", \"sessions_user_id_not_null\", true, \"NOT NULL user_id\"]", + "[\"constraint\", \"tokens\", \"tokens_expires_at_not_null\", true, \"NOT NULL expires_at\"]", + "[\"constraint\", \"tokens\", \"tokens_pkey\", true, \"PRIMARY KEY (token_hash)\"]", + "[\"constraint\", \"tokens\", \"tokens_purpose_check\", true, \"CHECK ((purpose = ANY (ARRAY['activate'::text, 'reset'::text, 'email_change'::text])))\"]", + "[\"constraint\", \"tokens\", \"tokens_purpose_not_null\", true, \"NOT NULL purpose\"]", + "[\"constraint\", \"tokens\", \"tokens_token_hash_not_null\", true, \"NOT NULL token_hash\"]", + "[\"constraint\", \"tokens\", \"tokens_user_id_fkey\", true, \"FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE\"]", + "[\"constraint\", \"tokens\", \"tokens_user_id_not_null\", true, \"NOT NULL user_id\"]", + "[\"constraint\", \"user_settings\", \"user_settings_doc_not_null\", true, \"NOT NULL doc\"]", + "[\"constraint\", \"user_settings\", \"user_settings_generation_not_null\", true, \"NOT NULL generation\"]", + "[\"constraint\", \"user_settings\", \"user_settings_pkey\", true, \"PRIMARY KEY (user_id)\"]", + "[\"constraint\", \"user_settings\", \"user_settings_revision_not_null\", true, \"NOT NULL revision\"]", + "[\"constraint\", \"user_settings\", \"user_settings_updated_at_not_null\", true, \"NOT NULL updated_at\"]", + "[\"constraint\", \"user_settings\", \"user_settings_user_id_fkey\", true, \"FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE\"]", + "[\"constraint\", \"user_settings\", \"user_settings_user_id_not_null\", true, \"NOT NULL user_id\"]", + "[\"constraint\", \"users\", \"users_created_at_not_null\", true, \"NOT NULL created_at\"]", + "[\"constraint\", \"users\", \"users_display_name_not_null\", true, \"NOT NULL display_name\"]", + "[\"constraint\", \"users\", \"users_email_bouncing_check\", true, \"CHECK ((email_bouncing = ANY (ARRAY[(0)::bigint, (1)::bigint])))\"]", + "[\"constraint\", \"users\", \"users_email_bouncing_not_null\", true, \"NOT NULL email_bouncing\"]", + "[\"constraint\", \"users\", \"users_email_key\", true, \"UNIQUE (email)\"]", + "[\"constraint\", \"users\", \"users_email_not_null\", true, \"NOT NULL email\"]", + "[\"constraint\", \"users\", \"users_id_not_null\", true, \"NOT NULL id\"]", + "[\"constraint\", \"users\", \"users_password_hash_not_null\", true, \"NOT NULL password_hash\"]", + "[\"constraint\", \"users\", \"users_pkey\", true, \"PRIMARY KEY (id)\"]", + "[\"constraint\", \"users\", \"users_role_check\", true, \"CHECK ((role = ANY (ARRAY['user'::text, 'admin'::text])))\"]", + "[\"constraint\", \"users\", \"users_role_not_null\", true, \"NOT NULL role\"]", + "[\"constraint\", \"users\", \"users_status_check\", true, \"CHECK ((status = ANY (ARRAY['pending'::text, 'active'::text, 'disabled'::text])))\"]", + "[\"constraint\", \"users\", \"users_status_not_null\", true, \"NOT NULL status\"]", + "[\"index\", \"audit_log\", \"CREATE INDEX audit_actor ON {schema}.audit_log USING btree (actor_user_id)\"]", + "[\"index\", \"audit_log\", \"CREATE INDEX audit_at ON {schema}.audit_log USING btree (at)\"]", + "[\"index\", \"audit_log\", \"CREATE INDEX audit_target ON {schema}.audit_log USING btree (target_user_id)\"]", + "[\"index\", \"audit_log\", \"CREATE UNIQUE INDEX audit_log_pkey ON {schema}.audit_log USING btree (id)\"]", + "[\"index\", \"mail_events\", \"CREATE UNIQUE INDEX mail_events_mail_id_event_at_key ON {schema}.mail_events USING btree (mail_id, event, at)\"]", + "[\"index\", \"mail_events\", \"CREATE UNIQUE INDEX mail_events_pkey ON {schema}.mail_events USING btree (id)\"]", + "[\"index\", \"mail_log\", \"CREATE INDEX mail_user ON {schema}.mail_log USING btree (user_id)\"]", + "[\"index\", \"mail_log\", \"CREATE UNIQUE INDEX mail_log_pkey ON {schema}.mail_log USING btree (id)\"]", + "[\"index\", \"mail_log\", \"CREATE UNIQUE INDEX mail_msgid ON {schema}.mail_log USING btree (provider_message_id) WHERE (provider_message_id IS NOT NULL)\"]", + "[\"index\", \"notification_prefs\", \"CREATE UNIQUE INDEX notification_prefs_pkey ON {schema}.notification_prefs USING btree (user_id)\"]", + "[\"index\", \"notification_prefs\", \"CREATE UNIQUE INDEX notification_prefs_unsub_token_key ON {schema}.notification_prefs USING btree (unsub_token)\"]", + "[\"index\", \"notification_state\", \"CREATE UNIQUE INDEX notification_state_pkey ON {schema}.notification_state USING btree (user_id, event, subject)\"]", + "[\"index\", \"notification_watches\", \"CREATE UNIQUE INDEX notification_watches_pkey ON {schema}.notification_watches USING btree (user_id, pubkey)\"]", + "[\"index\", \"proposals\", \"CREATE INDEX proposals_status ON {schema}.proposals USING btree (status)\"]", + "[\"index\", \"proposals\", \"CREATE UNIQUE INDEX proposals_kind_subject ON {schema}.proposals USING btree (kind, subject)\"]", + "[\"index\", \"proposals\", \"CREATE UNIQUE INDEX proposals_pkey ON {schema}.proposals USING btree (id)\"]", + "[\"index\", \"sessions\", \"CREATE INDEX sessions_user ON {schema}.sessions USING btree (user_id)\"]", + "[\"index\", \"sessions\", \"CREATE UNIQUE INDEX sessions_pkey ON {schema}.sessions USING btree (id)\"]", + "[\"index\", \"sessions\", \"CREATE UNIQUE INDEX sessions_token_hash_key ON {schema}.sessions USING btree (token_hash)\"]", + "[\"index\", \"tokens\", \"CREATE INDEX tokens_user ON {schema}.tokens USING btree (user_id, purpose)\"]", + "[\"index\", \"tokens\", \"CREATE UNIQUE INDEX tokens_pkey ON {schema}.tokens USING btree (token_hash)\"]", + "[\"index\", \"user_settings\", \"CREATE UNIQUE INDEX user_settings_pkey ON {schema}.user_settings USING btree (user_id)\"]", + "[\"index\", \"users\", \"CREATE UNIQUE INDEX users_email_key ON {schema}.users USING btree (email)\"]", + "[\"index\", \"users\", \"CREATE UNIQUE INDEX users_pkey ON {schema}.users USING btree (id)\"]", + "[\"table\", \"audit_log\", \"r\", false, false]", + "[\"table\", \"mail_events\", \"r\", false, false]", + "[\"table\", \"mail_log\", \"r\", false, false]", + "[\"table\", \"notification_prefs\", \"r\", false, false]", + "[\"table\", \"notification_state\", \"r\", false, false]", + "[\"table\", \"notification_watches\", \"r\", false, false]", + "[\"table\", \"proposals\", \"r\", false, false]", + "[\"table\", \"sessions\", \"r\", false, false]", + "[\"table\", \"tokens\", \"r\", false, false]", + "[\"table\", \"user_settings\", \"r\", false, false]", + "[\"table\", \"users\", \"r\", false, false]", + "[\"view\", \"approved_channels\", \" SELECT kind,\\n subject,\\n decided_at,\\n id\\n FROM proposals\\n WHERE (status = 'approved'::text);\"]" + ], + "telemetry": [ + "[\"column\", \"_async_migrations\", 1, \"name\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"_async_migrations\", 2, \"status\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"_async_migrations\", 3, \"started_at\", \"text\", \"NO\", \"to_char((CURRENT_TIMESTAMP AT TIME ZONE 'UTC'::text), 'YYYY-MM-DD HH24:MI:SS'::text)\", \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"_async_migrations\", 4, \"ended_at\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"_async_migrations\", 5, \"error\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"_migrations\", 1, \"name\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"advert_evidence_backfill\", 1, \"id\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"advert_evidence_backfill\", 2, \"tx_cursor\", \"bigint\", \"NO\", \"0\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"advert_evidence_backfill\", 3, \"obs_cursor\", \"bigint\", \"NO\", \"0\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"advert_route_evidence\", 1, \"id\", \"bigint\", \"NO\", null, null, \"YES\", \"BY DEFAULT\", \"NEVER\", null]", + "[\"column\", \"advert_route_evidence\", 2, \"tx_id\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"advert_route_evidence\", 3, \"bit\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_observers\", 1, \"pubkey\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_observers\", 2, \"name\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_observers\", 3, \"last_seen\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_receptions\", 1, \"id\", \"bigint\", \"NO\", null, null, \"YES\", \"BY DEFAULT\", \"NEVER\", null]", + "[\"column\", \"client_receptions\", 10, \"rx_at\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_receptions\", 11, \"ingested_at\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_receptions\", 12, \"src\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_receptions\", 2, \"rx_pubkey\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_receptions\", 3, \"heard_key\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_receptions\", 4, \"heard_keylen\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_receptions\", 5, \"rssi\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_receptions\", 6, \"snr\", \"double precision\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_receptions\", 7, \"lat\", \"double precision\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_receptions\", 8, \"lon\", \"double precision\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_receptions\", 9, \"pos_acc_m\", \"double precision\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 1, \"id\", \"bigint\", \"NO\", null, null, \"YES\", \"BY DEFAULT\", \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 10, \"battery_mv\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 11, \"queue_len\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 12, \"errors\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 13, \"noise_floor\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 14, \"last_rssi\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 15, \"last_snr\", \"double precision\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 16, \"tx_air_secs\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 17, \"rx_air_secs\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 18, \"recv\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 19, \"sent\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 2, \"rx_pubkey\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 20, \"flood_rx\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 21, \"direct_rx\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 22, \"flood_tx\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 23, \"direct_tx\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 24, \"recv_errors\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 3, \"sampled_at\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 4, \"ingested_at\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 5, \"lat\", \"double precision\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 6, \"lon\", \"double precision\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 7, \"pos_acc_m\", \"double precision\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 8, \"stationary\", \"bigint\", \"NO\", \"0\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 9, \"uptime_secs\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rx_observations\", 1, \"id\", \"bigint\", \"NO\", null, null, \"YES\", \"BY DEFAULT\", \"NEVER\", null]", + "[\"column\", \"client_rx_observations\", 10, \"scope_name\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rx_observations\", 11, \"hash_size\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rx_observations\", 12, \"hop_count\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rx_observations\", 13, \"path_json\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rx_observations\", 14, \"forwarder\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rx_observations\", 15, \"snr\", \"double precision\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rx_observations\", 16, \"rssi\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rx_observations\", 17, \"lat\", \"double precision\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rx_observations\", 18, \"lon\", \"double precision\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rx_observations\", 19, \"pos_acc_m\", \"double precision\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rx_observations\", 2, \"rx_pubkey\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rx_observations\", 3, \"rx_at\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rx_observations\", 4, \"ingested_at\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rx_observations\", 5, \"pkt_hash\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rx_observations\", 6, \"route_type\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rx_observations\", 7, \"payload_type\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rx_observations\", 8, \"code1\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rx_observations\", 9, \"code2\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"dropped_packets\", 1, \"id\", \"bigint\", \"NO\", null, null, \"YES\", \"BY DEFAULT\", \"NEVER\", null]", + "[\"column\", \"dropped_packets\", 2, \"hash\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"dropped_packets\", 3, \"raw_hex\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"dropped_packets\", 4, \"reason\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"dropped_packets\", 5, \"observer_id\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"dropped_packets\", 6, \"observer_name\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"dropped_packets\", 7, \"node_pubkey\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"dropped_packets\", 8, \"node_name\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"dropped_packets\", 9, \"dropped_at\", \"text\", \"YES\", \"to_char((CURRENT_TIMESTAMP AT TIME ZONE 'UTC'::text), 'YYYY-MM-DD HH24:MI:SS'::text)\", \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"inactive_nodes\", 1, \"public_key\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"inactive_nodes\", 10, \"temperature_c\", \"double precision\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"inactive_nodes\", 11, \"foreign_advert\", \"bigint\", \"YES\", \"0\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"inactive_nodes\", 12, \"default_scope\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"inactive_nodes\", 13, \"configured_scope\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"inactive_nodes\", 14, \"configured_scope_at\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"inactive_nodes\", 15, \"multibyte_sup\", \"bigint\", \"NO\", \"0\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"inactive_nodes\", 16, \"multibyte_evidence\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"inactive_nodes\", 2, \"name\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"inactive_nodes\", 3, \"role\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"inactive_nodes\", 4, \"lat\", \"double precision\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"inactive_nodes\", 5, \"lon\", \"double precision\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"inactive_nodes\", 6, \"last_seen\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"inactive_nodes\", 7, \"first_seen\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"inactive_nodes\", 8, \"advert_count\", \"bigint\", \"YES\", \"0\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"inactive_nodes\", 9, \"battery_mv\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"neighbor_edges\", 1, \"node_a\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"neighbor_edges\", 2, \"node_b\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"neighbor_edges\", 3, \"count\", \"bigint\", \"YES\", \"1\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"neighbor_edges\", 4, \"last_seen\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"node_declared_regions\", 1, \"id\", \"bigint\", \"NO\", null, null, \"YES\", \"BY DEFAULT\", \"NEVER\", null]", + "[\"column\", \"node_declared_regions\", 10, \"pos_acc_m\", \"double precision\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"node_declared_regions\", 11, \"repeater_clock\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"node_declared_regions\", 2, \"target\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"node_declared_regions\", 3, \"rx_pubkey\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"node_declared_regions\", 4, \"observed_at\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"node_declared_regions\", 5, \"ingested_at\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"node_declared_regions\", 6, \"regions_csv\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"node_declared_regions\", 7, \"truncated\", \"bigint\", \"NO\", \"0\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"node_declared_regions\", 8, \"lat\", \"double precision\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"node_declared_regions\", 9, \"lon\", \"double precision\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"nodes\", 1, \"public_key\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"nodes\", 10, \"temperature_c\", \"double precision\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"nodes\", 11, \"foreign_advert\", \"bigint\", \"YES\", \"0\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"nodes\", 12, \"default_scope\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"nodes\", 13, \"configured_scope\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"nodes\", 14, \"configured_scope_at\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"nodes\", 15, \"multibyte_sup\", \"bigint\", \"NO\", \"0\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"nodes\", 16, \"multibyte_evidence\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"nodes\", 2, \"name\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"nodes\", 3, \"role\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"nodes\", 4, \"lat\", \"double precision\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"nodes\", 5, \"lon\", \"double precision\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"nodes\", 6, \"last_seen\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"nodes\", 7, \"first_seen\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"nodes\", 8, \"advert_count\", \"bigint\", \"YES\", \"0\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"nodes\", 9, \"battery_mv\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observations\", 1, \"id\", \"bigint\", \"NO\", null, null, \"YES\", \"BY DEFAULT\", \"NEVER\", null]", + "[\"column\", \"observations\", 10, \"raw_hex\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observations\", 11, \"resolved_path\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observations\", 2, \"transmission_id\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observations\", 3, \"observer_idx\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observations\", 4, \"direction\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observations\", 5, \"snr\", \"double precision\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observations\", 6, \"rssi\", \"double precision\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observations\", 7, \"score\", \"double precision\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observations\", 8, \"path_json\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observations\", 9, \"timestamp\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observer_metrics\", 1, \"observer_id\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observer_metrics\", 2, \"timestamp\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observer_metrics\", 3, \"noise_floor\", \"double precision\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observer_metrics\", 4, \"tx_air_secs\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observer_metrics\", 5, \"rx_air_secs\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observer_metrics\", 6, \"recv_errors\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observer_metrics\", 7, \"battery_mv\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observer_metrics\", 8, \"packets_sent\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observer_metrics\", 9, \"packets_recv\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observers\", 1, \"rowid\", \"bigint\", \"NO\", null, null, \"YES\", \"BY DEFAULT\", \"NEVER\", null]", + "[\"column\", \"observers\", 10, \"client_version\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observers\", 11, \"radio\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observers\", 12, \"battery_mv\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observers\", 13, \"uptime_secs\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observers\", 14, \"noise_floor\", \"double precision\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observers\", 15, \"inactive\", \"bigint\", \"YES\", \"0\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observers\", 16, \"last_packet_at\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observers\", 17, \"clock_skew_seconds\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observers\", 18, \"clock_skew_count_24h\", \"bigint\", \"YES\", \"0\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observers\", 19, \"clock_last_naive_at\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observers\", 2, \"id\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observers\", 20, \"can_relay\", \"bigint\", \"YES\", \"1\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observers\", 21, \"can_relay_seen\", \"bigint\", \"YES\", \"0\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observers\", 3, \"name\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observers\", 4, \"iata\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observers\", 5, \"last_seen\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observers\", 6, \"first_seen\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observers\", 7, \"packet_count\", \"bigint\", \"YES\", \"0\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observers\", 8, \"model\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observers\", 9, \"firmware\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"scope_match_totals\", 1, \"id\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"scope_match_totals\", 2, \"since_unix\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"scope_match_totals\", 3, \"unique_matches\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"scope_match_totals\", 4, \"explicit_over_derived\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"scope_match_totals\", 5, \"ambiguous\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"scope_match_totals\", 6, \"none_matches\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"scope_match_totals\", 7, \"updated_unix\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"transmissions\", 1, \"id\", \"bigint\", \"NO\", null, null, \"YES\", \"BY DEFAULT\", \"NEVER\", null]", + "[\"column\", \"transmissions\", 10, \"last_seen\", \"bigint\", \"NO\", \"0\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"transmissions\", 11, \"created_at\", \"text\", \"YES\", \"to_char((CURRENT_TIMESTAMP AT TIME ZONE 'UTC'::text), 'YYYY-MM-DD HH24:MI:SS'::text)\", \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"transmissions\", 12, \"code1\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"transmissions\", 13, \"code2\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"transmissions\", 14, \"channel_hash\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"transmissions\", 15, \"scope_name\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"transmissions\", 2, \"raw_hex\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"transmissions\", 3, \"hash\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"transmissions\", 4, \"first_seen\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"transmissions\", 5, \"route_type\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"transmissions\", 6, \"payload_type\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"transmissions\", 7, \"payload_version\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"transmissions\", 8, \"decoded_json\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"transmissions\", 9, \"from_pubkey\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"constraint\", \"_async_migrations\", \"_async_migrations_name_not_null\", true, \"NOT NULL name\"]", + "[\"constraint\", \"_async_migrations\", \"_async_migrations_pkey\", true, \"PRIMARY KEY (name)\"]", + "[\"constraint\", \"_async_migrations\", \"_async_migrations_started_at_not_null\", true, \"NOT NULL started_at\"]", + "[\"constraint\", \"_async_migrations\", \"_async_migrations_status_not_null\", true, \"NOT NULL status\"]", + "[\"constraint\", \"_migrations\", \"_migrations_name_not_null\", true, \"NOT NULL name\"]", + "[\"constraint\", \"_migrations\", \"_migrations_pkey\", true, \"PRIMARY KEY (name)\"]", + "[\"constraint\", \"advert_evidence_backfill\", \"advert_evidence_backfill_id_check\", true, \"CHECK ((id = 1))\"]", + "[\"constraint\", \"advert_evidence_backfill\", \"advert_evidence_backfill_id_not_null\", true, \"NOT NULL id\"]", + "[\"constraint\", \"advert_evidence_backfill\", \"advert_evidence_backfill_obs_cursor_not_null\", true, \"NOT NULL obs_cursor\"]", + "[\"constraint\", \"advert_evidence_backfill\", \"advert_evidence_backfill_pkey\", true, \"PRIMARY KEY (id)\"]", + "[\"constraint\", \"advert_evidence_backfill\", \"advert_evidence_backfill_tx_cursor_not_null\", true, \"NOT NULL tx_cursor\"]", + "[\"constraint\", \"advert_route_evidence\", \"advert_route_evidence_bit_check\", true, \"CHECK ((\\\"bit\\\" = ANY (ARRAY[(1)::bigint, (2)::bigint])))\"]", + "[\"constraint\", \"advert_route_evidence\", \"advert_route_evidence_bit_not_null\", true, \"NOT NULL \\\"bit\\\"\"]", + "[\"constraint\", \"advert_route_evidence\", \"advert_route_evidence_id_not_null\", true, \"NOT NULL id\"]", + "[\"constraint\", \"advert_route_evidence\", \"advert_route_evidence_pkey\", true, \"PRIMARY KEY (id)\"]", + "[\"constraint\", \"advert_route_evidence\", \"advert_route_evidence_tx_id_bit_key\", true, \"UNIQUE (tx_id, \\\"bit\\\")\"]", + "[\"constraint\", \"advert_route_evidence\", \"advert_route_evidence_tx_id_fkey\", true, \"FOREIGN KEY (tx_id) REFERENCES transmissions(id) ON DELETE CASCADE\"]", + "[\"constraint\", \"advert_route_evidence\", \"advert_route_evidence_tx_id_not_null\", true, \"NOT NULL tx_id\"]", + "[\"constraint\", \"client_observers\", \"client_observers_pkey\", true, \"PRIMARY KEY (pubkey)\"]", + "[\"constraint\", \"client_observers\", \"client_observers_pubkey_not_null\", true, \"NOT NULL pubkey\"]", + "[\"constraint\", \"client_receptions\", \"client_receptions_heard_key_not_null\", true, \"NOT NULL heard_key\"]", + "[\"constraint\", \"client_receptions\", \"client_receptions_heard_keylen_not_null\", true, \"NOT NULL heard_keylen\"]", + "[\"constraint\", \"client_receptions\", \"client_receptions_id_not_null\", true, \"NOT NULL id\"]", + "[\"constraint\", \"client_receptions\", \"client_receptions_ingested_at_not_null\", true, \"NOT NULL ingested_at\"]", + "[\"constraint\", \"client_receptions\", \"client_receptions_lat_not_null\", true, \"NOT NULL lat\"]", + "[\"constraint\", \"client_receptions\", \"client_receptions_lon_not_null\", true, \"NOT NULL lon\"]", + "[\"constraint\", \"client_receptions\", \"client_receptions_pkey\", true, \"PRIMARY KEY (id)\"]", + "[\"constraint\", \"client_receptions\", \"client_receptions_rx_at_not_null\", true, \"NOT NULL rx_at\"]", + "[\"constraint\", \"client_receptions\", \"client_receptions_rx_pubkey_heard_key_rx_at_key\", true, \"UNIQUE (rx_pubkey, heard_key, rx_at)\"]", + "[\"constraint\", \"client_receptions\", \"client_receptions_rx_pubkey_not_null\", true, \"NOT NULL rx_pubkey\"]", + "[\"constraint\", \"client_receptions\", \"client_receptions_src_not_null\", true, \"NOT NULL src\"]", + "[\"constraint\", \"client_rf_samples\", \"client_rf_samples_id_not_null\", true, \"NOT NULL id\"]", + "[\"constraint\", \"client_rf_samples\", \"client_rf_samples_ingested_at_not_null\", true, \"NOT NULL ingested_at\"]", + "[\"constraint\", \"client_rf_samples\", \"client_rf_samples_lat_not_null\", true, \"NOT NULL lat\"]", + "[\"constraint\", \"client_rf_samples\", \"client_rf_samples_lon_not_null\", true, \"NOT NULL lon\"]", + "[\"constraint\", \"client_rf_samples\", \"client_rf_samples_pkey\", true, \"PRIMARY KEY (id)\"]", + "[\"constraint\", \"client_rf_samples\", \"client_rf_samples_rx_pubkey_not_null\", true, \"NOT NULL rx_pubkey\"]", + "[\"constraint\", \"client_rf_samples\", \"client_rf_samples_rx_pubkey_sampled_at_key\", true, \"UNIQUE (rx_pubkey, sampled_at)\"]", + "[\"constraint\", \"client_rf_samples\", \"client_rf_samples_sampled_at_not_null\", true, \"NOT NULL sampled_at\"]", + "[\"constraint\", \"client_rf_samples\", \"client_rf_samples_stationary_not_null\", true, \"NOT NULL stationary\"]", + "[\"constraint\", \"client_rf_samples\", \"client_rf_samples_uptime_secs_not_null\", true, \"NOT NULL uptime_secs\"]", + "[\"constraint\", \"client_rx_observations\", \"client_rx_observations_hash_size_not_null\", true, \"NOT NULL hash_size\"]", + "[\"constraint\", \"client_rx_observations\", \"client_rx_observations_hop_count_not_null\", true, \"NOT NULL hop_count\"]", + "[\"constraint\", \"client_rx_observations\", \"client_rx_observations_id_not_null\", true, \"NOT NULL id\"]", + "[\"constraint\", \"client_rx_observations\", \"client_rx_observations_ingested_at_not_null\", true, \"NOT NULL ingested_at\"]", + "[\"constraint\", \"client_rx_observations\", \"client_rx_observations_lat_not_null\", true, \"NOT NULL lat\"]", + "[\"constraint\", \"client_rx_observations\", \"client_rx_observations_lon_not_null\", true, \"NOT NULL lon\"]", + "[\"constraint\", \"client_rx_observations\", \"client_rx_observations_payload_type_not_null\", true, \"NOT NULL payload_type\"]", + "[\"constraint\", \"client_rx_observations\", \"client_rx_observations_pkey\", true, \"PRIMARY KEY (id)\"]", + "[\"constraint\", \"client_rx_observations\", \"client_rx_observations_pkt_hash_not_null\", true, \"NOT NULL pkt_hash\"]", + "[\"constraint\", \"client_rx_observations\", \"client_rx_observations_route_type_not_null\", true, \"NOT NULL route_type\"]", + "[\"constraint\", \"client_rx_observations\", \"client_rx_observations_rx_at_not_null\", true, \"NOT NULL rx_at\"]", + "[\"constraint\", \"client_rx_observations\", \"client_rx_observations_rx_pubkey_not_null\", true, \"NOT NULL rx_pubkey\"]", + "[\"constraint\", \"client_rx_observations\", \"client_rx_observations_rx_pubkey_pkt_hash_rx_at_key\", true, \"UNIQUE (rx_pubkey, pkt_hash, rx_at)\"]", + "[\"constraint\", \"dropped_packets\", \"dropped_packets_id_not_null\", true, \"NOT NULL id\"]", + "[\"constraint\", \"dropped_packets\", \"dropped_packets_pkey\", true, \"PRIMARY KEY (id)\"]", + "[\"constraint\", \"dropped_packets\", \"dropped_packets_reason_not_null\", true, \"NOT NULL reason\"]", + "[\"constraint\", \"inactive_nodes\", \"inactive_nodes_multibyte_sup_not_null\", true, \"NOT NULL multibyte_sup\"]", + "[\"constraint\", \"inactive_nodes\", \"inactive_nodes_pkey\", true, \"PRIMARY KEY (public_key)\"]", + "[\"constraint\", \"inactive_nodes\", \"inactive_nodes_public_key_not_null\", true, \"NOT NULL public_key\"]", + "[\"constraint\", \"neighbor_edges\", \"neighbor_edges_node_a_not_null\", true, \"NOT NULL node_a\"]", + "[\"constraint\", \"neighbor_edges\", \"neighbor_edges_node_b_not_null\", true, \"NOT NULL node_b\"]", + "[\"constraint\", \"neighbor_edges\", \"neighbor_edges_pkey\", true, \"PRIMARY KEY (node_a, node_b)\"]", + "[\"constraint\", \"node_declared_regions\", \"node_declared_regions_id_not_null\", true, \"NOT NULL id\"]", + "[\"constraint\", \"node_declared_regions\", \"node_declared_regions_ingested_at_not_null\", true, \"NOT NULL ingested_at\"]", + "[\"constraint\", \"node_declared_regions\", \"node_declared_regions_observed_at_not_null\", true, \"NOT NULL observed_at\"]", + "[\"constraint\", \"node_declared_regions\", \"node_declared_regions_pkey\", true, \"PRIMARY KEY (id)\"]", + "[\"constraint\", \"node_declared_regions\", \"node_declared_regions_regions_csv_not_null\", true, \"NOT NULL regions_csv\"]", + "[\"constraint\", \"node_declared_regions\", \"node_declared_regions_rx_pubkey_not_null\", true, \"NOT NULL rx_pubkey\"]", + "[\"constraint\", \"node_declared_regions\", \"node_declared_regions_target_not_null\", true, \"NOT NULL target\"]", + "[\"constraint\", \"node_declared_regions\", \"node_declared_regions_target_rx_pubkey_observed_at_key\", true, \"UNIQUE (target, rx_pubkey, observed_at)\"]", + "[\"constraint\", \"node_declared_regions\", \"node_declared_regions_truncated_not_null\", true, \"NOT NULL truncated\"]", + "[\"constraint\", \"nodes\", \"nodes_multibyte_sup_not_null\", true, \"NOT NULL multibyte_sup\"]", + "[\"constraint\", \"nodes\", \"nodes_pkey\", true, \"PRIMARY KEY (public_key)\"]", + "[\"constraint\", \"nodes\", \"nodes_public_key_not_null\", true, \"NOT NULL public_key\"]", + "[\"constraint\", \"observations\", \"observations_id_not_null\", true, \"NOT NULL id\"]", + "[\"constraint\", \"observations\", \"observations_pkey\", true, \"PRIMARY KEY (id)\"]", + "[\"constraint\", \"observations\", \"observations_timestamp_not_null\", true, \"NOT NULL \\\"timestamp\\\"\"]", + "[\"constraint\", \"observations\", \"observations_transmission_id_fkey\", true, \"FOREIGN KEY (transmission_id) REFERENCES transmissions(id)\"]", + "[\"constraint\", \"observations\", \"observations_transmission_id_not_null\", true, \"NOT NULL transmission_id\"]", + "[\"constraint\", \"observer_metrics\", \"observer_metrics_observer_id_not_null\", true, \"NOT NULL observer_id\"]", + "[\"constraint\", \"observer_metrics\", \"observer_metrics_pkey\", true, \"PRIMARY KEY (observer_id, \\\"timestamp\\\")\"]", + "[\"constraint\", \"observer_metrics\", \"observer_metrics_timestamp_not_null\", true, \"NOT NULL \\\"timestamp\\\"\"]", + "[\"constraint\", \"observers\", \"observers_id_not_null\", true, \"NOT NULL id\"]", + "[\"constraint\", \"observers\", \"observers_pkey\", true, \"PRIMARY KEY (id)\"]", + "[\"constraint\", \"observers\", \"observers_rowid_key\", true, \"UNIQUE (rowid)\"]", + "[\"constraint\", \"observers\", \"observers_rowid_not_null\", true, \"NOT NULL rowid\"]", + "[\"constraint\", \"scope_match_totals\", \"scope_match_totals_ambiguous_not_null\", true, \"NOT NULL ambiguous\"]", + "[\"constraint\", \"scope_match_totals\", \"scope_match_totals_explicit_over_derived_not_null\", true, \"NOT NULL explicit_over_derived\"]", + "[\"constraint\", \"scope_match_totals\", \"scope_match_totals_id_check\", true, \"CHECK ((id = 1))\"]", + "[\"constraint\", \"scope_match_totals\", \"scope_match_totals_id_not_null\", true, \"NOT NULL id\"]", + "[\"constraint\", \"scope_match_totals\", \"scope_match_totals_none_matches_not_null\", true, \"NOT NULL none_matches\"]", + "[\"constraint\", \"scope_match_totals\", \"scope_match_totals_pkey\", true, \"PRIMARY KEY (id)\"]", + "[\"constraint\", \"scope_match_totals\", \"scope_match_totals_since_unix_not_null\", true, \"NOT NULL since_unix\"]", + "[\"constraint\", \"scope_match_totals\", \"scope_match_totals_unique_matches_not_null\", true, \"NOT NULL unique_matches\"]", + "[\"constraint\", \"scope_match_totals\", \"scope_match_totals_updated_unix_not_null\", true, \"NOT NULL updated_unix\"]", + "[\"constraint\", \"transmissions\", \"transmissions_first_seen_not_null\", true, \"NOT NULL first_seen\"]", + "[\"constraint\", \"transmissions\", \"transmissions_hash_key\", true, \"UNIQUE (hash)\"]", + "[\"constraint\", \"transmissions\", \"transmissions_hash_not_null\", true, \"NOT NULL hash\"]", + "[\"constraint\", \"transmissions\", \"transmissions_id_not_null\", true, \"NOT NULL id\"]", + "[\"constraint\", \"transmissions\", \"transmissions_last_seen_not_null\", true, \"NOT NULL last_seen\"]", + "[\"constraint\", \"transmissions\", \"transmissions_pkey\", true, \"PRIMARY KEY (id)\"]", + "[\"constraint\", \"transmissions\", \"transmissions_raw_hex_not_null\", true, \"NOT NULL raw_hex\"]", + "[\"index\", \"_async_migrations\", \"CREATE UNIQUE INDEX _async_migrations_pkey ON {schema}._async_migrations USING btree (name)\"]", + "[\"index\", \"_migrations\", \"CREATE UNIQUE INDEX _migrations_pkey ON {schema}._migrations USING btree (name)\"]", + "[\"index\", \"advert_evidence_backfill\", \"CREATE UNIQUE INDEX advert_evidence_backfill_pkey ON {schema}.advert_evidence_backfill USING btree (id)\"]", + "[\"index\", \"advert_route_evidence\", \"CREATE UNIQUE INDEX advert_route_evidence_pkey ON {schema}.advert_route_evidence USING btree (id)\"]", + "[\"index\", \"advert_route_evidence\", \"CREATE UNIQUE INDEX advert_route_evidence_tx_id_bit_key ON {schema}.advert_route_evidence USING btree (tx_id, \\\"bit\\\")\"]", + "[\"index\", \"client_observers\", \"CREATE UNIQUE INDEX client_observers_pkey ON {schema}.client_observers USING btree (pubkey)\"]", + "[\"index\", \"client_receptions\", \"CREATE INDEX idx_client_recept_heard_geo ON {schema}.client_receptions USING btree (heard_key, heard_keylen, lat, lon)\"]", + "[\"index\", \"client_receptions\", \"CREATE INDEX idx_client_recept_latlon ON {schema}.client_receptions USING btree (lat, lon)\"]", + "[\"index\", \"client_receptions\", \"CREATE INDEX idx_client_recept_rxat ON {schema}.client_receptions USING btree (rx_at)\"]", + "[\"index\", \"client_receptions\", \"CREATE UNIQUE INDEX client_receptions_pkey ON {schema}.client_receptions USING btree (id)\"]", + "[\"index\", \"client_receptions\", \"CREATE UNIQUE INDEX client_receptions_rx_pubkey_heard_key_rx_at_key ON {schema}.client_receptions USING btree (rx_pubkey, heard_key, rx_at)\"]", + "[\"index\", \"client_rf_samples\", \"CREATE INDEX idx_crf_prune ON {schema}.client_rf_samples USING btree (sampled_at)\"]", + "[\"index\", \"client_rf_samples\", \"CREATE INDEX idx_crf_track ON {schema}.client_rf_samples USING btree (rx_pubkey, sampled_at)\"]", + "[\"index\", \"client_rf_samples\", \"CREATE UNIQUE INDEX client_rf_samples_pkey ON {schema}.client_rf_samples USING btree (id)\"]", + "[\"index\", \"client_rf_samples\", \"CREATE UNIQUE INDEX client_rf_samples_rx_pubkey_sampled_at_key ON {schema}.client_rf_samples USING btree (rx_pubkey, sampled_at)\"]", + "[\"index\", \"client_rx_observations\", \"CREATE INDEX idx_cro_forwarder ON {schema}.client_rx_observations USING btree (forwarder, rx_at)\"]", + "[\"index\", \"client_rx_observations\", \"CREATE INDEX idx_cro_hash ON {schema}.client_rx_observations USING btree (pkt_hash, rx_at)\"]", + "[\"index\", \"client_rx_observations\", \"CREATE INDEX idx_cro_prune ON {schema}.client_rx_observations USING btree (rx_at)\"]", + "[\"index\", \"client_rx_observations\", \"CREATE INDEX idx_cro_scope ON {schema}.client_rx_observations USING btree (scope_name, rx_at)\"]", + "[\"index\", \"client_rx_observations\", \"CREATE UNIQUE INDEX client_rx_observations_pkey ON {schema}.client_rx_observations USING btree (id)\"]", + "[\"index\", \"client_rx_observations\", \"CREATE UNIQUE INDEX client_rx_observations_rx_pubkey_pkt_hash_rx_at_key ON {schema}.client_rx_observations USING btree (rx_pubkey, pkt_hash, rx_at)\"]", + "[\"index\", \"dropped_packets\", \"CREATE INDEX idx_dropped_node ON {schema}.dropped_packets USING btree (node_pubkey)\"]", + "[\"index\", \"dropped_packets\", \"CREATE INDEX idx_dropped_observer ON {schema}.dropped_packets USING btree (observer_id)\"]", + "[\"index\", \"dropped_packets\", \"CREATE UNIQUE INDEX dropped_packets_pkey ON {schema}.dropped_packets USING btree (id)\"]", + "[\"index\", \"inactive_nodes\", \"CREATE INDEX idx_inactive_nodes_last_seen ON {schema}.inactive_nodes USING btree (last_seen)\"]", + "[\"index\", \"inactive_nodes\", \"CREATE UNIQUE INDEX inactive_nodes_pkey ON {schema}.inactive_nodes USING btree (public_key)\"]", + "[\"index\", \"neighbor_edges\", \"CREATE UNIQUE INDEX neighbor_edges_pkey ON {schema}.neighbor_edges USING btree (node_a, node_b)\"]", + "[\"index\", \"node_declared_regions\", \"CREATE INDEX idx_ndr_prune ON {schema}.node_declared_regions USING btree (observed_at)\"]", + "[\"index\", \"node_declared_regions\", \"CREATE INDEX idx_ndr_target ON {schema}.node_declared_regions USING btree (target, observed_at)\"]", + "[\"index\", \"node_declared_regions\", \"CREATE UNIQUE INDEX node_declared_regions_pkey ON {schema}.node_declared_regions USING btree (id)\"]", + "[\"index\", \"node_declared_regions\", \"CREATE UNIQUE INDEX node_declared_regions_target_rx_pubkey_observed_at_key ON {schema}.node_declared_regions USING btree (target, rx_pubkey, observed_at)\"]", + "[\"index\", \"nodes\", \"CREATE INDEX idx_nodes_foreign_advert ON {schema}.nodes USING btree (foreign_advert) WHERE (foreign_advert = 1)\"]", + "[\"index\", \"nodes\", \"CREATE INDEX idx_nodes_last_seen ON {schema}.nodes USING btree (last_seen)\"]", + "[\"index\", \"nodes\", \"CREATE UNIQUE INDEX nodes_pkey ON {schema}.nodes USING btree (public_key)\"]", + "[\"index\", \"observations\", \"CREATE INDEX idx_observations_observer_idx ON {schema}.observations USING btree (observer_idx)\"]", + "[\"index\", \"observations\", \"CREATE INDEX idx_observations_observer_idx_timestamp ON {schema}.observations USING btree (observer_idx, \\\"timestamp\\\")\"]", + "[\"index\", \"observations\", \"CREATE INDEX idx_observations_timestamp ON {schema}.observations USING btree (\\\"timestamp\\\")\"]", + "[\"index\", \"observations\", \"CREATE INDEX idx_observations_transmission_id ON {schema}.observations USING btree (transmission_id)\"]", + "[\"index\", \"observations\", \"CREATE INDEX idx_observations_tx_ts ON {schema}.observations USING btree (transmission_id, \\\"timestamp\\\")\"]", + "[\"index\", \"observations\", \"CREATE UNIQUE INDEX idx_observations_dedup ON {schema}.observations USING btree (transmission_id, observer_idx, COALESCE(path_json, ''::text))\"]", + "[\"index\", \"observations\", \"CREATE UNIQUE INDEX observations_pkey ON {schema}.observations USING btree (id)\"]", + "[\"index\", \"observer_metrics\", \"CREATE INDEX idx_observer_metrics_timestamp ON {schema}.observer_metrics USING btree (\\\"timestamp\\\")\"]", + "[\"index\", \"observer_metrics\", \"CREATE UNIQUE INDEX observer_metrics_pkey ON {schema}.observer_metrics USING btree (observer_id, \\\"timestamp\\\")\"]", + "[\"index\", \"observers\", \"CREATE INDEX idx_observers_iata_norm ON {schema}.observers USING btree (upper(TRIM(BOTH FROM iata)))\"]", + "[\"index\", \"observers\", \"CREATE INDEX idx_observers_last_seen ON {schema}.observers USING btree (last_seen)\"]", + "[\"index\", \"observers\", \"CREATE UNIQUE INDEX observers_pkey ON {schema}.observers USING btree (id)\"]", + "[\"index\", \"observers\", \"CREATE UNIQUE INDEX observers_rowid_key ON {schema}.observers USING btree (rowid)\"]", + "[\"index\", \"scope_match_totals\", \"CREATE UNIQUE INDEX scope_match_totals_pkey ON {schema}.scope_match_totals USING btree (id)\"]", + "[\"index\", \"transmissions\", \"CREATE INDEX idx_transmissions_channel_hash_payload ON {schema}.transmissions USING btree (channel_hash, payload_type, first_seen)\"]", + "[\"index\", \"transmissions\", \"CREATE INDEX idx_transmissions_first_seen ON {schema}.transmissions USING btree (first_seen, id)\"]", + "[\"index\", \"transmissions\", \"CREATE INDEX idx_transmissions_from_pubkey ON {schema}.transmissions USING btree (from_pubkey)\"]", + "[\"index\", \"transmissions\", \"CREATE INDEX idx_transmissions_hash ON {schema}.transmissions USING btree (hash)\"]", + "[\"index\", \"transmissions\", \"CREATE INDEX idx_transmissions_payload_type ON {schema}.transmissions USING btree (payload_type)\"]", + "[\"index\", \"transmissions\", \"CREATE INDEX idx_tx_channel_hash ON {schema}.transmissions USING btree (channel_hash) WHERE (payload_type = 5)\"]", + "[\"index\", \"transmissions\", \"CREATE INDEX idx_tx_code1 ON {schema}.transmissions USING btree (code1) WHERE (code1 IS NOT NULL)\"]", + "[\"index\", \"transmissions\", \"CREATE INDEX idx_tx_last_seen_zero ON {schema}.transmissions USING btree (id) WHERE (last_seen = 0)\"]", + "[\"index\", \"transmissions\", \"CREATE INDEX idx_tx_scope_name ON {schema}.transmissions USING btree (scope_name) WHERE (scope_name IS NOT NULL)\"]", + "[\"index\", \"transmissions\", \"CREATE UNIQUE INDEX transmissions_hash_key ON {schema}.transmissions USING btree (hash)\"]", + "[\"index\", \"transmissions\", \"CREATE UNIQUE INDEX transmissions_pkey ON {schema}.transmissions USING btree (id)\"]", + "[\"table\", \"_async_migrations\", \"r\", false, false]", + "[\"table\", \"_migrations\", \"r\", false, false]", + "[\"table\", \"advert_evidence_backfill\", \"r\", false, false]", + "[\"table\", \"advert_route_evidence\", \"r\", false, false]", + "[\"table\", \"client_observers\", \"r\", false, false]", + "[\"table\", \"client_receptions\", \"r\", false, false]", + "[\"table\", \"client_rf_samples\", \"r\", false, false]", + "[\"table\", \"client_rx_observations\", \"r\", false, false]", + "[\"table\", \"dropped_packets\", \"r\", false, false]", + "[\"table\", \"inactive_nodes\", \"r\", false, false]", + "[\"table\", \"neighbor_edges\", \"r\", false, false]", + "[\"table\", \"node_declared_regions\", \"r\", false, false]", + "[\"table\", \"nodes\", \"r\", false, false]", + "[\"table\", \"observations\", \"r\", false, false]", + "[\"table\", \"observer_metrics\", \"r\", false, false]", + "[\"table\", \"observers\", \"r\", false, false]", + "[\"table\", \"scope_match_totals\", \"r\", false, false]", + "[\"table\", \"transmissions\", \"r\", false, false]", + "[\"view\", \"packets_v\", \" SELECT o.id,\\n COALESCE(o.raw_hex, t.raw_hex) AS raw_hex,\\n to_char((to_timestamp((o.\\\"timestamp\\\")::double precision) AT TIME ZONE 'UTC'::text), 'YYYY-MM-DD HH24:MI:SS'::text) AS \\\"timestamp\\\",\\n obs.id AS observer_id,\\n obs.name AS observer_name,\\n o.direction,\\n o.snr,\\n o.rssi,\\n o.score,\\n t.hash,\\n t.route_type,\\n t.payload_type,\\n t.payload_version,\\n o.path_json,\\n t.decoded_json,\\n t.created_at\\n FROM ((observations o\\n JOIN transmissions t ON ((t.id = o.transmission_id)))\\n LEFT JOIN observers obs ON (((obs.rowid = o.observer_idx) AND ((obs.inactive IS NULL) OR (obs.inactive = 0)))));\"]" + ] +} diff --git a/cmd/migrate/postgres_layout_test.go b/cmd/migrate/postgres_layout_test.go new file mode 100644 index 000000000..ebb536765 --- /dev/null +++ b/cmd/migrate/postgres_layout_test.go @@ -0,0 +1,83 @@ +package main + +import ( + "encoding/json" + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/users" + "os" + "testing" +) + +func TestPostgresConversionLayoutMatchesNativeSchema(t *testing.T) { + for _, kind := range []string{"telemetry", "accounts"} { + t.Run(kind, func(t *testing.T) { + db := openImportDB(t, postgresSchema(t)) + var err error + if kind == "telemetry" { + err = dbschema.ApplyPostgres(db, nil) + } else { + err = users.ApplyPostgres(db) + } + if err != nil { + t.Fatal(err) + } + tables, err := layouts(kind) + if err != nil { + t.Fatal(err) + } + if err := validatePostgresLayout(db, kind, tables); err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`CREATE INDEX unexpected_extension ON ` + quote(tables[0].Name) + ` (` + quote(tables[0].Columns[0].Name) + `)`); err != nil { + t.Fatal(err) + } + if err := validatePostgresLayout(db, kind, tables); err == nil { + t.Fatal("custom source extension would be discarded") + } + }) + } +} + +// This explicit developer mode exports a prospective layout for review. It +// never changes the compiled manifest or skips its normal compatibility test. +func TestExportPostgresConversionLayout(t *testing.T) { + path := os.Getenv("CORESCOPE_EXPORT_POSTGRES_LAYOUT") + if path == "" { + t.Skip("explicit private layout export only") + } + output := map[string][]string{} + for _, kind := range []string{"telemetry", "accounts"} { + db := openImportDB(t, postgresSchema(t)) + var err error + if kind == "telemetry" { + err = dbschema.ApplyPostgres(db, nil) + } else { + err = users.ApplyPostgres(db) + } + if err != nil { + t.Fatal(err) + } + tables, err := layouts(kind) + if err != nil { + t.Fatal(err) + } + if output[kind], err = postgresLayout(db, kind, tables); err != nil { + t.Fatal(err) + } + } + data, err := json.MarshalIndent(output, "", " ") + if err != nil { + t.Fatal(err) + } + file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + if err != nil { + t.Fatal(err) + } + if _, err = file.Write(append(data, '\n')); err != nil { + file.Close() + t.Fatal(err) + } + if err := file.Close(); err != nil { + t.Fatal(err) + } +} diff --git a/cmd/migrate/resume_finalized_test.go b/cmd/migrate/resume_finalized_test.go new file mode 100644 index 000000000..6bb8eaaae --- /dev/null +++ b/cmd/migrate/resume_finalized_test.go @@ -0,0 +1,358 @@ +package main + +import ( + "bytes" + "context" + "database/sql" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/users" +) + +// Include tuple versions and sequence state so a refusal or a verified ready +// store cannot conceal rewrites of identical values, progress, or readiness. +func importDatabaseState(t *testing.T, db *sql.DB, kind string) string { + t.Helper() + tables, err := layouts(kind) + if err != nil { + t.Fatal(err) + } + var state strings.Builder + for _, table := range append(tables, importTable{Name: "corescope_import"}, importTable{Name: "corescope_import_progress"}, importTable{Name: "corescope_schema"}) { + rows, err := db.Query(`SELECT row_to_json(t)::text,t.xmin::text,t.ctid::text FROM ` + quote(table.Name) + ` t ORDER BY row_to_json(t)::text COLLATE "C"`) + if err != nil { + t.Fatal(err) + } + for rows.Next() { + var values, version, position string + if err := rows.Scan(&values, &version, &position); err != nil { + t.Fatal(err) + } + fmt.Fprintln(&state, table.Name, values, version, position) + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + rows.Close() + for _, column := range table.Columns { + if !column.Identity { + continue + } + var sequence string + if err := db.QueryRow(`SELECT pg_get_serial_sequence($1,$2)`, table.Name, column.Name).Scan(&sequence); err != nil { + t.Fatal(err) + } + var value int64 + var called bool + if err := db.QueryRow(`SELECT last_value,is_called FROM `+sequence).Scan(&value, &called); err != nil { + t.Fatal(err) + } + fmt.Fprintln(&state, sequence, value, called) + } + } + return state.String() +} + +func mixedImport(t *testing.T, startAccounts, readyTelemetry bool) [2]importOptions { + t.Helper() + state := filepath.Join(t.TempDir(), "migration state") + options := [2]importOptions{ + {Source: telemetrySource(t), DatabaseURL: postgresDatabase(t), StateDir: state, Kind: "telemetry"}, + {Source: accountSource(t, 5), DatabaseURL: postgresDatabase(t), StateDir: state, Kind: "accounts"}, + } + for i, o := range options { + if i == 1 && !startAccounts { + continue + } + if _, err := importSQLite(context.Background(), o); err != nil { + t.Fatal(err) + } + } + if readyTelemetry { + if _, err := finalizeImport(context.Background(), options[0].DatabaseURL, "telemetry"); err != nil { + t.Fatal(err) + } + } + t.Setenv("CORESCOPE_DATABASE_URL", options[0].DatabaseURL) + t.Setenv("CORESCOPE_USERS_DATABASE_URL", options[1].DatabaseURL) + return options +} + +func resumeMixedCommand(options [2]importOptions, out io.Writer) error { + return runCommand(context.Background(), []string{"-offline", "-resume", "-from-sqlite", options[0].Source, "-users-from-sqlite", options[1].Source, "-state-dir", options[0].StateDir}, out) +} + +func TestCLIResumeAfterFirstStoreFinalized(t *testing.T) { + options := mixedImport(t, true, true) + telemetry := openImportDB(t, options[0].DatabaseURL) + accounts := openImportDB(t, options[1].DatabaseURL) + before := importDatabaseState(t, telemetry, "telemetry") + beforeSources := make(map[string]string) + for _, o := range options { + for _, path := range []string{o.Source, filepath.Join(o.StateDir, o.Kind, "recovery.sqlite"), filepath.Join(o.StateDir, o.Kind, "normalized.sqlite")} { + digest, err := fingerprint(context.Background(), path) + if err != nil { + t.Fatal(err) + } + beforeSources[path] = digest + } + } + if err := users.AssertReady(accounts); err == nil { + t.Fatal("fixture must reproduce the crash between the two readiness commits") + } + var output bytes.Buffer + if err := resumeMixedCommand(options, &output); err != nil { + t.Fatalf("resume after telemetry finalized but accounts unready: %v", err) + } + if err := dbschema.AssertReady(telemetry); err != nil { + t.Fatal(err) + } + if err := users.AssertReady(accounts); err != nil { + t.Fatal(err) + } + if got := importDatabaseState(t, telemetry, "telemetry"); got != before { + t.Fatal("resume wrote already-ready telemetry data, progress, sequence, or readiness") + } + for _, o := range options { + if !strings.Contains(output.String(), o.Kind+" import verified and finalized") || strings.Contains(output.String(), o.DatabaseURL) { + t.Fatal("missing success evidence or leaked database URL") + } + } + for path, before := range beforeSources { + after, err := fingerprint(context.Background(), path) + if err != nil || after != before { + t.Fatal("resume changed original source or immutable snapshot", err) + } + } +} + +type importOutputFunc func([]byte) (int, error) + +func (f importOutputFunc) Write(data []byte) (int, error) { return f(data) } + +func TestCLIResumeFailureDoesNotResetAlreadyReadyStore(t *testing.T) { + options := mixedImport(t, true, true) + telemetry := openImportDB(t, options[0].DatabaseURL) + accounts := openImportDB(t, options[1].DatabaseURL) + before := importDatabaseState(t, telemetry, "telemetry") + // Fail the second finalization after both imports have been verified, so + // the actual CLI rollback path must distinguish previously ready stores. + out := importOutputFunc(func(data []byte) (int, error) { + if bytes.Contains(data, []byte(`"kind":"accounts"`)) { + if _, err := accounts.Exec(`UPDATE users SET display_name='post-verification change'`); err != nil { + return 0, err + } + } + return len(data), nil + }) + if err := resumeMixedCommand(options, out); err == nil || !strings.Contains(err.Error(), "final verification failed") { + t.Fatalf("second finalization corruption should be refused: %v", err) + } + if got := importDatabaseState(t, telemetry, "telemetry"); got != before { + t.Fatal("failed second finalization reset or rewrote the previously ready store") + } + if err := users.AssertReady(accounts); err == nil { + t.Fatal("changed accounts became ready") + } +} + +func TestCLIRollbackDoesNotCloseGateOpenedByAnotherFinalizer(t *testing.T) { + options := mixedImport(t, true, false) + telemetry := openImportDB(t, options[0].DatabaseURL) + accounts := openImportDB(t, options[1].DatabaseURL) + var finalizedState string + out := importOutputFunc(func(data []byte) (int, error) { + if bytes.Contains(data, []byte(`"kind":"accounts"`)) { + // The importer observed telemetry unready. A separate allowed + // finalizer opens it before this CLI reaches its finalization loop. + if _, err := finalizeImport(context.Background(), options[0].DatabaseURL, "telemetry"); err != nil { + return 0, err + } + finalizedState = importDatabaseState(t, telemetry, "telemetry") + if _, err := accounts.Exec(`UPDATE users SET display_name='fail second finalization'`); err != nil { + return 0, err + } + } + return len(data), nil + }) + if err := resumeMixedCommand(options, out); err == nil || !strings.Contains(err.Error(), "final verification failed") { + t.Fatalf("second finalization should fail after the separate finalizer ran: %v", err) + } + if finalizedState == "" { + t.Fatal("fixture did not execute the intervening finalization") + } + if err := dbschema.AssertReady(telemetry); err != nil { + t.Error("CLI reset a gate it did not open", err) + } + if got := importDatabaseState(t, telemetry, "telemetry"); got != finalizedState { + t.Error("CLI changed tuples or readiness owned by the separate finalizer") + } + if err := users.AssertReady(accounts); err == nil { + t.Fatal("corrupted accounts became ready") + } +} + +func TestCLIRollbackClosesGateOpenedByThisInvocation(t *testing.T) { + options := mixedImport(t, true, false) + telemetry := openImportDB(t, options[0].DatabaseURL) + accounts := openImportDB(t, options[1].DatabaseURL) + out := importOutputFunc(func(data []byte) (int, error) { + if bytes.Contains(data, []byte(`"kind":"accounts"`)) { + if _, err := accounts.Exec(`UPDATE users SET display_name='fail second finalization'`); err != nil { + return 0, err + } + } + return len(data), nil + }) + if err := resumeMixedCommand(options, out); err == nil || !strings.Contains(err.Error(), "final verification failed") { + t.Fatalf("second finalization should fail: %v", err) + } + if err := dbschema.AssertReady(telemetry); err == nil { + t.Fatal("CLI did not close the telemetry gate it opened before account failure") + } + if err := users.AssertReady(accounts); err == nil { + t.Fatal("corrupted accounts became ready") + } +} + +func TestCLIResumeDoesNotStartOccupiedSecondStore(t *testing.T) { + options := mixedImport(t, false, true) + telemetry := openImportDB(t, options[0].DatabaseURL) + accounts := openImportDB(t, options[1].DatabaseURL) + if _, err := accounts.Exec(`CREATE SEQUENCE keep_me START WITH 77`); err != nil { + t.Fatal(err) + } + before := importDatabaseState(t, telemetry, "telemetry") + if err := resumeMixedCommand(options, io.Discard); err == nil || !strings.Contains(err.Error(), "not empty") { + t.Fatalf("resume accepted an occupied unmarked destination: %v", err) + } + if got := importDatabaseState(t, telemetry, "telemetry"); got != before { + t.Fatal("refusal changed already-ready telemetry") + } + var value int + var called bool + if err := accounts.QueryRow(`SELECT last_value,is_called FROM keep_me`).Scan(&value, &called); err != nil || value != 77 || called { + t.Fatal("refusal changed occupied account destination", err) + } + if _, err := os.Stat(filepath.Join(options[1].StateDir, "accounts")); !os.IsNotExist(err) { + t.Fatal("occupied destination should be refused before source preparation") + } +} + +func TestCLIResumeStartsUntouchedSecondStore(t *testing.T) { + options := mixedImport(t, false, false) + if err := resumeMixedCommand(options, io.Discard); err != nil { + t.Fatalf("resume must finish the verified first store and start untouched accounts: %v", err) + } + if err := dbschema.AssertReady(openImportDB(t, options[0].DatabaseURL)); err != nil { + t.Fatal(err) + } + if err := users.AssertReady(openImportDB(t, options[1].DatabaseURL)); err != nil { + t.Fatal(err) + } +} + +func TestResumePreparationArtifactsArePreserved(t *testing.T) { + o := importOptions{Source: telemetrySource(t), DatabaseURL: postgresSchema(t), StateDir: t.TempDir(), Kind: "telemetry", Resume: true} + recovery := filepath.Join(o.StateDir, o.Kind, "recovery.sqlite") + if err := os.MkdirAll(filepath.Dir(recovery), 0o700); err != nil { + t.Fatal(err) + } + if err := snapshotSource(context.Background(), o.Source, recovery); err != nil { + t.Fatal(err) + } + before, err := os.ReadFile(recovery) + if err != nil { + t.Fatal(err) + } + _, err = importSQLite(context.Background(), o) + if err == nil || !strings.Contains(err.Error(), "preserve") || !strings.Contains(err.Error(), "state directory") { + t.Fatalf("interrupted preparation needs preserving recovery guidance: %v", err) + } + after, err := os.ReadFile(recovery) + if err != nil || !bytes.Equal(before, after) { + t.Fatal("refusal changed the only recovery snapshot", err) + } + var n int + if err := openImportDB(t, o.DatabaseURL).QueryRow(`SELECT count(*) FROM information_schema.tables WHERE table_schema=current_schema()`).Scan(&n); err != nil || n != 0 { + t.Fatal("preparation refusal changed the empty destination", err) + } + // Exercise the documented recovery: archive the incomplete per-store state + // without deleting it, then let resume prepare the still-empty store anew. + archived := filepath.Join(t.TempDir(), "preserved incomplete preparation") + if err := os.Rename(filepath.Dir(recovery), archived); err != nil { + t.Fatal(err) + } + if report, err := importSQLite(context.Background(), o); err != nil || !report.Verified { + t.Fatal("resume after preserving incomplete preparation failed", err) + } + retained, err := os.ReadFile(filepath.Join(archived, "recovery.sqlite")) + if err != nil || !bytes.Equal(before, retained) { + t.Fatal("retry changed archived recovery evidence", err) + } +} + +func TestReadyResumeRefusesChangedEvidenceWithoutWrites(t *testing.T) { + changes := map[string]string{ + "new PostgreSQL row": `INSERT INTO observers(id) VALUES('post-cutover')`, + "changed PostgreSQL row": `UPDATE observers SET name='changed' WHERE rowid=0`, + "changed sequence only": `SELECT nextval(pg_get_serial_sequence('observers','rowid'))`, + "incomplete progress": `UPDATE corescope_import_progress SET complete=false WHERE table_name='observers'`, + "changed progress digest": `UPDATE corescope_import_progress SET sha256=repeat('0',64) WHERE table_name='observers'`, + "unverified marker": `UPDATE corescope_import SET verified=false`, + "changed report": `UPDATE corescope_import SET report=replace(report,'"rows":2','"rows":200')`, + "changed run identity": `UPDATE corescope_import SET run_id='different'`, + "source": "", + "recovery snapshot": "", + "normalized snapshot": "", + } + for name, statement := range changes { + t.Run(name, func(t *testing.T) { + o := importOptions{Source: telemetrySource(t), DatabaseURL: postgresSchema(t), StateDir: t.TempDir(), Kind: "telemetry"} + if _, err := importSQLite(context.Background(), o); err != nil { + t.Fatal(err) + } + if _, err := finalizeImport(context.Background(), o.DatabaseURL, o.Kind); err != nil { + t.Fatal(err) + } + db := openImportDB(t, o.DatabaseURL) + if statement != "" { + if _, err := db.Exec(statement); err != nil { + t.Fatal(err) + } + } else { + path := o.Source + if name != "source" { + file := "normalized.sqlite" + if name == "recovery snapshot" { + file = "recovery.sqlite" + } + path = filepath.Join(o.StateDir, o.Kind, file) + } + changed, err := sql.Open("sqlite3", path) + if err != nil { + t.Fatal(err) + } + _, err = changed.Exec(`UPDATE observers SET name='changed' WHERE rowid=0`) + changed.Close() + if err != nil { + t.Fatal(err) + } + } + before := importDatabaseState(t, db, o.Kind) + o.Resume = true + if _, err := importSQLite(context.Background(), o); err == nil { + t.Fatal("changed evidence accepted for an already-ready destination") + } + if after := importDatabaseState(t, db, o.Kind); after != before { + t.Fatal("refusal changed data, progress, sequence, or readiness") + } + }) + } +} diff --git a/cmd/migrate/reverse.go b/cmd/migrate/reverse.go new file mode 100644 index 000000000..5c108b519 --- /dev/null +++ b/cmd/migrate/reverse.go @@ -0,0 +1,817 @@ +package main + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "database/sql" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "math" + "net/url" + "os" + "path/filepath" + "slices" + "strings" + + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/dbschema/legacy" + "github.com/meshcore-analyzer/pgutil" + "github.com/meshcore-analyzer/users" +) + +type reverseOptions struct { + DatabaseURL, Destination, StateDir, Kind string + BatchSize int + Resume bool + keepPending bool + afterBatch func() error + source *postgresFence +} + +type identityState struct { + Table string `json:"table"` + Column string `json:"column"` + LastValue int64 `json:"last_value"` + IsCalled bool `json:"is_called"` + SQLiteFloor int64 `json:"sqlite_floor"` +} + +type reverseManifest struct { + Version int `json:"version"` + RunID string `json:"run_id"` + Kind string `json:"kind"` + Destination string `json:"destination"` + SourceIdentity string `json:"source_identity"` + RecoverySHA256 string `json:"recovery_sha256"` + Source []tableReport `json:"source"` + Identities []identityState `json:"identities"` + AddedSQLiteMigrations []string `json:"added_sqlite_migrations"` +} + +type postgresFence struct { + db *sql.DB + tx *sql.Tx + kind string + tables []importTable + identity string +} + +func (f *postgresFence) Close() { + if f.tx != nil { + f.tx.Rollback() + } + if f.db != nil { + f.db.Close() + } +} + +// Local service leases are only one fence. Native table locks and the ingestor's +// commit-order lock also exclude real database writers during the offline copy. +func fencePostgres(ctx context.Context, dsn, kind string) (_ *postgresFence, err error) { + f := &postgresFence{kind: kind} + if f.tables, err = layouts(kind); err != nil { + return nil, err + } + if f.db, err = pgutil.Open(dsn, false); err != nil { + return nil, err + } + defer func() { + if err != nil { + f.Close() + } + }() + if kind == "telemetry" { + err = dbschema.AssertPostgresReady(f.db) + } else { + err = users.AssertPostgresReady(f.db) + } + if err != nil { + return nil, err + } + f.tx, err = f.db.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelRepeatableRead}) + if err != nil { + return nil, err + } + var locked bool + if err = f.tx.QueryRowContext(ctx, `SELECT pg_try_advisory_xact_lock(hashtextextended(current_schema(),$1))`, dbschema.WriterLockKey).Scan(&locked); err != nil { + return nil, err + } + if !locked { + return nil, errors.New("PostgreSQL source writer is active; stop all writers before conversion") + } + var names []string + for _, table := range f.tables { + names = append(names, quote(table.Name)) + } + if _, err = f.tx.ExecContext(ctx, `LOCK TABLE `+strings.Join(names, ",")+` IN SHARE MODE NOWAIT`); err != nil { + return nil, errors.New("PostgreSQL source is in use; stop all writers before conversion") + } + if err = validatePostgresLayout(f.db, kind, f.tables); err != nil { + return nil, err + } + config, e := pgutil.ParseConfig(dsn) + if e != nil { + return nil, e + } + var database, schema string + var dbOID, schemaOID int64 + if err = f.tx.QueryRowContext(ctx, `SELECT current_database(),current_schema(),(SELECT oid::bigint FROM pg_database WHERE datname=current_database()),(SELECT oid::bigint FROM pg_namespace WHERE nspname=current_schema())`).Scan(&database, &schema, &dbOID, &schemaOID); err != nil { + return nil, err + } + data, _ := json.Marshal([]any{database, schema, dbOID, schemaOID, config.Host, config.Port}) + sum := sha256.Sum256(data) + f.identity = hex.EncodeToString(sum[:]) + return f, nil +} + +func identityReferences(kind string) map[string][][2]string { + if kind == "telemetry" { + return map[string][][2]string{ + "observers": {{"observations", "observer_idx"}}, + "transmissions": {{"observations", "transmission_id"}, {"advert_route_evidence", "tx_id"}, {"advert_evidence_backfill", "tx_cursor"}}, + "observations": {{"advert_evidence_backfill", "obs_cursor"}}, + } + } + return map[string][][2]string{ + "users": {{"audit_log", "actor_user_id"}, {"audit_log", "target_user_id"}, {"users", "activated_by"}, {"sessions", "user_id"}, {"tokens", "user_id"}, {"mail_log", "user_id"}, {"proposals", "proposer_id"}, {"proposals", "reviewer_id"}, {"user_settings", "user_id"}, {"notification_prefs", "user_id"}, {"notification_watches", "user_id"}, {"notification_state", "user_id"}}, + "mail_log": {{"mail_events", "mail_id"}}, + } +} + +func postgresIdentities(ctx context.Context, f *postgresFence) ([]identityState, error) { + var out []identityState + refs := identityReferences(f.kind) + for _, table := range f.tables { + for _, col := range table.Columns { + if !col.Identity { + continue + } + var sequence string + if err := f.db.QueryRowContext(ctx, `SELECT pg_get_serial_sequence($1,$2)`, table.Name, col.Name).Scan(&sequence); err != nil { + return nil, err + } + var increment, min, max int64 + var cycle bool + if err := f.db.QueryRowContext(ctx, `SELECT seqincrement,seqmin,seqmax,seqcycle FROM pg_sequence WHERE seqrelid=$1::regclass`, sequence).Scan(&increment, &min, &max, &cycle); err != nil { + return nil, err + } + if increment != 1 || min != 1 || max != math.MaxInt64 || cycle { + return nil, errors.New("source identity sequence has unsupported custom allocation settings") + } + state := identityState{Table: table.Name, Column: col.Name} + if err := f.db.QueryRowContext(ctx, `SELECT last_value,is_called FROM `+sequence).Scan(&state.LastValue, &state.IsCalled); err != nil { + return nil, err + } + state.SQLiteFloor = state.LastValue + if !state.IsCalled { + state.SQLiteFloor-- + } + if state.SQLiteFloor < 0 { + return nil, errors.New("invalid source identity high water") + } + for _, ref := range append([][2]string{{table.Name, col.Name}}, refs[table.Name]...) { + var high int64 + if err := f.db.QueryRowContext(ctx, `SELECT COALESCE(MAX(`+quote(ref[1])+`),0) FROM `+quote(ref[0])).Scan(&high); err != nil { + return nil, err + } + if high > state.SQLiteFloor { + state.SQLiteFloor = high + } + } + out = append(out, state) + } + } + return out, nil +} + +func digestPostgresSource(ctx context.Context, f *postgresFence) ([]tableReport, error) { + var reports []tableReport + for _, table := range f.tables { + // Bound decompressed text before a large value reaches the Go driver. + sizes := []string{"0"} + for _, col := range table.Columns { + if col.Type == "TEXT" { + sizes = append(sizes, `COALESCE(octet_length(`+quote(col.Name)+`),0)::bigint`) + } else { + sizes = append(sizes, "8") + } + } + var oversized bool + if err := f.db.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM `+quote(table.Name)+` WHERE `+strings.Join(sizes, "+")+`>$1)`, maxImportRowBytes).Scan(&oversized); err != nil { + return nil, err + } + if oversized { + return nil, errors.New("a source row exceeds the 64 MiB migration safety limit") + } + report, err := digestTarget(ctx, f.db, table) + if err != nil { + return nil, err + } + reports = append(reports, report) + } + return reports, nil +} + +func writePrivateJSON(path string, value any) error { + data, err := json.MarshalIndent(value, "", " ") + if err != nil { + return err + } + file, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600) + if err != nil { + return err + } + _, writeErr := file.Write(append(data, '\n')) + syncErr := file.Sync() + closeErr := file.Close() + return errors.Join(writeErr, syncErr, closeErr) +} +func readPrivateJSON(path string, value any) error { + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + d := json.NewDecoder(io.LimitReader(file, 1024*1024)) + d.DisallowUnknownFields() + if err := d.Decode(value); err != nil { + return errors.New("invalid private conversion manifest") + } + if err := d.Decode(new(any)); err != io.EOF { + return errors.New("invalid trailing conversion manifest") + } + return nil +} +func fileDigest(ctx context.Context, path string) (string, error) { + file, err := os.Open(path) + if err != nil { + return "", err + } + defer file.Close() + h := sha256.New() + if _, err := io.CopyBuffer(h, contextReader{ctx, file}, make([]byte, 128*1024)); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} + +func prepareReverse(ctx context.Context, o reverseOptions, f *postgresFence) (reverseManifest, error) { + var manifest reverseManifest + dest, err := filepath.Abs(o.Destination) + if err != nil { + return manifest, err + } + if o.Destination == "" || o.StateDir == "" { + return manifest, errors.New("reverse conversion requires a fresh SQLite target and private recovery directory") + } + dir := filepath.Join(o.StateDir, o.Kind) + path := filepath.Join(dir, "reverse.json") + backup := filepath.Join(dir, "recovery.pg_dump") + source, err := digestPostgresSource(ctx, f) + if err != nil { + return manifest, err + } + identities, err := postgresIdentities(ctx, f) + if err != nil { + return manifest, err + } + err = readPrivateJSON(path, &manifest) + if err == nil { + if !o.Resume || manifest.Version != 1 || manifest.Kind != o.Kind || manifest.Destination != dest || manifest.SourceIdentity != f.identity || !slices.Equal(source, manifest.Source) || !slices.Equal(identities, manifest.Identities) { + return manifest, errors.New("reverse resume source, sequence state or destination differs from the immutable manifest") + } + digest, err := fileDigest(ctx, backup) + if err != nil { + return manifest, err + } + if digest != manifest.RecoverySHA256 { + return manifest, errors.New("native PostgreSQL recovery archive changed; resume refused") + } + return manifest, nil + } + if !errors.Is(err, os.ErrNotExist) { + return manifest, err + } + if o.Resume { + return manifest, errors.New("reverse preparation has no complete manifest; preserve recovery files and abort before choosing a new job") + } + if _, err := os.Lstat(dest); !errors.Is(err, os.ErrNotExist) { + return manifest, errors.New("reverse destination already exists or cannot be inspected; existing files were not changed") + } + entries, err := os.ReadDir(dir) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return manifest, err + } + if len(entries) != 0 { + return manifest, errors.New("reverse recovery directory is not empty") + } + if err := os.MkdirAll(dir, 0700); err != nil { + return manifest, err + } + // Recovery is native and contains the whole selected database. The source + // table fences remain held; removing search_path changes no database target. + u, err := url.Parse(o.DatabaseURL) + if err != nil { + return manifest, errors.New("invalid source database URL") + } + q := u.Query() + q.Del("search_path") + u.RawQuery = q.Encode() + if err := pgutil.Dump(ctx, u.String(), backup); err != nil { + return manifest, err + } + digest, err := fileDigest(ctx, backup) + if err != nil { + return manifest, err + } + var random [16]byte + if _, err := rand.Read(random[:]); err != nil { + return manifest, err + } + manifest = reverseManifest{Version: 1, RunID: hex.EncodeToString(random[:]), Kind: o.Kind, Destination: dest, SourceIdentity: f.identity, RecoverySHA256: digest, Source: source, Identities: identities} + if o.Kind == "telemetry" { + template, err := sql.Open(importSQLiteDriver, ":memory:") + if err != nil { + return manifest, err + } + defer template.Close() + template.SetMaxOpenConns(1) + if err := dbschema.ApplySQLite(template, func(string, ...interface{}) {}); err != nil { + return manifest, err + } + rows, err := template.Query(`SELECT name FROM _migrations ORDER BY name`) + if err != nil { + return manifest, err + } + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + rows.Close() + return manifest, err + } + var n int + if err := f.db.QueryRowContext(ctx, `SELECT count(*) FROM _migrations WHERE name=$1`, name).Scan(&n); err != nil { + rows.Close() + return manifest, err + } + if n == 0 { + manifest.AddedSQLiteMigrations = append(manifest.AddedSQLiteMigrations, name) + } + } + err = rows.Err() + rows.Close() + if err != nil { + return manifest, err + } + } + after, err := postgresIdentities(ctx, f) + if err != nil { + return manifest, err + } + if !slices.Equal(identities, after) { + return manifest, errors.New("source sequence allocation changed during backup; stop every writer") + } + return manifest, writePrivateJSON(path, manifest) +} + +func reversePostgres(ctx context.Context, o reverseOptions) (importReport, error) { + report := importReport{Kind: o.Kind} + if o.BatchSize == 0 { + o.BatchSize = 1000 + } + if o.BatchSize < 1 || o.BatchSize > 10000 { + return report, errors.New("copy batch size must be between 1 and 10000") + } + f := o.source + if f == nil { + var err error + f, err = fencePostgres(ctx, o.DatabaseURL, o.Kind) + if err != nil { + return report, err + } + defer f.Close() + } + manifest, err := prepareReverse(ctx, o, f) + if err != nil { + return report, err + } + destination, verifyOnly, err := openReverseTarget(ctx, o, manifest) + if err != nil { + return report, err + } + defer destination.Close() + for i, table := range f.tables { + got, err := copyReverseTable(ctx, f.db, destination, table, o, manifest, verifyOnly) + if err != nil { + return report, fmt.Errorf("reverse %s: %w", table.Name, err) + } + if got != manifest.Source[i] { + return report, errors.New("reverse source count or logical digest changed") + } + report.Tables = append(report.Tables, got) + } + for _, identity := range manifest.Identities { + var count int + var high int64 + if err := destination.QueryRowContext(ctx, `SELECT count(*),COALESCE(MAX(seq),0) FROM sqlite_sequence WHERE name=?`, identity.Table).Scan(&count, &high); err != nil { + return report, err + } + if verifyOnly { + if count != 1 || high != identity.SQLiteFloor { + return report, errors.New("completed SQLite identity allocation changed; resume refused") + } + continue + } + if _, err := destination.ExecContext(ctx, `DELETE FROM sqlite_sequence WHERE name=?`, identity.Table); err != nil { + return report, err + } + if _, err := destination.ExecContext(ctx, `INSERT INTO sqlite_sequence(name,seq) VALUES(?,?)`, identity.Table, identity.SQLiteFloor); err != nil { + return report, err + } + } + if !verifyOnly { + for _, name := range manifest.AddedSQLiteMigrations { + if _, err := destination.ExecContext(ctx, `INSERT OR IGNORE INTO _migrations(name) VALUES(?)`, name); err != nil { + return report, err + } + } + } + for _, name := range manifest.AddedSQLiteMigrations { + var n int + if err := destination.QueryRowContext(ctx, `SELECT count(*) FROM _migrations WHERE name=?`, name).Scan(&n); err != nil || n != 1 { + return report, errors.New("SQLite physical migration marker is missing") + } + } + if o.Kind == "telemetry" { + // A copied ledger can already mark this async task done. Build the real + // index offline; never rewrite its retained status to hide absent work. + if err := dbschema.EnsureSQLiteObserverTimeIndex(ctx, destination); err != nil { + return report, err + } + } + if err := assertSQLiteIntegrity(ctx, destination); err != nil { + return report, err + } + sourceAfter, err := digestPostgresSource(ctx, f) + if err != nil { + return report, err + } + identitiesAfter, err := postgresIdentities(ctx, f) + if err != nil { + return report, err + } + if !slices.Equal(sourceAfter, manifest.Source) || !slices.Equal(identitiesAfter, manifest.Identities) { + return report, errors.New("PostgreSQL source or sequence changed during conversion; target remains unselected") + } + if err := validatePostgresLayout(f.db, o.Kind, f.tables); err != nil { + return report, err + } + if !o.keepPending { + if !verifyOnly { + if _, err := destination.ExecContext(ctx, `DROP TABLE corescope_reverse_progress`); err != nil { + return report, err + } + } + if err := assertSQLiteData(ctx, destination, o.Kind); err != nil { + return report, err + } + } + report.Verified = true + return report, nil +} + +func createReverseTarget(ctx context.Context, destination, kind, runID string) (err error) { + if err := os.MkdirAll(filepath.Dir(destination), 0700); err != nil { + return err + } + file, err := os.CreateTemp(filepath.Dir(destination), ".corescope-reverse-*.sqlite") + if err != nil { + return err + } + temp := file.Name() + if err := file.Close(); err != nil { + return err + } + defer func() { + for _, suffix := range []string{"", "-wal", "-shm", "-journal"} { + _ = os.Remove(temp + suffix) + } + }() + uri, err := legacy.WriterDSN(temp) + if err != nil { + return err + } + db, err := sql.Open(importSQLiteDriver, uri) + if err != nil { + return err + } + defer db.Close() + db.SetMaxOpenConns(1) + if kind == "telemetry" { + err = dbschema.ApplySQLite(db, func(string, ...interface{}) {}) + } else { + err = users.ApplySQLite(db) + } + if err != nil { + return err + } + if _, err := db.ExecContext(ctx, `PRAGMA foreign_keys=OFF`); err != nil { + return err + } + tables, err := layouts(kind) + if err != nil { + return err + } + tx, err := db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + for i := len(tables) - 1; i >= 0; i-- { + if _, err := tx.ExecContext(ctx, `DELETE FROM `+quote(tables[i].Name)); err != nil { + return err + } + } + if _, err := tx.ExecContext(ctx, `CREATE TABLE corescope_reverse_progress(table_name TEXT PRIMARY KEY,rows_copied INTEGER NOT NULL,sha256 TEXT NOT NULL,complete INTEGER NOT NULL); INSERT INTO corescope_reverse_progress VALUES('',0,?,0)`, runID); err != nil { + return err + } + if err := tx.Commit(); err != nil { + return err + } + var busy, log, checkpointed int + if err := db.QueryRowContext(ctx, `PRAGMA wal_checkpoint(TRUNCATE)`).Scan(&busy, &log, &checkpointed); err != nil { + return err + } + if busy != 0 { + return errors.New("private SQLite staging checkpoint is busy") + } + if err := db.Close(); err != nil { + return err + } + // Publish the complete native pending marker with the file. Link never + // overwrites an existing path, unlike rename on Unix. Both paths share a + // directory/filesystem and the closed database has no outstanding WAL. + return os.Link(temp, destination) +} + +func openReverseTarget(ctx context.Context, o reverseOptions, m reverseManifest) (*sql.DB, bool, error) { + if _, err := os.Lstat(m.Destination); errors.Is(err, os.ErrNotExist) { + if err := createReverseTarget(ctx, m.Destination, o.Kind, m.RunID); err != nil { + return nil, false, err + } + } else if err != nil { + return nil, false, err + } else if !o.Resume { + return nil, false, errors.New("SQLite destination exists; use the exact verified job to resume") + } + uri, err := legacy.WriterDSN(m.Destination) + if err != nil { + return nil, false, err + } + db, err := sql.Open(importSQLiteDriver, uri) + if err != nil { + return nil, false, err + } + db.SetMaxOpenConns(1) + fail := func(err error) (*sql.DB, bool, error) { db.Close(); return nil, false, err } + var progress int + if err := db.QueryRowContext(ctx, `SELECT count(*) FROM sqlite_schema WHERE name='corescope_reverse_progress' AND type='table'`).Scan(&progress); err != nil { + return fail(err) + } + if progress == 0 { + return db, true, nil + } + var id string + if err := db.QueryRowContext(ctx, `SELECT sha256 FROM corescope_reverse_progress WHERE table_name=''`).Scan(&id); err != nil || id != m.RunID { + return fail(errors.New("SQLite target belongs to another or incomplete preparation job")) + } + if _, err := db.ExecContext(ctx, `PRAGMA foreign_keys=OFF`); err != nil { + return fail(err) + } + return db, false, nil +} +func digestSQLiteTable(ctx context.Context, source, target *sql.DB, table importTable, m reverseManifest) (tableReport, error) { + report := tableReport{Table: table.Name} + _, names := columnNames(table) + for i := range names { + names[i] = "+" + names[i] + } + order, err := orderBy(source, table, false, false) + if err != nil { + return report, err + } + where := "" + var args []any + if table.Name == "_migrations" && len(m.AddedSQLiteMigrations) > 0 { + var params []string + for _, name := range m.AddedSQLiteMigrations { + params = append(params, "?") + args = append(args, name) + } + where = ` WHERE name NOT IN (` + strings.Join(params, ",") + `)` + } + rows, err := target.QueryContext(ctx, `SELECT `+strings.Join(names, ",")+` FROM `+quote(table.Name)+where+` ORDER BY `+order, args...) + if err != nil { + return report, err + } + defer rows.Close() + h := sha256.New() + for rows.Next() { + values, err := scanCanonicalRow(rows, table) + if err != nil { + return report, err + } + hashRow(h, values) + report.Rows++ + } + report.SHA256 = hex.EncodeToString(h.Sum(nil)) + return report, rows.Err() +} + +func scanCanonicalRow(rows *sql.Rows, table importTable) ([]any, error) { + values := make([]any, len(table.Columns)) + dest := make([]any, len(values)) + for i := range dest { + dest[i] = &values[i] + } + if err := rows.Scan(dest...); err != nil { + return nil, err + } + for i, col := range table.Columns { + value, err := canonical(values[i], col.Type) + if err != nil { + return nil, fmt.Errorf("column %s: %w", col.Name, err) + } + if f, ok := value.(float64); ok && math.IsNaN(f) { + return nil, errors.New("SQLite cannot preserve a PostgreSQL NaN; source was not changed") + } + values[i] = value + } + return values, nil +} + +func copyReverseTable(ctx context.Context, source, target *sql.DB, table importTable, o reverseOptions, m reverseManifest, verifyOnly bool) (tableReport, error) { + report := tableReport{Table: table.Name} + existing, err := digestSQLiteTable(ctx, source, target, table, m) + if err != nil { + return report, err + } + var copied int64 + var stored string + var complete bool + if verifyOnly { + copied, stored, complete = existing.Rows, existing.SHA256, true + } else { + err := target.QueryRowContext(ctx, `SELECT rows_copied,sha256,complete FROM corescope_reverse_progress WHERE table_name=?`, table.Name).Scan(&copied, &stored, &complete) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return report, err + } + if stored == "" { + sum := sha256.Sum256(nil) + stored = hex.EncodeToString(sum[:]) + } + if existing.Rows != copied || existing.SHA256 != stored { + return report, errors.New("SQLite destination differs from committed copy progress") + } + } + _, names := columnNames(table) + order, err := orderBy(source, table, true, false) + if err != nil { + return report, err + } + rows, err := source.QueryContext(ctx, `SELECT `+strings.Join(names, ",")+` FROM `+quote(table.Name)+` ORDER BY `+order) + if err != nil { + return report, err + } + defer rows.Close() + placeholders := make([]string, len(names)) + for i := range placeholders { + placeholders[i] = fmt.Sprintf("?%d", i+1) + } + insert := `INSERT INTO ` + quote(table.Name) + ` (` + strings.Join(names, ",") + `) VALUES(` + strings.Join(placeholders, ",") + `)` + h := sha256.New() + var batch [][]any + bytes := 0 + flush := func(done bool) error { + if verifyOnly { + return nil + } + if len(batch) == 0 && !done { + return nil + } + tx, err := target.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + stmt, err := tx.PrepareContext(ctx, insert) + if err != nil { + return err + } + defer stmt.Close() + for _, values := range batch { + if _, err := stmt.ExecContext(ctx, values...); err != nil { + return err + } + } + if _, err := tx.ExecContext(ctx, `INSERT INTO corescope_reverse_progress VALUES(?,?,?,?) ON CONFLICT(table_name) DO UPDATE SET rows_copied=excluded.rows_copied,sha256=excluded.sha256,complete=excluded.complete`, table.Name, report.Rows, hex.EncodeToString(h.Sum(nil)), done); err != nil { + return err + } + if err := tx.Commit(); err != nil { + return err + } + wrote := len(batch) > 0 + batch = nil + bytes = 0 + if wrote && o.afterBatch != nil { + return o.afterBatch() + } + return nil + } + for rows.Next() { + values, err := scanCanonicalRow(rows, table) + if err != nil { + return report, err + } + hashRow(h, values) + report.Rows++ + if report.Rows <= copied { + if report.Rows == copied && hex.EncodeToString(h.Sum(nil)) != stored { + return report, errors.New("source prefix differs from reverse copy progress") + } + continue + } + if complete { + return report, errors.New("completed SQLite table has missing source rows") + } + batch = append(batch, values) + for _, value := range values { + if s, ok := value.(string); ok { + bytes += len(s) + } else { + bytes += 8 + } + } + if len(batch) >= o.BatchSize || bytes >= 8*1024*1024 { + if err := flush(false); err != nil { + return report, err + } + } + } + if err := rows.Err(); err != nil { + return report, err + } + if report.Rows < copied { + return report, errors.New("SQLite target contains extra rows") + } + if err := flush(true); err != nil { + return report, err + } + report.SHA256 = hex.EncodeToString(h.Sum(nil)) + got, err := digestSQLiteTable(ctx, source, target, table, m) + if err != nil { + return report, err + } + if got != report { + return report, errors.New("SQLite count or logical digest verification failed") + } + return report, nil +} + +func assertSQLiteData(ctx context.Context, db *sql.DB, kind string) error { + if err := assertSQLiteIntegrity(ctx, db); err != nil { + return err + } + if kind == "telemetry" { + return dbschema.AssertSQLiteReady(db) + } + return users.AssertSQLiteReady(db) +} + +func assertSQLiteIntegrity(ctx context.Context, db *sql.DB) error { + var integrity string + if err := db.QueryRowContext(ctx, `PRAGMA quick_check`).Scan(&integrity); err != nil { + return err + } + if integrity != "ok" { + return errors.New("SQLite target failed integrity check") + } + rows, err := db.QueryContext(ctx, `PRAGMA foreign_key_check`) + if err != nil { + return err + } + invalid := rows.Next() + rowErr := rows.Err() + rows.Close() + if rowErr != nil { + return rowErr + } + if invalid { + return errors.New("SQLite target relationships failed verification") + } + return nil +} diff --git a/cmd/migrate/reverse_test.go b/cmd/migrate/reverse_test.go new file mode 100644 index 000000000..d0727211c --- /dev/null +++ b/cmd/migrate/reverse_test.go @@ -0,0 +1,190 @@ +package main + +import ( + "context" + "errors" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/pgutil" + "github.com/meshcore-analyzer/users" +) + +func TestReverseTelemetryPreservesRowsAndFutureIDs(t *testing.T) { + dsn := postgresDatabase(t) + ctx := context.Background() + _, err := importSQLite(ctx, importOptions{Source: telemetrySource(t), DatabaseURL: dsn, StateDir: t.TempDir(), Kind: "telemetry"}) + if err != nil { + t.Fatal(err) + } + if _, err := finalizeImport(ctx, dsn, "telemetry"); err != nil { + t.Fatal(err) + } + pg := openImportDB(t, dsn) + if _, err := pg.Exec(`INSERT INTO _async_migrations(name,status,started_at,ended_at,error) VALUES('obs_observer_ts_idx_v1','done','raw started','raw ended',NULL)`); err != nil { + t.Fatal(err) + } + for _, q := range []string{`SELECT setval(pg_get_serial_sequence('transmissions','id'),100500,false)`, `UPDATE observations SET score=3.5,resolved_path=' {"raw": true} ' WHERE id=90`, `UPDATE transmissions SET first_seen='malformed date kept as text' WHERE id=80`} { + if _, err := pg.Exec(q); err != nil { + t.Fatal(err) + } + } + dest := filepath.Join(t.TempDir(), "reverse # % é.sqlite") + options := reverseOptions{DatabaseURL: dsn, Destination: dest, StateDir: t.TempDir(), Kind: "telemetry", BatchSize: 2} + injected := false + options.afterBatch = func() error { + if !injected { + injected = true + return errors.New("interrupt after committed batch") + } + return nil + } + if _, err := reversePostgres(ctx, options); err == nil || !injected { + t.Fatal("expected interruption after committed batch", err) + } + options.Resume = true + options.afterBatch = nil + report, err := reversePostgres(ctx, options) + if err != nil || !report.Verified { + t.Fatal("resume reverse", err) + } + db, err := openSQLite(dest, "rw") + if err != nil { + t.Fatal(err) + } + defer db.Close() + if err := dbschema.AssertSQLiteReady(db); err != nil { + t.Fatal(err) + } + var indexCount int + if err := db.QueryRow(`SELECT count(*) FROM sqlite_master WHERE type='index' AND name='idx_observations_observer_idx_timestamp'`).Scan(&indexCount); err != nil || indexCount != 1 { + t.Fatal("copied done ledger lacks its native index", err) + } + var status, started, ended string + if err := db.QueryRow(`SELECT status,started_at,ended_at FROM _async_migrations WHERE name='obs_observer_ts_idx_v1'`).Scan(&status, &started, &ended); err != nil || status != "done" || started != "raw started" || ended != "raw ended" { + t.Fatal("source migration ledger changed", err) + } + var node, parent, unused int + var plan string + if err := db.QueryRow(`EXPLAIN QUERY PLAN SELECT timestamp FROM observations WHERE observer_idx=999 ORDER BY timestamp`).Scan(&node, &parent, &unused, &plan); err != nil || !strings.Contains(plan, "idx_observations_observer_idx_timestamp") { + t.Fatal("observer/time query cannot use native index", plan, err) + } + var n int + if err := db.QueryRow(`SELECT count(*) FROM observations WHERE (id=0 AND observer_idx=0 AND path_json IS NULL) OR (id=90 AND observer_idx=999 AND path_json='' AND score=3.5 AND resolved_path=' {"raw": true} ') OR (id=91 AND observer_idx IS NULL)`).Scan(&n); err != nil || n != 3 { + t.Fatal("row parity", n, err) + } + var id int64 + if err := db.QueryRow(`INSERT INTO transmissions(raw_hex,hash,first_seen) VALUES('00','new','2026-01-03') RETURNING id`).Scan(&id); err != nil || id != 100500 { + t.Fatal("is_called=false future ID", id, err) + } + if err := db.QueryRow(`INSERT INTO observers(id) VALUES('future') RETURNING rowid`).Scan(&id); err != nil || id <= 999 { + t.Fatal("observer orphan reuse", id, err) + } + if _, err := os.Stat(filepath.Join(options.StateDir, "telemetry", "recovery.pg_dump")); err != nil { + t.Fatal("native recovery missing", err) + } + // Verify the actual native recovery archive, not only its filename/hash. + restored := postgresDatabase(t) + config, err := pgutil.ParseConfig(restored) + if err != nil { + t.Fatal(err) + } + command := exec.Command("pg_restore", "--no-password", "--no-owner", "--no-privileges", "--exit-on-error", "--dbname", config.Database, filepath.Join(options.StateDir, "telemetry", "recovery.pg_dump")) + command.Env, err = pgutil.CommandEnv(restored) + if err != nil { + t.Fatal(err) + } + if err := command.Run(); err != nil { + t.Fatal("native PostgreSQL recovery restore failed") + } + var manifest reverseManifest + if err := readPrivateJSON(filepath.Join(options.StateDir, "telemetry", "reverse.json"), &manifest); err != nil { + t.Fatal(err) + } + recovery := openImportDB(t, restored) + tables, err := layouts("telemetry") + if err != nil { + t.Fatal(err) + } + for i, table := range tables { + got, err := digestTarget(ctx, recovery, table) + if err != nil || got != manifest.Source[i] { + t.Fatal("native recovery data mismatch", table.Name, err) + } + } +} + +func TestReverseSequenceMutationLeavesNativePendingMarker(t *testing.T) { + owner := postgresDatabase(t) + ctx := context.Background() + db := openImportDB(t, owner) + if err := dbschema.ApplyPostgres(db, nil); err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`INSERT INTO observers(id) VALUES('only-observer')`); err != nil { + t.Fatal(err) + } + dest := filepath.Join(t.TempDir(), "pending.sqlite") + options := reverseOptions{DatabaseURL: owner, Destination: dest, StateDir: t.TempDir(), Kind: "telemetry", BatchSize: 1, afterBatch: func() error { + _, err := db.Exec(`SELECT nextval(pg_get_serial_sequence('observers','rowid'))`) + return err + }} + if _, err := reversePostgres(ctx, options); err == nil { + t.Fatal("changed source sequence accepted") + } + target, err := openSQLite(dest, "ro") + if err != nil { + t.Fatal(err) + } + defer target.Close() + if err := dbconfig.AssertSQLiteImportComplete(target); !errors.Is(err, dbconfig.ErrSQLiteImportIncomplete) { + t.Fatal("partial reverse target can be adopted elsewhere", err) + } +} + +func TestReverseAccountsPreservesAuthenticationAndIdentity(t *testing.T) { + dsn := postgresDatabase(t) + ctx := context.Background() + _, err := importSQLite(ctx, importOptions{Source: accountSource(t, 5), DatabaseURL: dsn, StateDir: t.TempDir(), Kind: "accounts"}) + if err != nil { + t.Fatal(err) + } + if _, err := finalizeImport(ctx, dsn, "accounts"); err != nil { + t.Fatal(err) + } + pg := openImportDB(t, dsn) + for _, q := range []string{`SELECT setval(pg_get_serial_sequence('mail_events','id'),9001,false)`, `INSERT INTO audit_log(at,actor_user_id,action,target_user_id) VALUES(2,888,'deleted',999)`} { + if _, err := pg.Exec(q); err != nil { + t.Fatal(err) + } + } + dest := filepath.Join(t.TempDir(), "accounts.sqlite") + options := reverseOptions{DatabaseURL: dsn, Destination: dest, StateDir: t.TempDir(), Kind: "accounts", BatchSize: 2} + report, err := reversePostgres(ctx, options) + if err != nil || !report.Verified { + t.Fatal("reverse accounts", err) + } + db, err := openSQLite(dest, "rw") + if err != nil { + t.Fatal(err) + } + defer db.Close() + if err := users.AssertSQLiteReady(db); err != nil { + t.Fatal(err) + } + var id int64 + if err := db.QueryRow(`SELECT id FROM mail_events WHERE mail_id=4`).Scan(&id); err != nil || id != 900 { + t.Fatal("event id", id, err) + } + if err := db.QueryRow(`INSERT INTO mail_events(mail_id,event,at) VALUES(4,'new',99) RETURNING id`).Scan(&id); err != nil || id != 9001 { + t.Fatal("event sequence", id, err) + } + if err := db.QueryRow(`INSERT INTO users(email,display_name,password_hash,created_at) VALUES('new@example.invalid','','hash',3) RETURNING id`).Scan(&id); err != nil || id != 1000 { + t.Fatal("historical user references", id, err) + } +} diff --git a/cmd/migrate/source.go b/cmd/migrate/source.go new file mode 100644 index 000000000..2c723628b --- /dev/null +++ b/cmd/migrate/source.go @@ -0,0 +1,119 @@ +package main + +import ( + "context" + "crypto/sha256" + "database/sql" + "errors" + "io" + "net/url" + "os" + + "github.com/mattn/go-sqlite3" + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/sqliteutil" +) + +const importSQLiteDriver = "corescope_sqlite_import" +const maxImportRowBytes = 64 * 1024 * 1024 + +func init() { + // Apply the bound before any SQLite row reaches the Go driver. A check + // after Scan would already have allocated an arbitrarily large value. + sql.Register(importSQLiteDriver, &sqlite3.SQLiteDriver{ConnectHook: func(conn *sqlite3.SQLiteConn) error { + conn.SetLimit(sqlite3.SQLITE_LIMIT_LENGTH, maxImportRowBytes) + return nil + }}) +} + +// snapshotSource adds offline source immutability checks to the shared native +// backup. Snapshot itself is also used by live backups that permit writers. +func snapshotSource(ctx context.Context, source, destination string) error { + before, err := sourceFingerprintContext(ctx, source) + if err != nil { + return err + } + if err := sqliteutil.Snapshot(ctx, source, destination); err != nil { + return err + } + after, err := sourceFingerprintContext(ctx, source) + if err != nil || before != after { + for _, suffix := range []string{"", "-wal", "-shm", "-journal"} { + _ = os.Remove(destination + suffix) + } + if err != nil { + return err + } + return errors.New("SQLite source changed during snapshot; stop all instance writers before retrying") + } + return nil +} + +func sqliteURL(path, mode string) (string, error) { + return dbconfig.SQLiteURI(path, url.Values{"mode": {mode}, "_busy_timeout": {"5000"}}) +} + +func openSQLite(path, mode string) (*sql.DB, error) { + uri, err := sqliteURL(path, mode) + if err != nil { + return nil, err + } + return sql.Open(importSQLiteDriver, uri) +} + +// Hash the main file and committed-write log without loading either into RAM. +// Shared-memory coordination files are not application data. +func sourceFingerprint(path string) ([2][32]byte, error) { + return sourceFingerprintContext(context.Background(), path) +} + +type contextReader struct { + ctx context.Context + r io.Reader +} + +func (r contextReader) Read(p []byte) (int, error) { + if err := r.ctx.Err(); err != nil { + return 0, err + } + return r.r.Read(p) +} + +func sourceFingerprintContext(ctx context.Context, path string) ([2][32]byte, error) { + var result [2][32]byte + if err := ctx.Err(); err != nil { + return result, err + } + for i, suffix := range []string{"", "-wal"} { + f, err := os.Open(path + suffix) + if i == 1 && errors.Is(err, os.ErrNotExist) { + continue + } + if err != nil { + return result, err + } + if i == 1 { + info, err := f.Stat() + if err != nil { + f.Close() + return result, err + } + // Read-only WAL-mode opens can create/remove a zero-byte sidecar. + // Every nonempty byte, including headers, still affects identity. + if info.Size() == 0 { + if err := f.Close(); err != nil { + return result, err + } + continue + } + } + h := sha256.New() + _, copyErr := io.CopyBuffer(h, contextReader{ctx, f}, make([]byte, 128*1024)) + closeErr := f.Close() + if err := errors.Join(copyErr, closeErr); err != nil { + return result, err + } + copy(result[i][:], h.Sum(nil)) + } + return result, nil +} diff --git a/cmd/migrate/source_test.go b/cmd/migrate/source_test.go new file mode 100644 index 000000000..32f790946 --- /dev/null +++ b/cmd/migrate/source_test.go @@ -0,0 +1,188 @@ +package main + +import ( + "context" + "crypto/sha256" + "database/sql" + "os" + "path/filepath" + "testing" + + "github.com/mattn/go-sqlite3" +) + +func TestSnapshotSourceIncludesWALWithoutChangingSource(t *testing.T) { + dir := t.TempDir() + source := filepath.Join(dir, "source.db") + destination := filepath.Join(dir, "recovery.db") + db, err := sql.Open("sqlite3", source+"?_journal_mode=WAL") + if err != nil { + t.Fatal(err) + } + defer db.Close() + db.SetMaxOpenConns(1) + for _, statement := range []string{ + `PRAGMA wal_autocheckpoint=0`, + `CREATE TABLE records (id INTEGER PRIMARY KEY, value TEXT)`, + `INSERT INTO records VALUES (0, NULL), (90, '')`, + `CREATE TABLE identities (key TEXT PRIMARY KEY)`, + `INSERT INTO identities(rowid,key) VALUES (0,'zero'),(900,'gap')`, + } { + if _, err := db.Exec(statement); err != nil { + t.Fatal(err) + } + } + before := make(map[string][32]byte) + for _, path := range []string{source, source + "-wal"} { + content, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + before[path] = sha256.Sum256(content) + } + if err := snapshotSource(context.Background(), source, destination); err != nil { + t.Fatalf("WAL-aware recovery snapshot: %v", err) + } + for path, expected := range before { + content, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if sha256.Sum256(content) != expected { + t.Fatalf("source was modified: %s", filepath.Base(path)) + } + } + copyDB, err := sql.Open("sqlite3", destination) + if err != nil { + t.Fatal(err) + } + defer copyDB.Close() + var count int + if err := copyDB.QueryRow(`SELECT COUNT(*) FROM records WHERE (id=0 AND value IS NULL) OR (id=90 AND value='')`).Scan(&count); err != nil { + t.Fatal(err) + } + if count != 2 { + t.Fatalf("snapshot lost committed WAL rows or null/empty values: got %d", count) + } + if err := copyDB.QueryRow(`SELECT COUNT(*) FROM identities WHERE (rowid=0 AND key='zero') OR (rowid=900 AND key='gap')`).Scan(&count); err != nil { + t.Fatal(err) + } + if count != 2 { + t.Fatal("snapshot changed hidden rowids") + } +} + +func TestSnapshotSourceDoesNotOverwriteRecoveryOrOriginal(t *testing.T) { + dir := t.TempDir() + source := filepath.Join(dir, "source.db") + db, err := sql.Open("sqlite3", source) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`CREATE TABLE records(id INTEGER)`); err != nil { + t.Fatal(err) + } + if err := db.Close(); err != nil { + t.Fatal(err) + } + for _, destination := range []string{source, filepath.Join(dir, "existing.db")} { + if destination != source { + if err := os.WriteFile(destination, []byte("existing recovery"), 0o600); err != nil { + t.Fatal(err) + } + } + before, err := os.ReadFile(destination) + if err != nil { + t.Fatal(err) + } + if err := snapshotSource(context.Background(), source, destination); err == nil { + t.Fatal("overwriting an existing file was accepted") + } + after, err := os.ReadFile(destination) + if err != nil { + t.Fatal(err) + } + if sha256.Sum256(before) != sha256.Sum256(after) { + t.Fatal("existing file was changed") + } + } +} + +func TestSnapshotSourceFailureLeavesNoPartialCopy(t *testing.T) { + for _, cancelled := range []bool{false, true} { + dir := t.TempDir() + source, destination := filepath.Join(dir, "source.db"), filepath.Join(dir, "recovery.db") + if err := os.WriteFile(source, []byte("not a SQLite database"), 0o600); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + if cancelled { + cancel() + } + if err := snapshotSource(ctx, source, destination); err == nil { + t.Fatal("invalid/cancelled snapshot accepted") + } + cancel() + if _, err := os.Stat(destination); !os.IsNotExist(err) { + t.Fatalf("partial output retained: %v", err) + } + } +} + +func TestSourceFingerprintEmptyWALMatchesAbsentWAL(t *testing.T) { + source := filepath.Join(t.TempDir(), "source.db") + if err := os.WriteFile(source, []byte("stable database bytes"), 0o600); err != nil { + t.Fatal(err) + } + before, err := sourceFingerprint(source) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(source+"-wal", nil, 0o600); err != nil { + t.Fatal(err) + } + empty, err := sourceFingerprint(source) + if err != nil { + t.Fatal(err) + } + if empty != before { + t.Fatal("an empty WAL changed the source identity") + } + if err := os.WriteFile(source+"-wal", []byte("nonempty WAL bytes must be observed"), 0o600); err != nil { + t.Fatal(err) + } + nonempty, err := sourceFingerprint(source) + if err != nil { + t.Fatal(err) + } + if nonempty == before { + t.Fatal("nonempty WAL bytes were ignored") + } +} + +func TestSQLiteSourceLengthLimitPrecedesScan(t *testing.T) { + db, err := sql.Open(importSQLiteDriver, ":memory:") + if err != nil { + t.Fatal(err) + } + defer db.Close() + conn, err := db.Conn(context.Background()) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + if err := conn.Raw(func(driver any) error { + sqlite := driver.(*sqlite3.SQLiteConn) + if got := sqlite.GetLimit(sqlite3.SQLITE_LIMIT_LENGTH); got != maxImportRowBytes { + t.Fatalf("native SQLite length limit=%d", got) + } + sqlite.SetLimit(sqlite3.SQLITE_LIMIT_LENGTH, 1024) + return nil + }); err != nil { + t.Fatal(err) + } + var data []byte + if err := conn.QueryRowContext(context.Background(), `SELECT zeroblob(2048)`).Scan(&data); err == nil { + t.Fatal("native length limit did not reject a large result before Scan") + } +} diff --git a/cmd/migrate/storage_command.go b/cmd/migrate/storage_command.go new file mode 100644 index 000000000..8b74ac761 --- /dev/null +++ b/cmd/migrate/storage_command.go @@ -0,0 +1,475 @@ +package main + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/jackc/pgx/v5/pgconn" + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/dbschema/legacy" + "github.com/meshcore-analyzer/pgutil" + "github.com/meshcore-analyzer/users" +) + +type storageOptions struct { + Action, SelectionFile, Backend, SQLitePath, UsersSQLitePath string + OwnerURL, UsersOwnerURL, StateDir, JobID, ConfigFile, ConfigDir string + Offline bool + afterStore func(string) error + beforeCommit func() error +} + +func firstSetting(values ...string) string { + for _, value := range values { + if value != "" { + return value + } + } + return "" +} + +func storageActionInputs(o storageOptions, recorded bool) (dbconfig.StorageInputs, error) { + var raw dbconfig.StorageInputs + cwd, err := os.Getwd() + if err != nil { + return raw, err + } + configDir, err := filepath.Abs(firstSetting(o.ConfigDir, ".")) + if err != nil { + return raw, err + } + if o.ConfigFile != "" { + raw, err = dbconfig.ReadStorageConfig(o.ConfigFile) + if err != nil { + return raw, err + } + } else { + for _, path := range []string{filepath.Join(configDir, "config.json"), filepath.Join(configDir, "data", "config.json")} { + config, e := dbconfig.ReadStorageConfig(path) + if e == nil { + raw = config + err = nil + break + } + if !errors.Is(e, os.ErrNotExist) { + return raw, e + } + } + if err != nil { + return raw, err + } + } + envPath := os.Getenv("DB_PATH") + if !recorded && o.SQLitePath == "" && envPath != "" && raw.DBPath != "" { + a, e := filepath.Abs(envPath) + if e != nil { + return raw, e + } + b, e := filepath.Abs(raw.DBPath) + if e != nil { + return raw, e + } + if a != b { + return raw, errors.New("legacy config dbPath and DB_PATH differ between services; supply one explicit -sqlite-path before adoption") + } + } + raw.BaseDir = cwd + raw.Backend = dbconfig.Backend(firstSetting(o.Backend, string(raw.Backend))) + raw.EnvBackend = dbconfig.Backend(os.Getenv("CORESCOPE_DB_BACKEND")) + if o.Backend != "" { + raw.EnvBackend = raw.Backend + } + raw.DBPath = firstSetting(o.SQLitePath, envPath, raw.DBPath) + raw.UsersDBPath = firstSetting(o.UsersSQLitePath, raw.UsersDBPath) + raw.DatabaseURL = firstSetting(os.Getenv("CORESCOPE_DATABASE_URL"), raw.DatabaseURL) + raw.ReaderDatabaseURL = os.Getenv("CORESCOPE_READER_DATABASE_URL") + raw.WriterDatabaseURL = os.Getenv("CORESCOPE_WRITER_DATABASE_URL") + raw.UsersDatabaseURL = firstSetting(os.Getenv("CORESCOPE_USERS_DATABASE_URL"), raw.UsersDatabaseURL) + raw.ApprovedChannelsDatabaseURL = firstSetting(os.Getenv("CORESCOPE_APPROVED_CHANNELS_DATABASE_URL"), raw.ApprovedChannelsDatabaseURL) + raw.StateDir = filepath.Dir(o.SelectionFile) // stable discovery, not the recovery directory + raw.FreshInstall = true // only permits resolution; native facts below authorize initialization + sqliteDefault := raw.Backend == dbconfig.SQLite || raw.EnvBackend == dbconfig.SQLite || (raw.Backend == "" && raw.EnvBackend == "" && raw.DatabaseURL == "" && raw.ReaderDatabaseURL == "" && raw.WriterDatabaseURL == "" && raw.UsersDatabaseURL == "" && raw.ApprovedChannelsDatabaseURL == "" && o.OwnerURL == "" && o.UsersOwnerURL == "") + if raw.DBPath == "" && sqliteDefault { + raw.DBPath = filepath.Join(configDir, "data", "meshcore.db") + } + return raw, nil +} + +func runStorageAction(ctx context.Context, o storageOptions, out io.Writer) (err error) { + defer func() { + var pgError *pgconn.PgError + if errors.As(err, &pgError) { + err = fmt.Errorf("storage operation failed (SQLSTATE %s); keep writers stopped and inspect the private recovery job", pgError.Code) + } + }() + if !filepath.IsAbs(o.SelectionFile) || filepath.Clean(o.SelectionFile) != o.SelectionFile || filepath.Base(o.SelectionFile) != "storage-selection.json" { + return errors.New("provide an absolute normalized -selection-file ending in storage-selection.json") + } + status, inspectErr := dbconfig.InspectSelection(o.SelectionFile) + if o.Action == "status" { + if err := json.NewEncoder(out).Encode(status); err != nil { + return err + } + return inspectErr + } + if !o.Offline { + return errors.New("storage setup and switching require -offline after stopping all telemetry and account writers") + } + if o.StateDir == "" { + o.StateDir = filepath.Join(filepath.Dir(o.SelectionFile), "migration") + } + switch o.Action { + case "setup", "adopt", "init", "switch", "resume", "abort": + default: + return errors.New("unsupported storage action; use -help") + } + if inspectErr != nil { + return inspectErr + } + o.OwnerURL = firstSetting(o.OwnerURL, os.Getenv("CORESCOPE_DATABASE_URL")) + o.UsersOwnerURL = firstSetting(o.UsersOwnerURL, os.Getenv("CORESCOPE_USERS_OWNER_DATABASE_URL")) + if o.Action == "resume" || o.Action == "abort" || o.Action == "switch" { + if err := runStorageSwitch(ctx, o); err != nil { + return err + } + } else { + if status.State == "pending" { + return dbconfig.ErrSelectionInProgress + } + if err := setupStorage(ctx, o, status.State == "ready"); err != nil { + return err + } + } + status, err = dbconfig.InspectSelection(o.SelectionFile) + if err != nil { + return err + } + return json.NewEncoder(out).Encode(status) +} + +func setupStorage(ctx context.Context, o storageOptions, recorded bool) error { + if recorded && o.Action == "init" { + return errors.New("installation already has a recorded backend; use a verified switch") + } + raw, err := storageActionInputs(o, recorded) + if err != nil { + return err + } + if recorded { + selection, lease, err := dbconfig.OpenSelection(o.SelectionFile) + if err != nil { + return err + } + defer lease.Close() + if selection.Accounts == nil && o.Action == "setup" && (selection.Backend == dbconfig.SQLite || o.UsersOwnerURL != "" || raw.UsersDatabaseURL != "" || raw.ApprovedChannelsDatabaseURL != "") { + lease.Close() + return initializeAccounts(ctx, o, selection, raw) + } + if selection.Backend == dbconfig.Postgres && o.OwnerURL != "" { + owners, err := bindOwnerURLs(selection, o.OwnerURL, o.UsersOwnerURL) + if err != nil { + return err + } + if err := grantRuntimeTargets(ctx, selection, owners, raw); err != nil { + return err + } + } + return validateRuntimeSelection(ctx, selection, raw) + } + ownerInputs := raw + // Owner endpoints are used only for schema operations. Runtime credentials + // stay separate and are validated against the resulting selected targets. + if o.OwnerURL != "" { + ownerInputs.DatabaseURL = o.OwnerURL + ownerInputs.ReaderDatabaseURL = "" + ownerInputs.WriterDatabaseURL = "" + } + if o.UsersOwnerURL != "" { + ownerInputs.UsersDatabaseURL = o.UsersOwnerURL + ownerInputs.ApprovedChannelsDatabaseURL = "" + } + storage, err := dbconfig.ResolveStorage(ownerInputs) + if err != nil { + return err + } + selected, err := dbconfig.NewSelection(storage) + if err != nil { + return err + } + if storage.Backend == dbconfig.SQLite { + legacyTelemetry, err := sqliteOccupied(ctx, storage.DBPath) + if err != nil { + return err + } + for _, store := range []struct{ kind, path string }{{"telemetry", storage.DBPath}, {"accounts", storage.UsersDBPath}} { + occupied, err := sqliteOccupied(ctx, store.path) + if err != nil { + return err + } + if store.kind == "accounts" && legacyTelemetry && !occupied { + selected.Accounts = nil + storage.UsersDBPath = "" + continue + } + if o.Action == "init" && occupied { + return errors.New("initialization target already contains SQLite schema; use setup/adopt for an existing installation") + } + if o.Action == "adopt" && store.kind == "telemetry" && !occupied { + return errors.New("adoption requires an existing telemetry database") + } + if err := applySQLiteTarget(ctx, store.path, store.kind); err != nil { + return err + } + } + } else { + if o.OwnerURL == "" { + return errors.New("unrecorded PostgreSQL setup requires its telemetry owner URL through CORESCOPE_DATABASE_URL") + } + if (raw.UsersDatabaseURL != "" || raw.ApprovedChannelsDatabaseURL != "") && o.UsersOwnerURL == "" { + return errors.New("PostgreSQL account setup requires CORESCOPE_USERS_OWNER_DATABASE_URL; runtime credentials are never owner credentials") + } + if err := validateInitialOwnerTargets(ctx, selected, storage); err != nil { + return err + } + storage, err = bindOwnerURLs(selected, o.OwnerURL, o.UsersOwnerURL) + if err != nil { + return err + } + if storage.UsersDatabaseURL != "" { + same, err := pgutil.SameDatabase(ctx, storage.WriterDatabaseURL, storage.UsersDatabaseURL) + if err != nil { + return err + } + if same { + return errors.New("telemetry and accounts require separate PostgreSQL databases") + } + } + knownSQLite := firstSetting(raw.DBPath, filepath.Join(filepath.Dir(o.SelectionFile), "meshcore.db")) + knownUsers := firstSetting(raw.UsersDBPath, filepath.Join(filepath.Dir(knownSQLite), "users.db")) + for _, store := range []struct{ kind, url, oldPath string }{{"telemetry", storage.WriterDatabaseURL, knownSQLite}, {"accounts", storage.UsersDatabaseURL, knownUsers}} { + occupied, err := sqliteOccupied(ctx, store.oldPath) + if err != nil { + return err + } + if store.url == "" { + if occupied { + return errors.New("an existing SQLite account store must be preserved; supply its PostgreSQL owner target and use a verified switch") + } + continue + } + db, err := pgutil.Open(store.url, false) + if err != nil { + return err + } + readyErr := assertPostgresReady(db, store.kind) + if readyErr == nil { + db.Close() + if o.Action == "init" { + return errors.New("PostgreSQL target is already initialized; use setup/adopt") + } + if occupied && o.Action != "adopt" { + if err := checkImportedSource(ctx, store.url, store.kind, store.oldPath); err != nil { + return errors.New("retained SQLite data needs its verified import or explicit adoption of the existing ready PostgreSQL store") + } + } + continue + } + var objects int + err = db.QueryRowContext(ctx, importDestinationObjects).Scan(&objects) + if err != nil { + db.Close() + return err + } + if objects != 0 || o.Action == "adopt" { + db.Close() + return errors.New("PostgreSQL target is not a ready supported store or a fresh empty schema") + } + if occupied { + db.Close() + return errors.New("refusing to initialize empty PostgreSQL while primary SQLite data exists; use a verified offline switch") + } + if raw.Backend != dbconfig.Postgres && raw.EnvBackend != dbconfig.Postgres { + db.Close() + return errors.New("fresh PostgreSQL initialization requires an explicit backend choice") + } + if store.kind == "telemetry" { + err = dbschema.ApplyPostgres(db, nil) + } else { + err = users.ApplyPostgres(db) + } + db.Close() + if err != nil { + return err + } + } + } + if selected.Backend == dbconfig.Postgres { + if err := grantRuntimeTargets(ctx, selected, storage, raw); err != nil { + return err + } + } + if err := validateRuntimeSelection(ctx, selected, raw); err != nil { + return err + } + _, err = dbconfig.AdoptSelection(o.SelectionFile, selected) + return err +} + +func sqliteOccupied(ctx context.Context, path string) (bool, error) { + if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) { + return false, nil + } else if err != nil { + return false, err + } + db, err := openSQLite(path, "ro") + if err != nil { + return false, err + } + defer db.Close() + var count int + if err := db.QueryRowContext(ctx, `SELECT count(*) FROM sqlite_schema WHERE name NOT LIKE 'sqlite_%'`).Scan(&count); err != nil { + return false, errors.New("existing SQLite target cannot be validated; preserve it instead of selecting an empty database") + } + return count != 0, nil +} + +func applySQLiteTarget(ctx context.Context, path, kind string) error { + if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { + return err + } + if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) { + f, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600) + if err != nil { + return err + } + if err := f.Close(); err != nil { + return err + } + } + uri, err := legacy.WriterDSN(path) + if err != nil { + return err + } + db, err := sql.Open(importSQLiteDriver, uri) + if err != nil { + return err + } + defer db.Close() + db.SetMaxOpenConns(1) + if err := dbconfig.AssertSQLiteImportComplete(db); err != nil { + return err + } + if kind == "telemetry" { + return dbschema.ApplySQLite(db, func(string, ...interface{}) {}) + } + return users.ApplySQLite(db) +} + +func assertPostgresReady(db *sql.DB, kind string) error { + if kind == "telemetry" { + return dbschema.AssertPostgresReady(db) + } + return users.AssertPostgresReady(db) +} + +func validateRuntimeSelection(ctx context.Context, selection dbconfig.Selection, raw dbconfig.StorageInputs) error { + bound, err := selection.ApplyTo(raw) + if err != nil { + return err + } + storage, err := dbconfig.ResolveStorage(bound) + if err != nil { + return err + } + if storage.Backend == dbconfig.SQLite { + a, err := os.Stat(storage.DBPath) + if err != nil { + return err + } + stores := []struct{ kind, path string }{{"telemetry", storage.DBPath}} + if selection.Accounts != nil { + b, err := os.Stat(storage.UsersDBPath) + if err != nil { + return err + } + if os.SameFile(a, b) { + return errors.New("telemetry and accounts cannot share the same SQLite file") + } + stores = append(stores, struct{ kind, path string }{"accounts", storage.UsersDBPath}) + } + for _, store := range stores { + db, err := openSQLite(store.path, "ro") + if err != nil { + return err + } + if err := dbconfig.AssertSQLiteImportComplete(db); err != nil { + db.Close() + return err + } + if store.kind == "telemetry" { + err = dbschema.AssertSQLiteReady(db) + } else { + err = users.AssertSQLiteReady(db) + } + db.Close() + if err != nil { + return err + } + } + return nil + } + if storage.ReaderDatabaseURL == "" || storage.WriterDatabaseURL == "" { + return errors.New("prepare both PostgreSQL reader and writer role URLs before selecting this backend") + } + reader, err := pgutil.Open(storage.ReaderDatabaseURL, true) + if err != nil { + return err + } + if err = dbschema.AssertPostgresReady(reader); err == nil { + err = pgutil.AssertReadOnly(reader) + } + reader.Close() + if err != nil { + return err + } + writer, err := pgutil.Open(storage.WriterDatabaseURL, false) + if err != nil { + return err + } + if err = dbschema.AssertPostgresReady(writer); err == nil { + err = dbschema.AssertWriter(writer) + } + writer.Close() + if err != nil { + return err + } + if selection.Accounts != nil { + if storage.UsersDatabaseURL == "" { + return errors.New("prepare the selected PostgreSQL account runtime URL before cutover") + } + store, err := users.OpenStorage(storage) + if err != nil { + return err + } + store.Close() + } + if storage.ApprovedChannelsDatabaseURL != "" { + db, err := pgutil.Open(storage.ApprovedChannelsDatabaseURL, true) + if err != nil { + return err + } + err = pgutil.AssertReadOnly(db) + db.Close() + if err != nil { + return err + } + } + return nil +} diff --git a/cmd/migrate/storage_command_test.go b/cmd/migrate/storage_command_test.go new file mode 100644 index 000000000..6f45e3074 --- /dev/null +++ b/cmd/migrate/storage_command_test.go @@ -0,0 +1,149 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/pgutil" +) + +func TestStorageStatusNeedsNoCredentialsOrDatabase(t *testing.T) { + t.Setenv("CORESCOPE_DATABASE_URL", "unreachable-private-secret") + t.Setenv("CORESCOPE_USERS_DATABASE_URL", "unreachable-private-secret") + selection := filepath.Join(t.TempDir(), "storage-selection.json") + var out bytes.Buffer + if err := runCommand(context.Background(), []string{"-storage-action=status", "-selection-file", selection}, &out); err != nil { + t.Fatal(err) + } + var status dbconfig.SelectionStatus + if err := json.Unmarshal(out.Bytes(), &status); err != nil { + t.Fatal(err) + } + if status.State != "unrecorded" || status.Backend != "" { + t.Fatalf("status=%+v", status) + } + if bytes.Contains(out.Bytes(), []byte("secret")) { + t.Fatal("status exposed connection input") + } +} + +func TestRecordedPostgresPortCannotDriftThroughEnvironment(t *testing.T) { + t.Setenv("PGPORT", "") + t.Setenv("PGSERVICE", "") + t.Setenv("PGOPTIONS", "") + base := t.TempDir() + selected, err := dbconfig.NewSelection(dbconfig.Storage{Backend: dbconfig.Postgres, StateDir: base, WriterDatabaseURL: "postgres://owner:secret@localhost/data"}) + if err != nil { + t.Fatal(err) + } + t.Setenv("PGPORT", "6543") + raw, err := selected.ApplyTo(dbconfig.StorageInputs{BaseDir: base, DatabaseURL: "postgres://rotated:secret@localhost/data"}) + if err != nil { + t.Fatal(err) + } + config, err := pgutil.ParseConfig(raw.WriterDatabaseURL) + if err != nil || config.Port != 5432 || config.Database != "data" || config.RuntimeParams["search_path"] != `"public"` { + t.Fatal("native parser drifted from recorded target", err) + } +} + +func TestSetupRefusesMalformedPrimaryBeforeValidFallback(t *testing.T) { + dir := t.TempDir() + if err := os.MkdirAll(filepath.Join(dir, "data"), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "config.json"), []byte(`{broken`), 0600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "data", "config.json"), []byte(`{"db":{"backend":"sqlite"}}`), 0600); err != nil { + t.Fatal(err) + } + selection := filepath.Join(dir, "data", "storage-selection.json") + if err := runCommand(context.Background(), []string{"-storage-action=setup", "-selection-file", selection, "-config-dir", dir, "-offline"}, &bytes.Buffer{}); err == nil { + t.Fatal("bad primary config silently chose a fallback") + } + if _, err := os.Stat(selection); !os.IsNotExist(err) { + t.Fatal("bad primary initialized an installation", err) + } +} + +func TestStorageSetupDefaultsSQLiteAndPreservesRecordedTargets(t *testing.T) { + for _, name := range []string{"CORESCOPE_DB_BACKEND", "DB_PATH", "CORESCOPE_DATABASE_URL", "CORESCOPE_READER_DATABASE_URL", "CORESCOPE_WRITER_DATABASE_URL", "CORESCOPE_USERS_DATABASE_URL", "CORESCOPE_USERS_OWNER_DATABASE_URL", "CORESCOPE_APPROVED_CHANNELS_DATABASE_URL"} { + t.Setenv(name, "") + } + root := t.TempDir() + selection := filepath.Join(root, "data", "storage-selection.json") + var out bytes.Buffer + args := []string{"-storage-action=setup", "-selection-file", selection, "-config-dir", root, "-offline"} + if err := runCommand(context.Background(), args, &out); err != nil { + t.Fatal(err) + } + got, err := dbconfig.ReadSelection(selection) + if err != nil { + t.Fatal(err) + } + if got.Backend != dbconfig.SQLite || got.Telemetry.SQLitePath != filepath.Join(root, "data", "meshcore.db") || got.Accounts == nil { + t.Fatal("fresh default or account target wrong") + } + db, err := openSQLite(got.Telemetry.SQLitePath, "rw") + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`INSERT INTO nodes(public_key,name) VALUES('retained','still here')`); err != nil { + t.Fatal(err) + } + db.Close() + before, err := sourceFingerprint(got.Telemetry.SQLitePath) + if err != nil { + t.Fatal(err) + } + t.Setenv("CORESCOPE_DB_BACKEND", "postgres") + t.Setenv("DB_PATH", filepath.Join(root, "obsolete.sqlite")) + out.Reset() + if err := runCommand(context.Background(), args, &out); err != nil { + t.Fatal(err) + } + after, err := dbconfig.ReadSelection(selection) + if err != nil || after.Generation != got.Generation || after.Backend != dbconfig.SQLite { + t.Fatal("update changed selection", err) + } + if current, err := sourceFingerprint(got.Telemetry.SQLitePath); err != nil || current != before { + t.Fatal("ready setup changed working data", err) + } + out.Reset() + args[0] = "-storage-action=init" + if err := runCommand(context.Background(), args, &out); err == nil { + t.Fatal("init replaced existing installation") + } +} + +func TestUnrecordedConfigPathConflictRequiresOverride(t *testing.T) { + t.Setenv("CORESCOPE_DB_BACKEND", "") + t.Setenv("CORESCOPE_DATABASE_URL", "") + root := t.TempDir() + config := filepath.Join(root, "custom.json") + if err := os.WriteFile(config, []byte(`{"dbPath":"one.sqlite","userManagement":{"enabled":false,"dbPath":"saved-users.sqlite"}}`), 0600); err != nil { + t.Fatal(err) + } + t.Setenv("DB_PATH", "two.sqlite") + args := []string{"-storage-action=setup", "-selection-file", filepath.Join(root, "storage-selection.json"), "-config", config, "-offline"} + var out bytes.Buffer + if err := runCommand(context.Background(), args, &out); err == nil { + t.Fatal("conflicting server/ingestor paths silently adopted") + } + chosen := filepath.Join(root, "chosen.sqlite") + accounts := filepath.Join(root, "chosen-users.sqlite") + args = append(args, "-sqlite-path", chosen, "-users-sqlite-path", accounts) + if err := runCommand(context.Background(), args, &out); err != nil { + t.Fatal(err) + } + got, err := dbconfig.ReadSelection(filepath.Join(root, "storage-selection.json")) + if err != nil || got.Telemetry.SQLitePath != chosen || got.Accounts.SQLitePath != accounts { + t.Fatal("explicit target override failed", err) + } +} diff --git a/cmd/migrate/switch.go b/cmd/migrate/switch.go new file mode 100644 index 000000000..d698df217 --- /dev/null +++ b/cmd/migrate/switch.go @@ -0,0 +1,425 @@ +package main + +import ( + "context" + "database/sql" + "errors" + "fmt" + "os" + "path/filepath" + "reflect" + + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/pgutil" + "github.com/meshcore-analyzer/users" +) + +type switchJob struct { + Version int `json:"version"` + ID string `json:"id"` + Source dbconfig.Selection `json:"source"` + Next dbconfig.Selection `json:"next"` +} + +type sqliteFence struct { + read, writer *sql.DB + conn *sql.Conn + path, digest string +} + +func (f *sqliteFence) Close() { + if f.conn != nil { + f.conn.ExecContext(context.Background(), `ROLLBACK`) + f.conn.Close() + } + if f.writer != nil { + f.writer.Close() + } + if f.read != nil { + f.read.Close() + } +} +func (f *sqliteFence) unchanged(ctx context.Context) error { + got, err := fingerprint(ctx, f.path) + if err != nil { + return err + } + if got != f.digest { + return errors.New("original SQLite source changed; keep writers stopped and preserve recovery files") + } + return nil +} + +func fenceSQLite(ctx context.Context, path, kind string) (_ *sqliteFence, err error) { + f := &sqliteFence{path: path} + defer func() { + if err != nil { + f.Close() + } + }() + // The read-only keeper outlives the writer handle, preventing a last-writer + // close from checkpointing the source's pre-existing WAL into its main file. + if f.read, err = openSQLite(path, "ro"); err != nil { + return nil, err + } + f.read.SetMaxOpenConns(1) + if err = dbconfig.AssertSQLiteImportComplete(f.read); err != nil { + return nil, err + } + if kind == "telemetry" { + err = dbschema.AssertSQLiteReady(f.read) + } else { + err = users.AssertSQLiteReady(f.read) + } + if err != nil { + return nil, err + } + if f.digest, err = fingerprint(ctx, path); err != nil { + return nil, err + } + if f.writer, err = openSQLite(path, "rw"); err != nil { + return nil, err + } + f.writer.SetMaxOpenConns(1) + if f.conn, err = f.writer.Conn(ctx); err != nil { + return nil, err + } + if _, err = f.conn.ExecContext(ctx, `PRAGMA busy_timeout=0`); err != nil { + return nil, err + } + if _, err = f.conn.ExecContext(ctx, `BEGIN IMMEDIATE`); err != nil { + return nil, errors.New("SQLite source writer is active; stop all writers before conversion") + } + if err = f.unchanged(ctx); err != nil { + return nil, err + } + return f, nil +} + +func bindOwnerURLs(selected dbconfig.Selection, owner, accounts string) (dbconfig.Storage, error) { + raw := dbconfig.StorageInputs{BaseDir: selected.StateDir, DatabaseURL: owner, UsersDatabaseURL: accounts} + bound, err := selected.ApplyTo(raw) + if err != nil { + return dbconfig.Storage{}, err + } + storage, err := dbconfig.ResolveStorage(bound) + if err != nil { + return storage, err + } + if selected.Backend == dbconfig.Postgres && selected.Accounts != nil && storage.UsersDatabaseURL == "" { + return storage, errors.New("the existing account store requires its PostgreSQL owner URL even when accounts are disabled") + } + return storage, nil +} + +func validateInitialOwnerTargets(ctx context.Context, selected dbconfig.Selection, storage dbconfig.Storage) error { + checks := []struct { + url string + target *dbconfig.PostgresTarget + }{{storage.WriterDatabaseURL, selected.Telemetry.Postgres}} + if selected.Accounts != nil { + checks = append(checks, struct { + url string + target *dbconfig.PostgresTarget + }{storage.UsersDatabaseURL, selected.Accounts.Postgres}) + } + for _, check := range checks { + if check.target == nil || check.url == "" { + return errors.New("initial PostgreSQL owner targets are incomplete") + } + db, err := pgutil.Open(check.url, true) + if err != nil { + return err + } + var database, schema string + err = db.QueryRowContext(ctx, `SELECT current_database(),current_schema()`).Scan(&database, &schema) + db.Close() + if err != nil || database != check.target.Database || schema != check.target.Schema { + return errors.New("initial PostgreSQL database/schema differs from its proposed selection; specify the intended database and search_path explicitly") + } + } + return nil +} + +func runStorageSwitch(ctx context.Context, o storageOptions) error { + state, err := filepath.Abs(o.StateDir) + if err != nil { + return err + } + if o.Action == "abort" { + if o.JobID == "" { + return errors.New("abort requires the exact -job-id") + } + update, err := dbconfig.ResumeSelectionUpdate(o.SelectionFile, o.JobID) + if err != nil { + return err + } + defer update.Close() + return update.Abort() // Staged targets and every recovery copy are preserved. + } + var job switchJob + var update *dbconfig.SelectionUpdate + raw, err := storageActionInputs(o, true) + if err != nil { + return err + } + if o.Action == "switch" { + if o.Backend != "sqlite" && o.Backend != "postgres" { + return errors.New("switch requires an explicit -backend=sqlite or postgres") + } + current, err := dbconfig.ReadSelection(o.SelectionFile) + if err != nil { + return err + } + if string(current.Backend) == o.Backend { + return errors.New("the requested backend is already selected") + } + targetInputs := raw + targetInputs.Backend = dbconfig.Backend(o.Backend) + targetInputs.EnvBackend = targetInputs.Backend + targetInputs.ExistingBackend = "" + targetInputs.StateDir = current.StateDir + if targetInputs.Backend == dbconfig.SQLite { + if o.SQLitePath == "" { + return errors.New("reverse switch requires a new explicit -sqlite-path; retained original files are never overwritten") + } + targetInputs.DBPath = o.SQLitePath + targetInputs.UsersDBPath = o.UsersSQLitePath + } else { + if o.OwnerURL == "" { + return errors.New("PostgreSQL destination requires its owner URL through the private environment") + } + if current.Accounts != nil && o.UsersOwnerURL == "" { + return errors.New("the existing account store must also be converted; configure CORESCOPE_USERS_OWNER_DATABASE_URL") + } + targetInputs.DatabaseURL = o.OwnerURL + targetInputs.ReaderDatabaseURL = "" + targetInputs.WriterDatabaseURL = "" + targetInputs.UsersDatabaseURL = o.UsersOwnerURL + targetInputs.ApprovedChannelsDatabaseURL = "" + } + target, err := dbconfig.ResolveStorage(targetInputs) + if err != nil { + return err + } + if current.Accounts == nil { + target.UsersDBPath = "" + target.UsersDatabaseURL = "" + target.ApprovedChannelsDatabaseURL = "" + } + next, err := dbconfig.NewSelection(target) + if err != nil { + return err + } + if next.Backend == dbconfig.Postgres { + if err := validateInitialOwnerTargets(ctx, next, target); err != nil { + return err + } + } + if next.Backend == dbconfig.SQLite { + paths := []string{next.Telemetry.SQLitePath} + if next.Accounts != nil { + paths = append(paths, next.Accounts.SQLitePath) + } + for _, path := range paths { + if _, err := os.Lstat(path); !errors.Is(err, os.ErrNotExist) { + return errors.New("switch SQLite targets must be fresh absent paths") + } + } + } + update, err = dbconfig.BeginSelectionUpdate(o.SelectionFile, current.Generation) + if err != nil { + return err + } + defer update.Close() + job = switchJob{Version: 1, ID: update.ID(), Source: current, Next: next} + if err := update.Stage(next); err != nil { + return err + } + dir := filepath.Join(state, job.ID) + if err := os.MkdirAll(dir, 0700); err != nil { + return err + } + if err := writePrivateJSON(filepath.Join(dir, "job.json"), job); err != nil { + return err + } + } else { + if o.JobID == "" { + return errors.New("resume requires the exact -job-id") + } + update, err = dbconfig.ResumeSelectionUpdate(o.SelectionFile, o.JobID) + if err != nil { + return err + } + defer update.Close() + if update.Target().Generation == "" { + return errors.New("pending job has no staged destination; abort this metadata-only update and start a new job") + } + if accountInitialization(update.Current(), update.Target()) { + return finishAccountSetup(ctx, o, update, raw, true) + } + if err := readPrivateJSON(filepath.Join(state, o.JobID, "job.json"), &job); err != nil { + return errors.New("pending switch has no complete readable job manifest; preserve any recovery files, abort the uncommitted update and start a new job") + } + if job.Version != 1 || job.ID != o.JobID || !reflect.DeepEqual(job.Source, update.Current()) || !reflect.DeepEqual(job.Next, update.Target()) || job.Next.StateDir != job.Source.StateDir || job.Next.Backend == job.Source.Backend { + return errors.New("switch recovery manifest does not match the pending installation generation") + } + } + dir := filepath.Join(state, job.ID) + ownerSelection := job.Next + if job.Source.Backend == dbconfig.Postgres { + ownerSelection = job.Source + } + owners, err := bindOwnerURLs(ownerSelection, o.OwnerURL, o.UsersOwnerURL) + if err != nil { + return err + } + if ownerSelection.Accounts != nil { + same, err := pgutil.SameDatabase(ctx, owners.WriterDatabaseURL, owners.UsersDatabaseURL) + if err != nil { + return err + } + if same { + return errors.New("telemetry and account databases must remain separate") + } + } + var sqliteSources []*sqliteFence + var postgresSources []*postgresFence + stores := []struct { + kind string + source dbconfig.Target + target dbconfig.Target + url string + }{{"telemetry", job.Source.Telemetry, job.Next.Telemetry, owners.WriterDatabaseURL}} + if job.Source.Accounts != nil { + if job.Next.Accounts == nil { + return errors.New("switch manifest would omit an existing account store") + } + stores = append(stores, struct { + kind string + source, target dbconfig.Target + url string + }{"accounts", *job.Source.Accounts, *job.Next.Accounts, owners.UsersDatabaseURL}) + } + // Fence both sources before copying either one. Every defer runs on all + // failure paths; the durable selection journal remains to block startup. + for _, store := range stores { + if job.Source.Backend == dbconfig.SQLite { + f, err := fenceSQLite(ctx, store.source.SQLitePath, store.kind) + if err != nil { + return err + } + sqliteSources = append(sqliteSources, f) + defer f.Close() + } else { + f, err := fencePostgres(ctx, store.url, store.kind) + if err != nil { + return err + } + postgresSources = append(postgresSources, f) + defer f.Close() + } + } + var reports []importReport + for i, store := range stores { + var report importReport + if job.Source.Backend == dbconfig.SQLite { + resume := o.Action == "resume" + if resume { + if _, err := os.Stat(filepath.Join(dir, store.kind, "manifest.json")); errors.Is(err, os.ErrNotExist) { + resume = false + } + } + report, err = importSQLite(ctx, importOptions{Source: store.source.SQLitePath, DatabaseURL: store.url, StateDir: dir, Kind: store.kind, Resume: resume}) + } else { + resume := o.Action == "resume" + if resume { + if _, err := os.Stat(filepath.Join(dir, store.kind, "reverse.json")); errors.Is(err, os.ErrNotExist) { + resume = false + } + } + report, err = reversePostgres(ctx, reverseOptions{DatabaseURL: store.url, Destination: store.target.SQLitePath, StateDir: dir, Kind: store.kind, Resume: resume, source: postgresSources[i], keepPending: true}) + } + if err != nil { + return err + } + if !report.Verified { + return errors.New("store copy returned without complete verification") + } + reports = append(reports, report) + if o.afterStore != nil { + if err := o.afterStore(store.kind); err != nil { + return err + } + } + } + for _, source := range sqliteSources { + if err := source.unchanged(ctx); err != nil { + return err + } + } + verification := struct { + JobID string `json:"job_id"` + Reports []importReport `json:"reports"` + }{job.ID, reports} + if err := writePrivateJSON(filepath.Join(dir, "verified.json"), verification); err != nil { + if !errors.Is(err, os.ErrExist) { + return err + } + previous := verification + previous.Reports = nil + if err := readPrivateJSON(filepath.Join(dir, "verified.json"), &previous); err != nil { + return err + } + if !reflect.DeepEqual(previous, verification) { + return errors.New("saved verification receipt differs from the reverified stores") + } + } + if job.Next.Backend == dbconfig.Postgres { + for _, store := range stores { + if _, err := finalizeImport(ctx, store.url, store.kind); err != nil { + return err + } + } + if err := grantRuntimeTargets(ctx, job.Next, owners, raw); err != nil { + return err + } + } else { + targets := []struct{ kind, path string }{{"telemetry", job.Next.Telemetry.SQLitePath}} + if job.Next.Accounts != nil { + targets = append(targets, struct{ kind, path string }{"accounts", job.Next.Accounts.SQLitePath}) + } + for _, target := range targets { + db, err := openSQLite(target.path, "rw") + if err != nil { + return err + } + _, err = db.ExecContext(ctx, `DROP TABLE IF EXISTS corescope_reverse_progress`) + if err == nil { + err = assertSQLiteData(ctx, db, target.kind) + } + db.Close() + if err != nil { + return err + } + } + } + if err := validateRuntimeSelection(ctx, job.Next, raw); err != nil { + return err + } + if o.beforeCommit != nil { + if err := o.beforeCommit(); err != nil { + return err + } + } + for _, source := range sqliteSources { + if err := source.unchanged(ctx); err != nil { + return err + } + } + if _, err := update.Commit(job.Next); err != nil { + return fmt.Errorf("selection commit needs recovery for job %s: %w", job.ID, err) + } + return nil +} diff --git a/cmd/migrate/switch_test.go b/cmd/migrate/switch_test.go new file mode 100644 index 000000000..eb2c79b68 --- /dev/null +++ b/cmd/migrate/switch_test.go @@ -0,0 +1,454 @@ +package main + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "net/url" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/pgutil" +) + +func restrictedRole(t *testing.T, owner string) string { + t.Helper() + db := openImportDB(t, owner) + var random [12]byte + if _, err := rand.Read(random[:]); err != nil { + t.Fatal(err) + } + role := "corescope_convert_" + hex.EncodeToString(random[:]) + password := "test_" + hex.EncodeToString(random[:]) + if _, err := db.Exec(`CREATE ROLE ` + quote(role) + ` LOGIN PASSWORD '` + password + `'`); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if _, err := db.Exec(`DROP OWNED BY ` + quote(role) + `;DROP ROLE ` + quote(role)); err != nil { + t.Error(err) + } + }) + u, err := url.Parse(owner) + if err != nil { + t.Fatal("invalid test owner URL") + } + u.User = url.UserPassword(role, password) + return u.String() +} + +func switchFixture(t *testing.T) (storageOptions, dbconfig.Selection) { + t.Helper() + telemetry, accounts := postgresDatabase(t), postgresDatabase(t) + t.Setenv("CORESCOPE_DATABASE_URL", telemetry) + t.Setenv("CORESCOPE_USERS_OWNER_DATABASE_URL", accounts) + t.Setenv("CORESCOPE_READER_DATABASE_URL", restrictedRole(t, telemetry)) + t.Setenv("CORESCOPE_WRITER_DATABASE_URL", restrictedRole(t, telemetry)) + t.Setenv("CORESCOPE_USERS_DATABASE_URL", restrictedRole(t, accounts)) + t.Setenv("CORESCOPE_APPROVED_CHANNELS_DATABASE_URL", restrictedRole(t, accounts)) + t.Setenv("CORESCOPE_DB_BACKEND", "") + t.Setenv("DB_PATH", "") + source, account := telemetrySource(t), accountSource(t, 5) + if err := applySQLiteTarget(context.Background(), source, "telemetry"); err != nil { + t.Fatal(err) + } + if err := applySQLiteTarget(context.Background(), account, "accounts"); err != nil { + t.Fatal(err) + } + dir := t.TempDir() + selected, err := dbconfig.NewSelection(dbconfig.Storage{Backend: dbconfig.SQLite, StateDir: dir, DBPath: source, UsersDBPath: account}) + if err != nil { + t.Fatal(err) + } + selection := dbconfig.SelectionPath(dir) + if _, err := dbconfig.AdoptSelection(selection, selected); err != nil { + t.Fatal(err) + } + return storageOptions{Action: "switch", SelectionFile: selection, Backend: "postgres", OwnerURL: telemetry, UsersOwnerURL: accounts, StateDir: t.TempDir(), Offline: true, ConfigDir: t.TempDir()}, selected +} + +func TestVerifiedTwoStoreSwitchRoundTripAndRuntimeGrants(t *testing.T) { + o, source := switchFixture(t) + ctx := context.Background() + before, err := sourceFingerprint(source.Telemetry.SQLitePath) + if err != nil { + t.Fatal(err) + } + beforeAccount, err := sourceFingerprint(source.Accounts.SQLitePath) + if err != nil { + t.Fatal(err) + } + var out bytes.Buffer + if err := runStorageAction(ctx, o, &out); err != nil { + t.Fatal("forward switch", err) + } + selected, err := dbconfig.ReadSelection(o.SelectionFile) + if err != nil || selected.Backend != dbconfig.Postgres { + t.Fatal("PG selection", err) + } + if got, err := sourceFingerprint(source.Telemetry.SQLitePath); err != nil || got != before { + t.Fatal("telemetry source changed", err) + } + if got, err := sourceFingerprint(source.Accounts.SQLitePath); err != nil || got != beforeAccount { + t.Fatal("account source changed", err) + } + for _, pair := range [][2]string{{"CORESCOPE_WRITER_DATABASE_URL", "corescope_schema"}, {"CORESCOPE_USERS_DATABASE_URL", "schema_version"}} { + db, err := pgutil.Open(os.Getenv(pair[0]), false) + if err != nil { + t.Fatal(err) + } + var allowed bool + if err := db.QueryRow(`SELECT has_table_privilege($1,'INSERT,UPDATE,DELETE,TRUNCATE')`, pair[1]).Scan(&allowed); err != nil || allowed { + t.Fatal("metadata DML granted", allowed, err) + } + db.Close() + } + channel, err := pgutil.Open(os.Getenv("CORESCOPE_APPROVED_CHANNELS_DATABASE_URL"), true) + if err != nil { + t.Fatal(err) + } + if _, err := channel.Exec(`SELECT password_hash FROM users`); err == nil { + t.Fatal("projection reader can access account secrets") + } + channel.Close() + o.Backend = "sqlite" + o.SQLitePath = filepath.Join(t.TempDir(), "telemetry.sqlite") + o.UsersSQLitePath = filepath.Join(t.TempDir(), "accounts.sqlite") + out.Reset() + if err := runStorageAction(ctx, o, &out); err != nil { + t.Fatal("reverse switch", err) + } + selected, err = dbconfig.ReadSelection(o.SelectionFile) + if err != nil || selected.Backend != dbconfig.SQLite || selected.StateDir != source.StateDir || selected.Telemetry.SQLitePath != o.SQLitePath { + t.Fatal("reverse selection", err) + } + db, err := openSQLite(o.UsersSQLitePath, "ro") + if err != nil { + t.Fatal(err) + } + defer db.Close() + var id int + if err := db.QueryRow(`SELECT id FROM mail_events WHERE mail_id=4`).Scan(&id); err != nil || id != 900 { + t.Fatal("round trip event", id, err) + } + if strings.Contains(out.String(), "postgres://") { + t.Fatal("CLI leaked credentials") + } +} + +func TestSwitchFailureKeepsOldSelectionAndResumeVerifiesAgain(t *testing.T) { + o, source := switchFixture(t) + ctx := context.Background() + o.afterStore = func(kind string) error { + if kind == "telemetry" { + return errors.New("injected failure before accounts") + } + return nil + } + if err := runStorageAction(ctx, o, &bytes.Buffer{}); err == nil { + t.Fatal("partial switch succeeded") + } + status, err := dbconfig.InspectSelection(o.SelectionFile) + if err != nil || status.State != "pending" || status.Backend != dbconfig.SQLite || status.Generation != source.Generation { + t.Fatal("partial switch selected target", status, err) + } + if _, _, err := dbconfig.OpenSelection(o.SelectionFile); !errors.Is(err, dbconfig.ErrSelectionInProgress) { + t.Fatal("partial switch allowed runtime", err) + } + o.Action = "resume" + o.JobID = status.JobID + o.afterStore = nil + o.beforeCommit = func() error { return errors.New("injected cutover failure") } + if err := runStorageAction(ctx, o, &bytes.Buffer{}); err == nil { + t.Fatal("cutover failure passed") + } + status, err = dbconfig.InspectSelection(o.SelectionFile) + if err != nil || status.Generation != source.Generation || status.State != "pending" { + t.Fatal("failed cutover changed selection", err) + } + o.beforeCommit = nil + if err := runStorageAction(ctx, o, &bytes.Buffer{}); err != nil { + t.Fatal("verified resume", err) + } + final, err := dbconfig.ReadSelection(o.SelectionFile) + if err != nil || final.Backend != dbconfig.Postgres { + t.Fatal("resume failed to select", err) + } +} + +func TestSQLiteSourceFencePreservesWALAndRejectsWriters(t *testing.T) { + source := telemetrySource(t) + ctx := context.Background() + if err := applySQLiteTarget(ctx, source, "telemetry"); err != nil { + t.Fatal(err) + } + writer, err := openSQLite(source, "rw") + if err != nil { + t.Fatal(err) + } + if _, err := writer.Exec(`PRAGMA journal_mode=WAL; PRAGMA wal_autocheckpoint=0; INSERT INTO observers(id) VALUES('wal-retained')`); err != nil { + t.Fatal(err) + } + keeper, err := openSQLite(source, "ro") + if err != nil { + t.Fatal(err) + } + defer keeper.Close() + if err := keeper.Ping(); err != nil { + t.Fatal(err) + } + writer.Close() + before, err := sourceFingerprint(source) + if err != nil { + t.Fatal(err) + } + f, err := fenceSQLite(ctx, source, "telemetry") + if err != nil { + t.Fatal(err) + } + other, err := openSQLite(source, "rw") + if err != nil { + t.Fatal(err) + } + if _, err := other.Exec(`PRAGMA busy_timeout=1; INSERT INTO observers(id) VALUES('must-not-write')`); err == nil { + t.Fatal("source accepted a concurrent writer") + } + other.Close() + f.Close() + keeper.Close() + after, err := sourceFingerprint(source) + if err != nil || after != before { + t.Fatal("source fence checkpointed or modified original WAL", err) + } +} + +func TestSwitchSourceMutationAndDestinationSubstitutionRefuse(t *testing.T) { + o, source := switchFixture(t) + ctx := context.Background() + o.afterStore = func(string) error { return errors.New("stop after first store") } + if err := runStorageAction(ctx, o, &bytes.Buffer{}); err == nil { + t.Fatal("expected interruption") + } + status, err := dbconfig.InspectSelection(o.SelectionFile) + if err != nil { + t.Fatal(err) + } + var job switchJob + jobPath := filepath.Join(o.StateDir, status.JobID, "job.json") + if err := readPrivateJSON(jobPath, &job); err != nil { + t.Fatal(err) + } + original := job + job.Next.Telemetry.Postgres = &dbconfig.PostgresTarget{Host: "substituted.invalid", Port: 5432, Database: "other", Schema: "public"} + data, err := json.Marshal(job) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(jobPath, data, 0600); err != nil { + t.Fatal(err) + } + o.Action = "resume" + o.JobID = status.JobID + o.afterStore = nil + if err := runStorageAction(ctx, o, &bytes.Buffer{}); err == nil || !strings.Contains(err.Error(), "manifest") { + t.Fatal("changed target accepted", err) + } + data, err = json.Marshal(original) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(jobPath, data, 0600); err != nil { + t.Fatal(err) + } + changed, err := openSQLite(source.Telemetry.SQLitePath, "rw") + if err != nil { + t.Fatal(err) + } + if _, err := changed.Exec(`UPDATE observations SET score=7.5 WHERE id=90`); err != nil { + t.Fatal(err) + } + changed.Close() + if err := runStorageAction(ctx, o, &bytes.Buffer{}); err == nil { + t.Fatal("changed original source accepted") + } + o.Action = "abort" + if err := runStorageAction(ctx, o, &bytes.Buffer{}); err != nil { + t.Fatal("abort", err) + } + selected, err := dbconfig.ReadSelection(o.SelectionFile) + if err != nil || selected.Generation != source.Generation { + t.Fatal("abort changed original selection", err) + } + if _, err := os.Stat(filepath.Join(o.StateDir, status.JobID, "telemetry", "recovery.sqlite")); err != nil { + t.Fatal("abort removed recovery", err) + } +} + +func TestSetupPostgresCannotAbandonExistingSQLite(t *testing.T) { + owner := postgresDatabase(t) + t.Setenv("CORESCOPE_DATABASE_URL", owner) + t.Setenv("CORESCOPE_USERS_DATABASE_URL", "") + t.Setenv("CORESCOPE_USERS_OWNER_DATABASE_URL", "") + t.Setenv("CORESCOPE_APPROVED_CHANNELS_DATABASE_URL", "") + t.Setenv("CORESCOPE_READER_DATABASE_URL", "") + t.Setenv("CORESCOPE_WRITER_DATABASE_URL", "") + t.Setenv("CORESCOPE_DB_BACKEND", "sqlite") + source := telemetrySource(t) + t.Setenv("DB_PATH", source) + before, err := sourceFingerprint(source) + if err != nil { + t.Fatal(err) + } + dir := t.TempDir() + config := filepath.Join(dir, "config.json") + if err := os.WriteFile(config, []byte(`{"db":{"backend":"sqlite"}}`), 0600); err != nil { + t.Fatal(err) + } + args := []string{"-storage-action=setup", "-backend=postgres", "-selection-file", filepath.Join(dir, "storage-selection.json"), "-config", config, "-offline"} + err = runCommand(context.Background(), args, &bytes.Buffer{}) + if err == nil || !strings.Contains(err.Error(), "primary SQLite data") { + t.Fatal("fresh PG abandoned source or explicit choice did not override hints", err) + } + db := openImportDB(t, owner) + var n int + if err := db.QueryRow(importDestinationObjects).Scan(&n); err != nil || n != 0 { + t.Fatal("empty PG target was initialized", n, err) + } + if after, err := sourceFingerprint(source); err != nil || before != after { + t.Fatal("SQLite source was changed", err) + } +} + +func TestFreshPostgresSetupOverridesBootstrapDefaultAndGrantsBeforeSelection(t *testing.T) { + owner, accounts := postgresDatabase(t), postgresDatabase(t) + t.Setenv("CORESCOPE_DATABASE_URL", owner) + t.Setenv("CORESCOPE_USERS_OWNER_DATABASE_URL", accounts) + t.Setenv("CORESCOPE_READER_DATABASE_URL", restrictedRole(t, owner)) + t.Setenv("CORESCOPE_WRITER_DATABASE_URL", restrictedRole(t, owner)) + t.Setenv("CORESCOPE_USERS_DATABASE_URL", restrictedRole(t, accounts)) + t.Setenv("CORESCOPE_APPROVED_CHANNELS_DATABASE_URL", restrictedRole(t, accounts)) + t.Setenv("CORESCOPE_DB_BACKEND", "sqlite") + t.Setenv("DB_PATH", "") + dir := t.TempDir() + config := filepath.Join(dir, "config.json") + if err := os.WriteFile(config, []byte(`{"db":{"backend":"sqlite"},"userManagement":{"enabled":false}}`), 0600); err != nil { + t.Fatal(err) + } + selection := filepath.Join(dir, "storage-selection.json") + args := []string{"-storage-action=setup", "-backend=postgres", "-selection-file", selection, "-config", config, "-offline"} + var out bytes.Buffer + if err := runCommand(context.Background(), args, &out); err != nil { + t.Fatal(err) + } + selected, err := dbconfig.ReadSelection(selection) + if err != nil || selected.Backend != dbconfig.Postgres || selected.Accounts == nil { + t.Fatal("explicit PG setup", err) + } + if bytes.Contains(out.Bytes(), []byte("postgres://")) { + t.Fatal("owner credentials in setup output") + } + // Updates retain PG although the unchanged bootstrap config/env still say SQLite. + args = []string{"-storage-action=setup", "-selection-file", selection, "-config", config, "-offline"} + if err := runCommand(context.Background(), args, &bytes.Buffer{}); err != nil { + t.Fatal("PG update", err) + } + after, err := dbconfig.ReadSelection(selection) + if err != nil || after.Generation != selected.Generation { + t.Fatal("update replaced installation", err) + } +} + +func TestInitialAdoptionRefusesImplicitRoleSchemaDrift(t *testing.T) { + owner := postgresDatabase(t) + config, err := pgutil.ParseConfig(owner) + if err != nil { + t.Fatal(err) + } + db := openImportDB(t, owner) + if _, err := db.Exec(`CREATE SCHEMA ` + quote(config.User)); err != nil { + t.Fatal(err) + } + t.Setenv("CORESCOPE_DATABASE_URL", owner) + t.Setenv("CORESCOPE_USERS_OWNER_DATABASE_URL", "") + t.Setenv("CORESCOPE_USERS_DATABASE_URL", "") + t.Setenv("CORESCOPE_APPROVED_CHANNELS_DATABASE_URL", "") + t.Setenv("CORESCOPE_READER_DATABASE_URL", "") + t.Setenv("CORESCOPE_WRITER_DATABASE_URL", "") + t.Setenv("DB_PATH", "") + dir := t.TempDir() + selection := filepath.Join(dir, "storage-selection.json") + err = runStorageAction(context.Background(), storageOptions{Action: "setup", Backend: "postgres", OwnerURL: owner, Offline: true, SelectionFile: selection, ConfigDir: dir}, &bytes.Buffer{}) + if err == nil || !strings.Contains(err.Error(), "database/schema differs") { + t.Fatal("implicit owner schema drift was accepted", err) + } + if _, err := os.Stat(selection); !os.IsNotExist(err) { + t.Fatal("ambiguous schema was selected", err) + } + var tables int + if err := db.QueryRow(`SELECT count(*) FROM information_schema.tables WHERE table_schema NOT IN ('pg_catalog','information_schema')`).Scan(&tables); err != nil || tables != 0 { + t.Fatal("ambiguous schema bootstrap wrote tables", tables, err) + } +} + +func TestReverseWithoutAccountSourceResumesAfterCutoverFailure(t *testing.T) { + owner := postgresDatabase(t) + db := openImportDB(t, owner) + if err := dbschema.ApplyPostgres(db, nil); err != nil { + t.Fatal(err) + } + t.Setenv("CORESCOPE_DATABASE_URL", owner) + t.Setenv("CORESCOPE_USERS_OWNER_DATABASE_URL", "") + t.Setenv("CORESCOPE_USERS_DATABASE_URL", "") + t.Setenv("CORESCOPE_APPROVED_CHANNELS_DATABASE_URL", "") + dir := t.TempDir() + selected, err := dbconfig.NewSelection(dbconfig.Storage{Backend: dbconfig.Postgres, StateDir: dir, WriterDatabaseURL: owner}) + if err != nil { + t.Fatal(err) + } + selection := dbconfig.SelectionPath(dir) + if _, err := dbconfig.AdoptSelection(selection, selected); err != nil { + t.Fatal(err) + } + o := storageOptions{Action: "switch", Backend: "sqlite", Offline: true, SelectionFile: selection, OwnerURL: owner, StateDir: t.TempDir(), ConfigDir: t.TempDir(), SQLitePath: filepath.Join(t.TempDir(), "telemetry.sqlite"), UsersSQLitePath: filepath.Join(t.TempDir(), "accounts.sqlite"), beforeCommit: func() error { return errors.New("cutover interrupted") }} + if err := runStorageAction(context.Background(), o, &bytes.Buffer{}); err == nil { + t.Fatal("injected failure passed") + } + status, err := dbconfig.InspectSelection(selection) + if err != nil || status.State != "pending" { + t.Fatal("pending job absent", err) + } + o.Action = "resume" + o.JobID = status.JobID + o.beforeCommit = nil + if err := runStorageAction(context.Background(), o, &bytes.Buffer{}); err != nil { + t.Fatal("empty optional account resume", err) + } + after, err := dbconfig.ReadSelection(selection) + if err != nil || after.Backend != dbconfig.SQLite || after.Accounts != nil { + t.Fatal("resume did not preserve uninitialized accounts", err) + } + if _, err := os.Stat(o.UsersSQLitePath); !os.IsNotExist(err) { + t.Fatal("reverse switch invented an account database", err) + } + newOwner := postgresDatabase(t) + t.Setenv("CORESCOPE_DATABASE_URL", newOwner) + t.Setenv("CORESCOPE_READER_DATABASE_URL", restrictedRole(t, newOwner)) + t.Setenv("CORESCOPE_WRITER_DATABASE_URL", restrictedRole(t, newOwner)) + o.Action = "switch" + o.Backend = "postgres" + o.OwnerURL = newOwner + o.JobID = "" + if err := runStorageAction(context.Background(), o, &bytes.Buffer{}); err != nil { + t.Fatal("switch nil accounts forward", err) + } + after, err = dbconfig.ReadSelection(selection) + if err != nil || after.Backend != dbconfig.Postgres || after.Accounts != nil { + t.Fatal("forward switch changed account absence", err) + } +} diff --git a/cmd/migrate/target_identity_test.go b/cmd/migrate/target_identity_test.go new file mode 100644 index 000000000..af4325690 --- /dev/null +++ b/cmd/migrate/target_identity_test.go @@ -0,0 +1,62 @@ +package main + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "net/url" + "os" + "path/filepath" + "testing" + + "github.com/meshcore-analyzer/pgutil" +) + +func TestImportIdentityUsesConfiguredEndpoint(t *testing.T) { + dsn := postgresSchema(t) + u, err := url.Parse(dsn) + if err != nil { + t.Fatal("parse test URL") + } + // Connect through the stable name instead of its transient resolved address. + u.Host = "localhost:" + u.Port() + o := importOptions{Source: accountSource(t, 5), DatabaseURL: u.String(), StateDir: t.TempDir(), Kind: "accounts"} + if _, err := importSQLite(context.Background(), o); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(filepath.Join(o.StateDir, o.Kind, "manifest.json")) + if err != nil { + t.Fatal(err) + } + var manifest sourceManifest + if err := json.Unmarshal(data, &manifest); err != nil { + t.Fatal(err) + } + db := openImportDB(t, o.DatabaseURL) + var database, schema string + var databaseOID, schemaOID int64 + if err := db.QueryRow(`SELECT current_database(),current_schema(),(SELECT oid::bigint FROM pg_database WHERE datname=current_database()),(SELECT oid::bigint FROM pg_namespace WHERE nspname=current_schema())`).Scan(&database, &schema, &databaseOID, &schemaOID); err != nil { + t.Fatal(err) + } + config, err := pgutil.ParseConfig(o.DatabaseURL) + if err != nil { + t.Fatal(err) + } + identity, _ := json.Marshal([]any{database, schema, databaseOID, schemaOID, config.Host, config.Port}) + want := sha256.Sum256(identity) + if manifest.TargetDigest != hex.EncodeToString(want[:]) { + t.Fatal("manifest identity used a transient resolved address instead of the configured endpoint") + } + o.Resume = true + if _, err := importSQLite(context.Background(), o); err != nil { + t.Fatalf("same configured endpoint did not resume: %v", err) + } + // The IP alias reaches the same database/OIDs but is a different configured + // endpoint. Operators must retain their original endpoint when resuming. + u.Host = "127.0.0.1:" + u.Port() + o.DatabaseURL = u.String() + if _, err := importSQLite(context.Background(), o); err == nil { + t.Fatal("changed configured endpoint accepted") + } +} diff --git a/cmd/server/account_export.go b/cmd/server/account_export.go new file mode 100644 index 000000000..9169f0ada --- /dev/null +++ b/cmd/server/account_export.go @@ -0,0 +1,200 @@ +package main + +import ( + "encoding/json" + "fmt" + "log" + "net/http" + "time" + + "github.com/meshcore-analyzer/users" +) + +// accountExport is GET /api/account/export: everything users.db holds about +// the logged-in account (docs/specs/2026-10-08-account-export-and-users-backup-design.md). +// Credentials are left out on purpose: the password hash, session, link and +// unsubscribe tokens; of a pending email change only the new address is +// exported. Other accounts appear as ids only (activatedBy, audit). +type accountExport struct { + FormatVersion int `json:"formatVersion"` + ExportedAt string `json:"exportedAt"` + Instance string `json:"instance"` + Profile exportProfile `json:"profile"` + Sessions []exportSession `json:"sessions"` + Settings *exportSettings `json:"settings"` + Proposals []exportProposal `json:"proposals"` + Notifications exportNotifications `json:"notifications"` + Audit []exportAuditEntry `json:"audit"` + Mail []exportMail `json:"mail"` +} + +type exportProfile struct { + ID int64 `json:"id"` + Email string `json:"email"` + PendingEmail *string `json:"pendingEmail"` // unconfirmed email change: the address only + DisplayName string `json:"displayName"` + Role users.Role `json:"role"` + Status users.Status `json:"status"` + CreatedAt string `json:"createdAt"` + LastLoginAt *string `json:"lastLoginAt"` + EmailBouncing bool `json:"emailBouncing"` + ActivatedAt *string `json:"activatedAt"` + ActivatedBy *int64 `json:"activatedBy"` +} + +type exportSession struct { + CreatedAt string `json:"createdAt"` + LastSeenAt string `json:"lastSeenAt"` + UserAgent string `json:"userAgent,omitempty"` +} + +type exportSettings struct { + Revision int64 `json:"revision"` + UpdatedAt string `json:"updatedAt"` + Doc settingsDoc `json:"doc"` +} + +type exportProposal struct { + Kind string `json:"kind"` + Subject string `json:"subject"` + Status users.ProposalStatus `json:"status"` + Note string `json:"note"` + CreatedAt string `json:"createdAt"` + DecidedAt *string `json:"decidedAt"` +} + +type exportNotifications struct { + Prefs *exportNotifyPrefs `json:"prefs"` + Watches []exportWatch `json:"watches"` +} + +type exportNotifyPrefs struct { + Enabled bool `json:"enabled"` + Events []string `json:"events"` +} + +type exportWatch struct { + Pubkey string `json:"pubkey"` + CreatedAt string `json:"createdAt"` +} + +type exportAuditEntry struct { + At string `json:"at"` + Action string `json:"action"` + ActorUserID *int64 `json:"actorUserId"` + TargetUserID *int64 `json:"targetUserId"` + Detail map[string]string `json:"detail"` +} + +type exportMail struct { + To string `json:"to"` + Purpose string `json:"purpose"` + SentAt string `json:"sentAt"` + Status string `json:"status"` + StatusAt string `json:"statusAt"` + Reason string `json:"reason,omitempty"` + Events []mailEventJSON `json:"events"` +} + +// buildAccountExport reads every section for u. It only reads users.db. +func (a *authService) buildAccountExport(u *users.User, now time.Time) (*accountExport, error) { + x := &accountExport{ + FormatVersion: 1, + ExportedAt: rfc3339(now), + Instance: a.set.baseURL.String(), + Profile: exportProfile{ID: u.ID, Email: u.Email, DisplayName: u.DisplayName, Role: u.Role, Status: u.Status, + CreatedAt: rfc3339(u.CreatedAt), LastLoginAt: rfc3339Ptr(u.LastLoginAt), EmailBouncing: u.EmailBouncing, + ActivatedAt: rfc3339Ptr(u.ActivatedAt), ActivatedBy: u.ActivatedBy}, + Sessions: []exportSession{}, + Proposals: []exportProposal{}, + Notifications: exportNotifications{Watches: []exportWatch{}}, + Audit: []exportAuditEntry{}, + Mail: []exportMail{}, + } + pending, err := a.st.PendingEmailChange(u.ID) + if err != nil { + return nil, fmt.Errorf("pending email: %w", err) + } + if pending != "" { + x.Profile.PendingEmail = &pending + } + sessions, err := a.st.ListSessions(u.ID) + if err != nil { + return nil, fmt.Errorf("sessions: %w", err) + } + for _, s := range sessions { + x.Sessions = append(x.Sessions, exportSession{CreatedAt: rfc3339(s.CreatedAt), LastSeenAt: rfc3339(s.LastSeenAt), UserAgent: s.UserAgent}) + } + rec, err := a.st.SettingsRecordFor(u.ID) + if err != nil { + return nil, fmt.Errorf("settings: %w", err) + } + if rec != nil { + var doc settingsDoc + if err := json.Unmarshal([]byte(rec.Doc), &doc); err != nil { + return nil, fmt.Errorf("settings decode (%d bytes): %w", len(rec.Doc), err) + } + x.Settings = &exportSettings{Revision: rec.Version.Revision, UpdatedAt: rfc3339(rec.UpdatedAt), Doc: doc} + } + props, err := a.st.AllProposalsByUser(u.ID) + if err != nil { + return nil, fmt.Errorf("proposals: %w", err) + } + for _, p := range props { + x.Proposals = append(x.Proposals, exportProposal{Kind: p.Kind, Subject: p.Subject, Status: p.Status, Note: p.Note, + CreatedAt: rfc3339(p.CreatedAt), DecidedAt: rfc3339Ptr(p.DecidedAt)}) + } + prefs, err := a.st.StoredNotifyPrefs(u.ID) + if err != nil { + return nil, fmt.Errorf("notification prefs: %w", err) + } + if prefs != nil { + x.Notifications.Prefs = &exportNotifyPrefs{Enabled: prefs.Enabled, Events: prefs.Events} + } + watches, err := a.st.WatchesFor(u.ID) + if err != nil { + return nil, fmt.Errorf("watches: %w", err) + } + for _, w := range watches { + x.Notifications.Watches = append(x.Notifications.Watches, exportWatch{Pubkey: w.Pubkey, CreatedAt: rfc3339(w.CreatedAt)}) + } + entries, err := a.st.AuditAllFor(u.ID) + if err != nil { + return nil, fmt.Errorf("audit: %w", err) + } + for _, e := range entries { + x.Audit = append(x.Audit, exportAuditEntry{At: rfc3339(e.At), Action: e.Action, + ActorUserID: e.ActorUserID, TargetUserID: e.TargetUserID, Detail: e.Detail}) + } + mails, err := a.st.MailAllForUser(u.ID) + if err != nil { + return nil, fmt.Errorf("mail: %w", err) + } + for _, m := range mails { + em := exportMail{To: m.ToEmail, Purpose: m.Purpose, SentAt: rfc3339(m.SentAt), Status: m.LastEvent, + StatusAt: rfc3339(m.LastEventAt), Reason: m.LastReason, Events: []mailEventJSON{}} + for _, e := range m.Events { + em.Events = append(em.Events, mailEventJSON{Event: e.Event, At: rfc3339(e.At), Reason: e.Reason}) + } + x.Mail = append(x.Mail, em) + } + return x, nil +} + +// handleAccountExport answers the export as a download. GET, so a plain +// link works and no CSRF token is needed. The user.export audit row is +// written after the export is built, so the file shows the state before it. +func (s *Server) handleAccountExport(w http.ResponseWriter, _ *http.Request, u *users.User, _ *users.Session) { + a := s.auth + now := time.Now().UTC() + x, err := a.buildAccountExport(u, now) + if err != nil { + log.Printf("[users] export for user #%d: %v", u.ID, err) + writeError(w, http.StatusInternalServerError, "internal error") + return + } + a.audit(idPtr(u.ID), "user.export", idPtr(u.ID), nil) + w.Header().Set("Content-Disposition", `attachment; filename="corescope-account-`+now.Format("2006-01-02")+`.json"`) + w.Header().Set("Cache-Control", "no-store") + writeJSON(w, x) +} diff --git a/cmd/server/account_export_test.go b/cmd/server/account_export_test.go new file mode 100644 index 000000000..440fec438 --- /dev/null +++ b/cmd/server/account_export_test.go @@ -0,0 +1,241 @@ +package main + +import ( + "regexp" + "strings" + "testing" + "time" + + "github.com/meshcore-analyzer/users" +) + +const exportPubkey = "aabbccddeeff00112233445566778899aabbccddeeff00112233445566778899" + +var exportFilenameRE = regexp.MustCompile(`^attachment; filename="corescope-account-\d{4}-\d{2}-\d{2}\.json"$`) + +func TestAccountExportRequiresSession(t *testing.T) { + f := newAuthFixture(t) + expectStatus(t, f.do("GET", "/api/account/export", nil), 401) +} + +func TestAccountExportSections(t *testing.T) { + f, boss, uma := adminFixture(t) + st, uid, bossID := f.st, uma.me.ID, boss.me.ID + if _, err := st.PutSettings(uid, users.SettingsVersion{}, `{"v":1,"keys":{"meshcore-theme":"dark"}}`); err != nil { + t.Fatal(err) + } + if _, err := st.Propose(users.KindHashtagChannel, "#export-test", uid, users.ProposalLimits{}); err != nil { + t.Fatal(err) + } + prefs, err := st.SetNotifyPrefs(uid, true, []string{users.NotifyNodeOffline}) + if err != nil { + t.Fatal(err) + } + if err := st.AddWatch(uid, exportPubkey, 10); err != nil { + t.Fatal(err) + } + if err := st.Audit(&bossID, "user.role", &uid, map[string]string{"role": "user"}); err != nil { + t.Fatal(err) + } + if _, err := st.LogMail(&uid, "uma@example.org", "reset", ""); err != nil { + t.Fatal(err) + } + if _, _, err := st.RecordMailEvent("", "delivered", time.Now(), ""); err != nil { + t.Fatal(err) + } + reset, err := st.IssueToken(uid, users.PurposeReset, time.Hour, "") + if err != nil { + t.Fatal(err) + } + change, err := st.IssueToken(uid, users.PurposeEmailChange, time.Hour, "uma-new@example.org") + if err != nil { + t.Fatal(err) + } + full, err := st.GetByID(uid) + if err != nil { + t.Fatal(err) + } + + w := f.do("GET", "/api/account/export", nil, as(uma)) + expectStatus(t, w, 200) + if ct := w.Header().Get("Content-Type"); ct != "application/json" { + t.Errorf("Content-Type = %q", ct) + } + if cc := w.Header().Get("Cache-Control"); cc != "no-store" { + t.Errorf("Cache-Control = %q", cc) + } + if cd := w.Header().Get("Content-Disposition"); !exportFilenameRE.MatchString(cd) { + t.Errorf("Content-Disposition = %q", cd) + } + body := w.Body.String() + x := decode[accountExport](t, w) + + if x.FormatVersion != 1 || x.Instance != testBase || x.ExportedAt == "" { + t.Fatalf("header fields = %d %q %q", x.FormatVersion, x.Instance, x.ExportedAt) + } + p := x.Profile + if p.ID != uid || p.Email != "uma@example.org" || p.DisplayName != "Uma" || p.Role != users.RoleUser || + p.Status != users.StatusActive || p.CreatedAt == "" || p.EmailBouncing { + t.Fatalf("profile = %+v", p) + } + if p.PendingEmail == nil || *p.PendingEmail != "uma-new@example.org" { + t.Fatalf("pendingEmail = %v; want the unconfirmed address", p.PendingEmail) + } + if full.ActivatedAt == nil || p.ActivatedAt == nil || *p.ActivatedAt != rfc3339(*full.ActivatedAt) || p.ActivatedBy != nil { + t.Fatalf("activation (link): profile = %+v, stored at %v", p, full.ActivatedAt) + } + if len(x.Sessions) != 1 || x.Sessions[0].CreatedAt == "" || x.Sessions[0].LastSeenAt == "" { + t.Fatalf("sessions = %+v", x.Sessions) + } + if x.Settings == nil || x.Settings.Revision != 1 || x.Settings.UpdatedAt == "" || x.Settings.Doc.Keys["meshcore-theme"] != "dark" { + t.Fatalf("settings = %+v", x.Settings) + } + if len(x.Proposals) != 1 || x.Proposals[0].Subject != "#export-test" || x.Proposals[0].Kind != users.KindHashtagChannel || + x.Proposals[0].Status != users.ProposalPending || x.Proposals[0].DecidedAt != nil { + t.Fatalf("proposals = %+v", x.Proposals) + } + n := x.Notifications + if n.Prefs == nil || !n.Prefs.Enabled || len(n.Prefs.Events) != 1 || n.Prefs.Events[0] != users.NotifyNodeOffline || + len(n.Watches) != 1 || n.Watches[0].Pubkey != exportPubkey || n.Watches[0].CreatedAt == "" { + t.Fatalf("notifications = %+v", n) + } + var role *exportAuditEntry + for i := range x.Audit { + switch x.Audit[i].Action { + case "user.role": + role = &x.Audit[i] + case "user.export": + t.Fatal("the export contains its own audit row") + } + } + if role == nil || role.ActorUserID == nil || *role.ActorUserID != bossID || role.TargetUserID == nil || + *role.TargetUserID != uid || role.Detail["role"] != "user" || role.At == "" { + t.Fatalf("user.role audit entry = %+v", role) + } + var resetMail *exportMail + for i := range x.Mail { + if x.Mail[i].Purpose == "reset" { + resetMail = &x.Mail[i] + } + } + if resetMail == nil || resetMail.To != "uma@example.org" || resetMail.Status != "delivered" || resetMail.SentAt == "" || + len(resetMail.Events) != 1 || resetMail.Events[0].Event != "delivered" { + t.Fatalf("reset mail = %+v", resetMail) + } + + secrets := map[string]string{ + "password hash": full.PasswordHash, + "session cookie": uma.cookie.Value, + "session token hash": users.HashToken(uma.cookie.Value), + "csrf token": uma.csrf, + "unsubscribe token": prefs.UnsubToken, + "reset token": reset, + "reset token hash": users.HashToken(reset), + "email change token": change, + "email change hash": users.HashToken(change), + } + for name, v := range secrets { + if v == "" { + t.Fatalf("%s is empty: the check below would prove nothing", name) + } + if strings.Contains(body, v) { + t.Errorf("export contains the %s", name) + } + } + for _, other := range []string{"boss@example.org", `"Boss"`} { + if strings.Contains(body, other) { + t.Errorf("export contains another account's %s", other) + } + } +} + +func TestAccountExportWritesAuditRowAfterwards(t *testing.T) { + f := newAuthFixture(t) + uma := f.registerAndActivate(t, "uma@example.org", "Uma", pw) + expectStatus(t, f.do("GET", "/api/account/export", nil, as(uma)), 200) + entries, err := f.st.AuditList(users.AuditFilter{Actions: []string{"user.export"}}) + if err != nil || len(entries) != 1 { + t.Fatalf("user.export rows = %d, %v", len(entries), err) + } + e := entries[0] + if e.ActorUserID == nil || *e.ActorUserID != uma.me.ID || e.TargetUserID == nil || *e.TargetUserID != uma.me.ID { + t.Fatalf("user.export row = %+v", e) + } + second := decode[accountExport](t, f.do("GET", "/api/account/export", nil, as(uma))) + seen := 0 + for _, a := range second.Audit { + if a.Action == "user.export" { + seen++ + } + } + if seen != 1 { + t.Fatalf("second export lists %d user.export rows; want the 1 from the first export", seen) + } +} + +func TestAccountExportEmptySectionsAndNoWrites(t *testing.T) { + f := newAuthFixture(t) + uma := f.registerAndActivate(t, "uma@example.org", "Uma", pw) + w := f.do("GET", "/api/account/export", nil, as(uma)) + expectStatus(t, w, 200) + body := w.Body.String() + for _, want := range []string{`"settings":null`, `"proposals":[]`, `"prefs":null`, `"watches":[]`, `"pendingEmail":null`} { + if !strings.Contains(body, want) { + t.Errorf("export lacks %s: %s", want, body) + } + } + if p, err := f.st.StoredNotifyPrefs(uma.me.ID); err != nil || p != nil { + t.Fatalf("the export created notification prefs: %+v, %v", p, err) + } +} + +func TestAccountExportNoEventsIsEmptyList(t *testing.T) { + f := newAuthFixture(t) + uma := f.registerAndActivate(t, "uma@example.org", "Uma", pw) + if _, err := f.st.SetNotifyPrefs(uma.me.ID, false, nil); err != nil { + t.Fatal(err) + } + w := f.do("GET", "/api/account/export", nil, as(uma)) + expectStatus(t, w, 200) + if body := w.Body.String(); !strings.Contains(body, `"prefs":{"enabled":false,"events":[]}`) { + t.Fatalf("prefs without events: %s", body) + } +} + +func TestAccountExportStoreFailure(t *testing.T) { + f := newAuthFixture(t) + uma := f.registerAndActivate(t, "uma@example.org", "Uma", pw) + f.breakTable(t, "proposals") + w := f.do("GET", "/api/account/export", nil, as(uma)) + expectStatus(t, w, 500) + if cd := w.Header().Get("Content-Disposition"); cd != "" { + t.Fatalf("Content-Disposition on a failed export: %q", cd) + } + if entries, _ := f.st.AuditList(users.AuditFilter{Actions: []string{"user.export"}}); len(entries) != 0 { + t.Fatal("user.export recorded for a failed export") + } +} + +func TestAccountExportManualActivation(t *testing.T) { + f, boss, _ := adminFixture(t) + f.do("POST", "/api/auth/register", registerRequest{Email: "late@example.org", DisplayName: "Late", Password: pw}) + late, err := f.st.GetByEmail("late@example.org") + if err != nil { + t.Fatal(err) + } + expectStatus(t, f.do("POST", userPath(late.ID, "/activate"), nil, as(boss)), 200) + lateC := f.login(t, "late@example.org", pw) + x := decode[accountExport](t, f.do("GET", "/api/account/export", nil, as(lateC))) + if x.Profile.ActivatedBy == nil || *x.Profile.ActivatedBy != boss.me.ID || x.Profile.ActivatedAt == nil { + t.Fatalf("activation (manual): profile = %+v", x.Profile) + } + var activate *exportMail + for i := range x.Mail { + if x.Mail[i].Purpose == "activate" { + activate = &x.Mail[i] + } + } + if activate == nil || activate.To != "late@example.org" { + t.Fatalf("activation mail = %+v", activate) + } +} diff --git a/cmd/server/account_handlers.go b/cmd/server/account_handlers.go new file mode 100644 index 000000000..3b9d386b9 --- /dev/null +++ b/cmd/server/account_handlers.go @@ -0,0 +1,226 @@ +package main + +import ( + "errors" + "log" + "net/http" + "strconv" + "time" + + "github.com/gorilla/mux" + "github.com/meshcore-analyzer/mailer" + "github.com/meshcore-analyzer/users" +) + +// checkCurrentPassword writes 403 and returns false on a wrong password. +func checkCurrentPassword(w http.ResponseWriter, u *users.User, password string) bool { + if ok, err := users.VerifyPassword(u.PasswordHash, password); err != nil || !ok { + writeError(w, http.StatusForbidden, "current password is incorrect") + return false + } + return true +} + +func (s *Server) handleAccountPatch(w http.ResponseWriter, r *http.Request, u *users.User, sess *users.Session) { + var req profileRequest + if !decodeJSON(w, r, &req) { + return + } + name, err := users.ValidateDisplayName(req.DisplayName) + if writeValidation(w, err) { + return + } + if err := s.auth.st.SetDisplayName(u.ID, name); err != nil { + log.Printf("[users] set display name for user #%d: %v", u.ID, err) + writeError(w, http.StatusInternalServerError, "internal error") + return + } + u.DisplayName = name + writeJSON(w, meFrom(u, sess)) +} + +func (s *Server) handleAccountPassword(w http.ResponseWriter, r *http.Request, u *users.User, sess *users.Session) { + a := s.auth + var req passwordChangeRequest + if !decodeJSON(w, r, &req) || !checkCurrentPassword(w, u, req.CurrentPassword) { + return + } + if writeValidation(w, users.ValidatePassword(req.NewPassword)) { + return + } + hash, err := users.HashPassword(req.NewPassword) + if err == nil { + err = a.st.SetPassword(u.ID, hash) + } + if err != nil { + log.Printf("[users] password change for user #%d: %v", u.ID, err) + writeError(w, http.StatusInternalServerError, "internal error") + return + } + // Sessions first: a token-store failure below must not leave another + // device's session alive. + if err := a.st.DeleteUserSessions(u.ID, sess.ID); err != nil { + log.Printf("[users] password change: end other sessions for user #%d: %v", u.ID, err) + writeError(w, http.StatusInternalServerError, "internal error") + return + } + if err := a.invalidateTokens("password change", u.ID, users.PurposeEmailChange, users.PurposeReset); err != nil { + writeError(w, http.StatusInternalServerError, "internal error") + return + } + a.audit(idPtr(u.ID), "user.password.change", idPtr(u.ID), nil) + writeJSON(w, okResponse{OK: true, Message: "Password changed. Your other devices were logged out."}) +} + +// handleAccountEmail answers and mails the requester identically whether +// the new address is free or taken (no enumeration): the old address gets +// the notice either way, and only a free address gets the confirmation. +func (s *Server) handleAccountEmail(w http.ResponseWriter, r *http.Request, u *users.User, _ *users.Session) { + a := s.auth + var req emailChangeRequest + if !decodeJSON(w, r, &req) { + return + } + newEmail, err := users.NormalizeEmail(req.NewEmail) + if writeValidation(w, err) { + return + } + if newEmail == u.Email { + writeError(w, http.StatusBadRequest, "that is already your address") + return + } + if !a.allow(w, r, a.signup, "emailchange:#"+strconv.FormatInt(u.ID, 10), "email:"+newEmail) || + !checkCurrentPassword(w, u, req.CurrentPassword) { + return + } + switch _, err := a.st.GetByEmail(newEmail); { + case err == nil: + // Taken: no confirmation, but the newest request still replaces + // any earlier link, as in the free branch. + if err := a.invalidateTokens("email change", u.ID, users.PurposeEmailChange); err != nil { + writeError(w, http.StatusInternalServerError, "internal error") + return + } + // The notice is the only mail here; if it fails, answer like a + // failed confirmation in the free branch. + if err := a.sendMail(r.Context(), u, "email-change-notice", a.emailChangeNoticeMail(u, newEmail)); err != nil { + writeError(w, http.StatusServiceUnavailable, msgMailFailed) + return + } + case errors.Is(err, users.ErrNotFound): + err = a.mailToken(r.Context(), u, users.PurposeEmailChange, 24*time.Hour, newEmail, "email-change", + func(tok string) mailer.Message { return a.emailChangeConfirmMail(u, newEmail, tok) }) + if err != nil { + writeError(w, http.StatusServiceUnavailable, msgMailFailed) + return + } + // Best effort: sendMail logs a failure; the confirmation already went out. + _ = a.sendMail(r.Context(), u, "email-change-notice", a.emailChangeNoticeMail(u, newEmail)) + default: + log.Printf("[users] email change lookup for user #%d: %v", u.ID, err) + writeError(w, http.StatusInternalServerError, "internal error") + return + } + a.audit(idPtr(u.ID), "user.email.change.requested", idPtr(u.ID), nil) + writeJSON(w, okResponse{OK: true, Message: "Check the new address for a confirmation link."}) +} + +// handleConfirmEmail needs no session: the link may be opened on any device. +func (s *Server) handleConfirmEmail(w http.ResponseWriter, r *http.Request) { + a := s.auth + var req tokenRequest + if !decodeJSON(w, r, &req) { + return + } + uid, newEmail, err := a.st.ConsumeToken(req.Token, users.PurposeEmailChange) + if err != nil { + writeTokenError(w, err) + return + } + // Only an active account may change its address; anything else reads + // like a dead link. + u, err := a.st.GetByID(uid) + if err != nil && !errors.Is(err, users.ErrNotFound) { + log.Printf("[users] confirm email: load user #%d: %v", uid, err) + writeError(w, http.StatusInternalServerError, "internal error") + return + } + if err != nil || u.Status != users.StatusActive { + writeTokenError(w, users.ErrTokenInvalid) + return + } + if err := a.st.SetEmail(uid, newEmail); err != nil { + if errors.Is(err, users.ErrEmailTaken) { + writeError(w, http.StatusConflict, "that address is already in use") + return + } + log.Printf("[users] confirm email for user #%d: %v", uid, err) + writeError(w, http.StatusInternalServerError, "internal error") + return + } + a.audit(idPtr(uid), "user.email.change", idPtr(uid), nil) + writeJSON(w, okResponse{OK: true, Message: "Your address was changed."}) +} + +func (s *Server) handleAccountSessions(w http.ResponseWriter, _ *http.Request, u *users.User, sess *users.Session) { + list, err := s.auth.st.ListSessions(u.ID) + if err != nil { + log.Printf("[users] list sessions for user #%d: %v", u.ID, err) + writeError(w, http.StatusInternalServerError, "internal error") + return + } + out := make([]sessionJSON, 0, len(list)) + for _, x := range list { + out = append(out, sessionToJSON(x, sess.ID)) + } + writeJSON(w, out) +} + +func (s *Server) handleAccountSessionDelete(w http.ResponseWriter, r *http.Request, u *users.User, sess *users.Session) { + id, err := strconv.ParseInt(mux.Vars(r)["id"], 10, 64) + if err != nil { + writeError(w, http.StatusBadRequest, "invalid session id") + return + } + if err := s.auth.st.DeleteSession(u.ID, id); err != nil { + if errors.Is(err, users.ErrNotFound) { + writeError(w, http.StatusNotFound, "session not found") + return + } + log.Printf("[users] delete session for user #%d: %v", u.ID, err) + writeError(w, http.StatusInternalServerError, "internal error") + return + } + if id == sess.ID { + s.auth.clearSessionCookie(w) + } + writeJSON(w, okResponse{OK: true}) +} + +func (s *Server) handleAccountDelete(w http.ResponseWriter, r *http.Request, u *users.User, _ *users.Session) { + a := s.auth + var req passwordConfirmRequest + if !decodeJSON(w, r, &req) || !checkCurrentPassword(w, u, req.CurrentPassword) { + return + } + if u.Role == users.RoleAdmin { + n, err := a.st.CountActiveAdmins() + if err != nil { + log.Printf("[users] self-delete: count admins for user #%d: %v", u.ID, err) + writeError(w, http.StatusInternalServerError, "internal error") + return + } + if n <= 1 { + writeError(w, http.StatusConflict, "you are the last admin; promote someone else first") + return + } + } + if err := a.st.Delete(u.ID); err != nil { + log.Printf("[users] self-delete user #%d: %v", u.ID, err) + writeError(w, http.StatusInternalServerError, "internal error") + return + } + a.audit(idPtr(u.ID), "user.delete.self", idPtr(u.ID), nil) + a.clearSessionCookie(w) + writeJSON(w, okResponse{OK: true, Message: "Your account was deleted."}) +} diff --git a/cmd/server/account_test.go b/cmd/server/account_test.go new file mode 100644 index 000000000..80dc75b3e --- /dev/null +++ b/cmd/server/account_test.go @@ -0,0 +1,241 @@ +package main + +import ( + "errors" + "fmt" + "net/http/httptest" + "net/url" + "strings" + "testing" + + "github.com/meshcore-analyzer/users" +) + +func TestAccountProfileAndCSRF(t *testing.T) { + f := newAuthFixture(t) + c := f.registerAndActivate(t, "gil@example.org", "Gil", pw) + noCSRF := &client{cookie: c.cookie} + expectStatus(t, f.do("PATCH", "/api/account", profileRequest{DisplayName: "Gilbert"}, as(noCSRF)), 403) + expectStatus(t, f.do("PATCH", "/api/account", profileRequest{DisplayName: "Gilbert"}, as(c), header("Origin", "https://evil.example")), 403) + w := f.do("PATCH", "/api/account", profileRequest{DisplayName: "Gilbert"}, as(c)) + expectStatus(t, w, 200) + if decode[meResponse](t, w).DisplayName != "Gilbert" { + t.Fatal("display name not changed") + } + expectStatus(t, f.do("PATCH", "/api/account", profileRequest{DisplayName: "G"}, as(c)), 400) +} + +func TestAccountPasswordChangeKeepsCurrentSession(t *testing.T) { + f := newAuthFixture(t) + a := f.registerAndActivate(t, "hal@example.org", "Hal", pw) + b := f.login(t, "hal@example.org", pw) + expectStatus(t, f.do("POST", "/api/account/password", passwordChangeRequest{CurrentPassword: "wrong one!!", NewPassword: "new secret pass"}, as(a)), 403) + expectStatus(t, f.do("POST", "/api/account/password", passwordChangeRequest{CurrentPassword: pw, NewPassword: "new secret pass"}, as(a)), 200) + expectStatus(t, f.do("GET", "/api/auth/me", nil, as(a)), 200) + expectStatus(t, f.do("GET", "/api/auth/me", nil, as(b)), 401) +} + +func TestAccountEmailChange(t *testing.T) { + f := newAuthFixture(t) + c := f.registerAndActivate(t, "ivy@example.org", "Ivy", pw) + expectStatus(t, f.do("POST", "/api/account/email", emailChangeRequest{NewEmail: "ivy.new@example.org", CurrentPassword: pw}, as(c)), 200) + sent := f.fake.Sent() + confirm, notice := sent[len(sent)-2], sent[len(sent)-1] + if confirm.To != "ivy.new@example.org" || notice.To != "ivy@example.org" || !strings.Contains(notice.Text, "ivy.new@example.org") { + t.Fatalf("confirm=%+v notice=%+v", confirm, notice) + } + tok := tokenRE.FindStringSubmatch(confirm.Text)[1] + expectStatus(t, f.do("POST", "/api/account/confirm-email", tokenRequest{Token: tok}, header("Origin", "")), 403) // no Origin + expectStatus(t, f.do("POST", "/api/account/confirm-email", tokenRequest{Token: tok}), 200) + f.login(t, "ivy.new@example.org", pw) + expectStatus(t, f.do("POST", "/api/auth/login", loginRequest{Email: "ivy@example.org", Password: pw}), 401) +} + +func TestAccountSessionsListAndRevoke(t *testing.T) { + f := newAuthFixture(t) + a := f.registerAndActivate(t, "jo@example.org", "Jo", pw) + b := f.login(t, "jo@example.org", pw) + w := f.do("GET", "/api/account/sessions", nil, as(a)) + expectStatus(t, w, 200) + list := decode[[]sessionJSON](t, w) + if len(list) != 2 { + t.Fatalf("sessions = %+v", list) + } + var other int64 + for _, s := range list { + if !s.Current { + other = s.ID + } + } + expectStatus(t, f.do("DELETE", fmt.Sprintf("/api/account/sessions/%d", other), nil, as(a)), 200) + expectStatus(t, f.do("GET", "/api/auth/me", nil, as(b)), 401) + expectStatus(t, f.do("DELETE", "/api/account/sessions/999999", nil, as(a)), 404) +} + +func TestAccountDelete(t *testing.T) { + f := newAuthFixture(t, "boss@example.org") + boss := f.registerAndActivate(t, "boss@example.org", "Boss", pw) + expectStatus(t, f.do("DELETE", "/api/account", passwordConfirmRequest{CurrentPassword: pw}, as(boss)), 409) // last admin + u := f.registerAndActivate(t, "kim@example.org", "Kim", pw) + expectStatus(t, f.do("DELETE", "/api/account", passwordConfirmRequest{CurrentPassword: "wrong one!!"}, as(u)), 403) + w := f.do("DELETE", "/api/account", passwordConfirmRequest{CurrentPassword: pw}, as(u)) + expectStatus(t, w, 200) + if ck := w.Result().Cookies(); len(ck) == 0 || ck[0].MaxAge >= 0 { + t.Fatalf("cookie not cleared: %+v", ck) + } + expectStatus(t, f.do("POST", "/api/auth/login", loginRequest{Email: "kim@example.org", Password: pw}), 401) +} + +// pendingLinks requests an email change to newEmail and a password reset for +// c (whose address is email) and returns both unused links. +func pendingLinks(t *testing.T, f *authFixture, c *client, email, newEmail string) (confirm, reset string) { + t.Helper() + expectStatus(t, f.do("POST", "/api/account/email", emailChangeRequest{NewEmail: newEmail, CurrentPassword: pw}, as(c), fromIP("192.0.2.1")), 200) + sent := f.fake.Sent() + confirm, _ = url.QueryUnescape(tokenRE.FindStringSubmatch(sent[len(sent)-2].Text)[1]) + expectStatus(t, f.do("POST", "/api/auth/forgot", emailRequest{Email: email}, fromIP("192.0.2.2")), 200) + return confirm, f.lastToken(t) +} + +func TestAuthResetKillsPendingEmailChange(t *testing.T) { + f := newAuthFixture(t) + c := f.registerAndActivate(t, "nia@example.org", "Nia", pw) + confirm, reset := pendingLinks(t, f, c, "nia@example.org", "attacker@example.org") + expectStatus(t, f.do("POST", "/api/auth/reset", resetRequest{Token: reset, Password: "a brand new secret"}), 200) + expectStatus(t, f.do("POST", "/api/account/confirm-email", tokenRequest{Token: confirm}), 410) + f.login(t, "nia@example.org", "a brand new secret") +} + +func TestAccountPasswordChangeKillsPendingLinks(t *testing.T) { + f := newAuthFixture(t) + c := f.registerAndActivate(t, "oli@example.org", "Oli", pw) + confirm, reset := pendingLinks(t, f, c, "oli@example.org", "attacker@example.org") + expectStatus(t, f.do("POST", "/api/account/password", passwordChangeRequest{CurrentPassword: pw, NewPassword: "new secret pass"}, as(c)), 200) + expectStatus(t, f.do("POST", "/api/account/confirm-email", tokenRequest{Token: confirm}), 410) + expectStatus(t, f.do("POST", "/api/auth/reset", resetRequest{Token: reset, Password: "another new secret"}), 410) + f.login(t, "oli@example.org", "new secret pass") +} + +// A token-store failure still answers 500, but only after the other +// sessions are gone: a thief's session must not outlive the change. +func TestAccountPasswordChangeTokenStoreFailureIs500(t *testing.T) { + f := newAuthFixture(t) + c := f.registerAndActivate(t, "pam@example.org", "Pam", pw) + thief := f.login(t, "pam@example.org", pw) + f.breakTable(t, "tokens") + expectStatus(t, f.do("POST", "/api/account/password", passwordChangeRequest{CurrentPassword: pw, NewPassword: "new secret pass"}, as(c)), 500) + expectStatus(t, f.do("GET", "/api/auth/me", nil, as(thief)), 401) + expectStatus(t, f.do("GET", "/api/auth/me", nil, as(c)), 200) +} + +// Same for reset: the email-change invalidation fails, all sessions are +// already ended. +func TestAuthResetTokenStoreFailureEndsSessionsFirst(t *testing.T) { + f := newAuthFixture(t) + c := f.registerAndActivate(t, "pia@example.org", "Pia", pw) + thief := f.login(t, "pia@example.org", pw) + _, reset := pendingLinks(t, f, c, "pia@example.org", "attacker@example.org") + f.execDB(t, testNativeSQL(`CREATE TRIGGER down BEFORE UPDATE ON tokens WHEN OLD.purpose='email_change' BEGIN SELECT RAISE(ABORT,'token store down'); END`, `CREATE FUNCTION down() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'token store down'; END $$; CREATE TRIGGER down BEFORE UPDATE ON tokens FOR EACH ROW WHEN (OLD.purpose='email_change') EXECUTE FUNCTION down()`)) + expectStatus(t, f.do("POST", "/api/auth/reset", resetRequest{Token: reset, Password: "a brand new secret"}), 500) + expectStatus(t, f.do("GET", "/api/auth/me", nil, as(thief)), 401) + expectStatus(t, f.do("GET", "/api/auth/me", nil, as(c)), 401) +} + +func TestAccountEmailChangeTakenLooksLikeFree(t *testing.T) { + f := newAuthFixture(t) + c := f.registerAndActivate(t, "quin@example.org", "Quin", pw) + f.registerAndActivate(t, "rae@example.org", "Rae", pw) + free := f.do("POST", "/api/account/email", emailChangeRequest{NewEmail: "quin.new@example.org", CurrentPassword: pw}, as(c), fromIP("192.0.2.1")) + before := len(f.fake.Sent()) + taken := f.do("POST", "/api/account/email", emailChangeRequest{NewEmail: "rae@example.org", CurrentPassword: pw}, as(c), fromIP("192.0.2.2")) + if free.Code != 200 || free.Code != taken.Code || free.Body.String() != taken.Body.String() { + t.Fatalf("responses differ: %d %q vs %d %q", free.Code, free.Body, taken.Code, taken.Body) + } + sent := f.fake.Sent()[before:] + if len(sent) != 1 || sent[0].To != "quin@example.org" || !strings.Contains(sent[0].Text, "rae@example.org") { + t.Fatalf("taken branch sent %+v, want only the notice to the old address", sent) + } +} + +func TestAccountEmailChangeMailFailureLeavesNoLink(t *testing.T) { + f := newAuthFixture(t) + c := f.registerAndActivate(t, "sam@example.org", "Sam", pw) + f.fake.SetSendErr(errors.New("brevo down")) + expectStatus(t, f.do("POST", "/api/account/email", emailChangeRequest{NewEmail: "sam.new@example.org", CurrentPassword: pw}, as(c)), 503) + if n := f.unusedTokens(t, c.me.ID, users.PurposeEmailChange); n != 0 { + t.Fatalf("%d usable email-change links after a failed mail", n) + } +} + +// Taken branch: the notice to the old address is the only mail; when it +// fails the answer is 503, the address stays and no earlier link survives. +func TestAccountEmailChangeTakenNoticeFailureIs503(t *testing.T) { + f := newAuthFixture(t) + c := f.registerAndActivate(t, "sol@example.org", "Sol", pw) + f.registerAndActivate(t, "tam@example.org", "Tam", pw) + expectStatus(t, f.do("POST", "/api/account/email", emailChangeRequest{NewEmail: "sol.new@example.org", CurrentPassword: pw}, as(c), fromIP("192.0.2.1")), 200) + f.fake.SetSendErr(errors.New("brevo down")) + w := f.do("POST", "/api/account/email", emailChangeRequest{NewEmail: "tam@example.org", CurrentPassword: pw}, as(c), fromIP("192.0.2.2")) + expectStatus(t, w, 503) + if !strings.Contains(w.Body.String(), msgMailFailed) { + t.Fatalf("body = %s", w.Body.String()) + } + if got, _ := f.st.GetByID(c.me.ID); got.Email != "sol@example.org" { + t.Fatalf("address changed to %q", got.Email) + } + if n := f.unusedTokens(t, c.me.ID, users.PurposeEmailChange); n != 0 { + t.Fatalf("%d usable email-change links after a failed notice", n) + } +} + +func TestAccountEmailChangeRateLimitPerUser(t *testing.T) { + f := newAuthFixture(t) + c := f.registerAndActivate(t, "tia@example.org", "Tia", pw) + change := func(i int) *httptest.ResponseRecorder { + return f.do("POST", "/api/account/email", emailChangeRequest{NewEmail: fmt.Sprintf("tia%d@example.org", i), CurrentPassword: pw}, + as(c), fromIP(fmt.Sprintf("198.51.100.%d", i+1))) + } + for i := 0; i < 5; i++ { + expectStatus(t, change(i), 200) + } + expectStatus(t, change(5), 429) +} + +func TestAccountEmailChangeRateLimitPerAddress(t *testing.T) { + f := newAuthFixture(t) + ip := 0 + change := func(c *client) *httptest.ResponseRecorder { + ip++ + return f.do("POST", "/api/account/email", emailChangeRequest{NewEmail: "target@example.org", CurrentPassword: pw}, + as(c), fromIP(fmt.Sprintf("198.51.100.%d", ip))) + } + a := f.registerAndActivate(t, "uli@example.org", "Uli", pw) + b := f.registerAndActivate(t, "val@example.org", "Val", pw) + d := f.registerAndActivate(t, "wes@example.org", "Wes", pw) + for i := 0; i < 3; i++ { + expectStatus(t, change(a), 200) + } + for i := 0; i < 2; i++ { + expectStatus(t, change(b), 200) + } + expectStatus(t, change(d), 429) +} + +func TestConfirmEmailForNonActiveUserIs410(t *testing.T) { + f := newAuthFixture(t) + c := f.registerAndActivate(t, "xan@example.org", "Xan", pw) + expectStatus(t, f.do("POST", "/api/account/email", emailChangeRequest{NewEmail: "xan.new@example.org", CurrentPassword: pw}, as(c)), 200) + sent := f.fake.Sent() + tok, _ := url.QueryUnescape(tokenRE.FindStringSubmatch(sent[len(sent)-2].Text)[1]) + if err := f.st.SetStatus(c.me.ID, users.StatusDisabled); err != nil { // e.g. a direct DB edit; disable itself burns the link + t.Fatal(err) + } + w := f.do("POST", "/api/account/confirm-email", tokenRequest{Token: tok}) + expectStatus(t, w, 410) + if !strings.Contains(w.Body.String(), "this link is invalid or was already used") { + t.Fatalf("body = %s", w.Body.String()) + } + if got, _ := f.st.GetByID(c.me.ID); got.Email != "xan@example.org" { + t.Fatalf("address changed for a disabled user: %q", got.Email) + } +} diff --git a/cmd/server/admin_audit_handlers.go b/cmd/server/admin_audit_handlers.go new file mode 100644 index 000000000..b16969d55 --- /dev/null +++ b/cmd/server/admin_audit_handlers.go @@ -0,0 +1,159 @@ +package main + +import ( + "errors" + "log" + "net/http" + "net/url" + "regexp" + "strconv" + "time" + + "github.com/meshcore-analyzer/users" +) + +// auditUserRef names an actor or target. Deleted accounts keep only the id. +type auditUserRef struct { + ID int64 `json:"id"` + DisplayName string `json:"displayName,omitempty"` + Email string `json:"email,omitempty"` + Deleted bool `json:"deleted,omitempty"` +} + +type auditListEntryJSON struct { + ID int64 `json:"id"` + At string `json:"at"` + Action string `json:"action"` + Actor *auditUserRef `json:"actor"` + Target *auditUserRef `json:"target"` + Detail map[string]string `json:"detail"` +} + +type auditListJSON struct { + Entries []auditListEntryJSON `json:"entries"` + Next *int64 `json:"next"` +} + +// One action ("user.login.failed") or a group ending in ".*" ("user.login.*"). +var auditActionRE = regexp.MustCompile(`^[a-z][a-z0-9_]*(\.[a-z0-9_]+)*(\.\*)?$`) + +func parseAuditTime(q url.Values, name string) (*time.Time, error) { + v := q.Get(name) + if v == "" { + return nil, nil + } + t, err := time.Parse(time.RFC3339, v) + if err != nil { + return nil, errors.New("invalid " + name + ": use RFC 3339") + } + return &t, nil +} + +func parsePositiveID(v, name string) (int64, error) { + id, err := strconv.ParseInt(v, 10, 64) + if err != nil || id <= 0 { + return 0, errors.New("invalid " + name) + } + return id, nil +} + +// parseAuditFilter reads the query of GET /api/admin/audit. Any invalid +// parameter is an error (answered 400); limit above the cap is capped. +func parseAuditFilter(q url.Values) (users.AuditFilter, error) { + f := users.AuditFilter{Limit: 100} + if a := q.Get("action"); a != "" { + if len(a) > 64 || !auditActionRE.MatchString(a) { + return f, errors.New("invalid action filter") + } + f.Actions = []string{a} + } + if v := q.Get("user"); v != "" { + id, err := parsePositiveID(v, "user filter") + if err != nil { + return f, err + } + f.UserID = &id + } + var err error + if f.From, err = parseAuditTime(q, "from"); err != nil { + return f, err + } + if f.To, err = parseAuditTime(q, "to"); err != nil { + return f, err + } + if f.From != nil && f.To != nil && f.From.After(*f.To) { + return f, errors.New("from is after to") + } + if v := q.Get("before"); v != "" { + if f.BeforeID, err = parsePositiveID(v, "before"); err != nil { + return f, err + } + } + if v := q.Get("limit"); v != "" { + n, err := strconv.Atoi(v) + if err != nil || n <= 0 { + return f, errors.New("invalid limit") + } + if n > users.AuditListMax { + n = users.AuditListMax + } + f.Limit = n + } + return f, nil +} + +// auditUserIDs lists the distinct actor and target ids of a page. +func auditUserIDs(list []users.AuditEntry) []int64 { + seen := map[int64]bool{} + var ids []int64 + for _, e := range list { + for _, p := range []*int64{e.ActorUserID, e.TargetUserID} { + if p != nil && !seen[*p] { + seen[*p] = true + ids = append(ids, *p) + } + } + } + return ids +} + +func auditRef(id *int64, known map[int64]users.User) *auditUserRef { + if id == nil { + return nil + } + u, ok := known[*id] + if !ok { + return &auditUserRef{ID: *id, Deleted: true} + } + return &auditUserRef{ID: u.ID, DisplayName: u.DisplayName, Email: u.Email} +} + +func (s *Server) handleAdminAudit(w http.ResponseWriter, r *http.Request, _ *users.User, _ *users.Session) { + f, err := parseAuditFilter(r.URL.Query()) + if err != nil { + writeError(w, http.StatusBadRequest, err.Error()) + return + } + list, err := s.auth.st.AuditList(f) + if err != nil { + log.Printf("[users] admin audit list: %v", err) + writeError(w, http.StatusInternalServerError, "internal error") + return + } + known, err := s.auth.st.UsersByID(auditUserIDs(list)) + if err != nil { + log.Printf("[users] admin audit users: %v", err) + writeError(w, http.StatusInternalServerError, "internal error") + return + } + out := auditListJSON{Entries: make([]auditListEntryJSON, 0, len(list))} + for _, e := range list { + out.Entries = append(out.Entries, auditListEntryJSON{ID: e.ID, At: rfc3339(e.At), Action: e.Action, + Actor: auditRef(e.ActorUserID, known), Target: auditRef(e.TargetUserID, known), Detail: e.Detail}) + } + if len(list) == f.Limit { + next := list[len(list)-1].ID + out.Next = &next + } + writeJSON(w, out) +} diff --git a/cmd/server/admin_audit_test.go b/cmd/server/admin_audit_test.go new file mode 100644 index 000000000..7f0012cca --- /dev/null +++ b/cmd/server/admin_audit_test.go @@ -0,0 +1,115 @@ +package main + +import ( + "fmt" + "net/url" + "strings" + "testing" + + "github.com/meshcore-analyzer/users" +) + +func auditPage(t *testing.T, f *authFixture, c *client, query string) auditListJSON { + t.Helper() + w := f.do("GET", "/api/admin/audit"+query, nil, as(c)) + expectStatus(t, w, 200) + return decode[auditListJSON](t, w) +} + +func entryActions(p auditListJSON) string { + parts := make([]string, len(p.Entries)) + for i, e := range p.Entries { + parts[i] = e.Action + } + return strings.Join(parts, ",") +} + +func TestAdminAuditRequiresAdmin(t *testing.T) { + f, _, uma := adminFixture(t) + expectStatus(t, f.do("GET", "/api/admin/audit", nil), 401) + expectStatus(t, f.do("GET", "/api/admin/audit", nil, as(uma)), 403) +} + +func TestAdminAuditFiltersAndUsers(t *testing.T) { + f, boss, uma := adminFixture(t) + for i := 0; i < 2; i++ { + expectStatus(t, f.do("POST", "/api/auth/login", loginRequest{Email: "uma@example.org", Password: "wrong password!"}), 401) + } + f.login(t, "uma@example.org", pw) + + p := auditPage(t, f, boss, "?action=user.login.failed") + if len(p.Entries) != 2 || p.Next != nil { + t.Fatalf("failed logins = %+v", p) + } + e := p.Entries[0] + if e.Actor != nil || e.Target == nil || e.Target.ID != uma.me.ID || e.Target.DisplayName != "Uma" || + e.Target.Email != "uma@example.org" || e.Target.Deleted || e.Detail["reason"] != "wrong_password" || e.At == "" { + t.Fatalf("entry = %+v (target %+v)", e, e.Target) + } + if got := entryActions(auditPage(t, f, boss, "?action=user.login.*")); got != "user.login,user.login.failed,user.login.failed" { + t.Fatalf("group = %s", got) + } + if got := entryActions(auditPage(t, f, boss, fmt.Sprintf("?user=%d", boss.me.ID))); got != "user.activate,user.register" { + t.Fatalf("user filter = %s", got) + } + future := url.QueryEscape("2100-01-01T00:00:00Z") + if p := auditPage(t, f, boss, "?from="+future); len(p.Entries) != 0 || p.Next != nil { + t.Fatalf("future period = %+v", p) + } +} + +func TestAdminAuditPaginates(t *testing.T) { + f, boss, _ := adminFixture(t) // 4 rows: register and activate for each user + first := auditPage(t, f, boss, "?limit=3") + if len(first.Entries) != 3 || first.Next == nil || *first.Next != first.Entries[2].ID { + t.Fatalf("first page = %+v", first) + } + second := auditPage(t, f, boss, fmt.Sprintf("?limit=3&before=%d", *first.Next)) + if len(second.Entries) != 1 || second.Next != nil || second.Entries[0].ID >= *first.Next { + t.Fatalf("second page = %+v", second) + } +} + +func TestAdminAuditDeletedUser(t *testing.T) { + f, boss, uma := adminFixture(t) + expectStatus(t, f.do("DELETE", userPath(uma.me.ID, ""), nil, as(boss)), 200) + p := auditPage(t, f, boss, fmt.Sprintf("?user=%d&action=user.register", uma.me.ID)) + if len(p.Entries) != 1 || p.Entries[0].Target == nil || !p.Entries[0].Target.Deleted || + p.Entries[0].Target.ID != uma.me.ID || p.Entries[0].Target.Email != "" || p.Entries[0].Target.DisplayName != "" { + t.Fatalf("deleted target = %+v", p) + } + del := auditPage(t, f, boss, "?action=user.delete") + if len(del.Entries) != 1 || del.Entries[0].Actor == nil || del.Entries[0].Actor.ID != boss.me.ID || del.Entries[0].Actor.Deleted { + t.Fatalf("delete row = %+v", del) + } +} + +func TestAdminAuditRejectsBadParameters(t *testing.T) { + f, boss, _ := adminFixture(t) + for _, q := range []string{ + "?action=DROP%20TABLE", "?action=user.*.x", "?action=*", "?user=abc", "?user=0", + "?from=yesterday", "?to=2026-13-01T00:00:00Z", "?before=-1", "?before=x", "?limit=0", "?limit=ten", + "?from=2026-10-02T00:00:00Z&to=2026-10-01T00:00:00Z", + } { + if w := f.do("GET", "/api/admin/audit"+q, nil, as(boss)); w.Code != 400 { + t.Errorf("%s = %d; want 400", q, w.Code) + } + } +} + +func TestParseAuditFilterDefaultsAndCap(t *testing.T) { + flt, err := parseAuditFilter(url.Values{"limit": {"100000"}}) + if err != nil || flt.Limit != users.AuditListMax { + t.Fatalf("limit 100000 = %+v, %v; want %d", flt, err, users.AuditListMax) + } + flt, err = parseAuditFilter(url.Values{}) + if err != nil || flt.Limit != 100 || flt.Actions != nil || flt.UserID != nil || flt.From != nil || flt.BeforeID != 0 { + t.Fatalf("defaults = %+v, %v", flt, err) + } +} + +func TestAdminAuditStoreErrorIs500(t *testing.T) { + f, boss, _ := adminFixture(t) + f.breakTable(t, "audit_log") + expectStatus(t, f.do("GET", "/api/admin/audit", nil, as(boss)), 500) +} diff --git a/cmd/server/admin_stats_handlers.go b/cmd/server/admin_stats_handlers.go new file mode 100644 index 000000000..aa39fe5d9 --- /dev/null +++ b/cmd/server/admin_stats_handlers.go @@ -0,0 +1,102 @@ +package main + +import ( + "log" + "net/http" + "time" + + "github.com/meshcore-analyzer/users" +) + +// The field sets match users.DayCount, users.MailCounts and +// users.GuessedAccount, so plain type conversions fill them. +type dayCountJSON struct { + Day string `json:"day"` + Count int `json:"count"` +} + +type mailCountsJSON struct { + Delivered int `json:"delivered"` + Bounced int `json:"bounced"` + Blocked int `json:"blocked"` + Spam int `json:"spam"` + Pending int `json:"pending"` + Other int `json:"other"` +} + +type guessedAccountJSON struct { + UserID int64 `json:"userId"` + DisplayName string `json:"displayName"` + Failed int `json:"failed"` +} + +// adminNotifyJSON: notification figures, counts only (watch lists are +// private). Present only with notifications on. +type adminNotifyJSON struct { + MailsLast24h int `json:"mailsLast24h"` + MaxMailsPerDay int `json:"maxMailsPerDay"` + Watches int `json:"watches"` + WatchingUsers int `json:"watchingUsers"` +} + +type adminStatsJSON struct { + Total int `json:"total"` + Active int `json:"active"` + Pending int `json:"pending"` + Disabled int `json:"disabled"` + Admins int `json:"admins"` + StuckPending int `json:"stuckPending"` + Bouncing int `json:"bouncing"` + New7d int `json:"new7d"` + New30d int `json:"new30d"` + NewPerDay []dayCountJSON `json:"newPerDay"` + Active7d int `json:"active7d"` + Active30d int `json:"active30d"` + Logins24h int `json:"logins24h"` + FailedLogins24h int `json:"failedLogins24h"` + Mail7d mailCountsJSON `json:"mail7d"` + Guessing []guessedAccountJSON `json:"guessing"` + Notify *adminNotifyJSON `json:"notify,omitempty"` +} + +func adminStatsFrom(st users.Stats) adminStatsJSON { + out := adminStatsJSON{Total: st.Total, Active: st.Active, Pending: st.Pending, Disabled: st.Disabled, + Admins: st.Admins, StuckPending: st.StuckPending, Bouncing: st.Bouncing, New7d: st.New7d, New30d: st.New30d, + Active7d: st.Active7d, Active30d: st.Active30d, Logins24h: st.Logins24h, FailedLogins24h: st.FailedLogins24h, + Mail7d: mailCountsJSON(st.Mail7d), + NewPerDay: make([]dayCountJSON, 0, len(st.NewPerDay)), + Guessing: make([]guessedAccountJSON, 0, len(st.Guessing))} + for _, d := range st.NewPerDay { + out.NewPerDay = append(out.NewPerDay, dayCountJSON(d)) + } + for _, g := range st.Guessing { + out.Guessing = append(out.Guessing, guessedAccountJSON(g)) + } + return out +} + +func (s *Server) handleAdminStats(w http.ResponseWriter, _ *http.Request, _ *users.User, _ *users.Session) { + st, err := s.auth.st.Stats(time.Now()) + if err != nil { + log.Printf("[users] admin stats: %v", err) + writeError(w, http.StatusInternalServerError, "internal error") + return + } + out := adminStatsFrom(st) + if s.auth.notify != nil { + total, _, err := s.auth.st.NotifyMailCounts(s.auth.notify.now().Add(-24 * time.Hour)) + if err != nil { + log.Printf("[users] admin stats notification mails: %v", err) + writeError(w, http.StatusInternalServerError, "internal error") + return + } + watches, watching, err := s.auth.st.NotifyWatchStats() + if err != nil { + log.Printf("[users] admin stats watches: %v", err) + writeError(w, http.StatusInternalServerError, "internal error") + return + } + out.Notify = &adminNotifyJSON{MailsLast24h: total, MaxMailsPerDay: s.auth.set.notify.maxMailsPerDay, Watches: watches, WatchingUsers: watching} + } + writeJSON(w, out) +} diff --git a/cmd/server/admin_stats_test.go b/cmd/server/admin_stats_test.go new file mode 100644 index 000000000..e93838229 --- /dev/null +++ b/cmd/server/admin_stats_test.go @@ -0,0 +1,46 @@ +package main + +import ( + "strings" + "testing" +) + +func TestAdminStatsRequiresAdmin(t *testing.T) { + f, _, uma := adminFixture(t) + expectStatus(t, f.do("GET", "/api/admin/stats", nil), 401) + expectStatus(t, f.do("GET", "/api/admin/stats", nil, as(uma)), 403) +} + +func TestAdminStats(t *testing.T) { + f, boss, uma := adminFixture(t) + expectStatus(t, f.do("POST", "/api/auth/register", registerRequest{Email: "pending@example.org", DisplayName: "Pen", Password: pw}), 200) + for i := 0; i < 5; i++ { + expectStatus(t, f.do("POST", "/api/auth/login", loginRequest{Email: "uma@example.org", Password: "wrong password!"}), 401) + } + w := f.do("GET", "/api/admin/stats", nil, as(boss)) + expectStatus(t, w, 200) + st := decode[adminStatsJSON](t, w) + if st.Total != 3 || st.Active != 2 || st.Pending != 1 || st.Admins != 1 || st.StuckPending != 0 || + st.New7d != 3 || st.New30d != 3 || len(st.NewPerDay) != 30 || st.Active7d != 2 || + st.Logins24h != 0 || st.FailedLogins24h != 5 || st.Mail7d.Pending != 3 { + t.Fatalf("stats = %+v", st) + } + if len(st.Guessing) != 1 || st.Guessing[0].UserID != uma.me.ID || st.Guessing[0].DisplayName != "Uma" || st.Guessing[0].Failed != 5 { + t.Fatalf("guessing = %+v", st.Guessing) + } +} + +func TestAdminStatsEmptyListsAreArrays(t *testing.T) { + f, boss, _ := adminFixture(t) + w := f.do("GET", "/api/admin/stats", nil, as(boss)) + expectStatus(t, w, 200) + if !strings.Contains(w.Body.String(), `"guessing":[]`) { + t.Fatalf("body = %s; want \"guessing\":[]", w.Body.String()) + } +} + +func TestAdminStatsStoreErrorIs500(t *testing.T) { + f, boss, _ := adminFixture(t) + f.breakTable(t, "mail_log") + expectStatus(t, f.do("GET", "/api/admin/stats", nil, as(boss)), 500) +} diff --git a/cmd/server/admin_users_handlers.go b/cmd/server/admin_users_handlers.go new file mode 100644 index 000000000..1860e596c --- /dev/null +++ b/cmd/server/admin_users_handlers.go @@ -0,0 +1,336 @@ +package main + +import ( + "errors" + "log" + "net/http" + "strconv" + "time" + + "github.com/gorilla/mux" + "github.com/meshcore-analyzer/mailer" + "github.com/meshcore-analyzer/users" +) + +// adminStoreFail answers a failed store call: ErrNotFound is 404, anything +// else is logged (user id only) and answered 500. +func adminStoreFail(w http.ResponseWriter, op string, id int64, err error) { + if errors.Is(err, users.ErrNotFound) { + writeError(w, http.StatusNotFound, "user not found") + return + } + log.Printf("[users] admin %s for user #%d: %v", op, id, err) + writeError(w, http.StatusInternalServerError, "internal error") +} + +// adminTarget loads {id} from the path; writes 400/404/500 and returns nil +// on failure. +func (s *Server) adminTarget(w http.ResponseWriter, r *http.Request) *users.User { + id, err := strconv.ParseInt(mux.Vars(r)["id"], 10, 64) + if err != nil { + writeError(w, http.StatusBadRequest, "invalid user id") + return nil + } + u, err := s.auth.st.GetByID(id) + if err != nil { + adminStoreFail(w, "load", id, err) + return nil + } + return u +} + +// guardRemoval blocks disabling/deleting/demoting yourself (except a +// self-demotion, allowed via allowSelf), config admins, and the last active +// admin. Writes 409 (or 500 if the admin count fails) and returns false +// when blocked. Count-then-mutate is not atomic; two admins removing each +// other concurrently can still both pass. +func (s *Server) guardRemoval(w http.ResponseWriter, actor, target *users.User, allowSelf bool) bool { + if target.ID == actor.ID && !allowSelf { + writeError(w, http.StatusConflict, "use My account to change your own account") + return false + } + if target.Role == users.RoleAdmin && s.auth.isConfigAdmin(target.Email) { + writeError(w, http.StatusConflict, "this admin is listed in adminEmails; remove them from the config first") + return false + } + if target.Role == users.RoleAdmin && target.Status == users.StatusActive { + n, err := s.auth.st.CountActiveAdmins() + if err != nil { + log.Printf("[users] admin guard: count admins for user #%d: %v", target.ID, err) + writeError(w, http.StatusInternalServerError, "internal error") + return false + } + if n <= 1 { + writeError(w, http.StatusConflict, "this is the last admin; promote someone else first") + return false + } + } + return true +} + +func (s *Server) writeAdminRow(w http.ResponseWriter, id int64) { + u, err := s.auth.st.GetByID(id) + if err != nil { + adminStoreFail(w, "reload", id, err) + return + } + last, err := s.auth.st.LatestMailByUser() + if err != nil { + log.Printf("[users] admin latest mail: %v", err) + } + writeJSON(w, s.auth.adminRow(*u, last)) +} + +func (s *Server) handleAdminUsers(w http.ResponseWriter, r *http.Request, _ *users.User, _ *users.Session) { + q := r.URL.Query() + f := users.ListFilter{Status: users.Status(q.Get("status")), Role: users.Role(q.Get("role")), Query: q.Get("q")} + switch f.Status { + case "", users.StatusPending, users.StatusActive, users.StatusDisabled: + default: + writeError(w, http.StatusBadRequest, "invalid status filter") + return + } + if f.Role != "" && !f.Role.Valid() { + writeError(w, http.StatusBadRequest, "invalid role filter") + return + } + switch q.Get("bouncing") { + case "": + case "1": + f.Bouncing = true + default: + writeError(w, http.StatusBadRequest, "invalid bouncing filter") + return + } + list, err := s.auth.st.List(f) + if err != nil { + log.Printf("[users] admin list: %v", err) + writeError(w, http.StatusInternalServerError, "internal error") + return + } + last, err := s.auth.st.LatestMailByUser() + if err != nil { + log.Printf("[users] admin latest mail: %v", err) + } + out := make([]adminUserJSON, 0, len(list)) + for _, u := range list { + out = append(out, s.auth.adminRow(u, last)) + } + writeJSON(w, out) +} + +func (s *Server) handleAdminUserDetail(w http.ResponseWriter, r *http.Request, _ *users.User, _ *users.Session) { + u := s.adminTarget(w, r) + if u == nil { + return + } + a := s.auth + last, err := a.st.LatestMailByUser() + if err != nil { + log.Printf("[users] admin latest mail: %v", err) + } + d := adminUserDetailJSON{User: a.adminRow(*u, last), Sessions: []sessionJSON{}, Mail: []mailJSON{}, Audit: []auditJSON{}} + list, err := a.st.ListSessions(u.ID) + if err != nil { + adminStoreFail(w, "list sessions", u.ID, err) + return + } + for _, x := range list { + d.Sessions = append(d.Sessions, sessionToJSON(x, 0)) + } + mails, err := a.st.MailForUser(u.ID, 50) + if err != nil { + adminStoreFail(w, "list mail", u.ID, err) + return + } + for _, m := range mails { + d.Mail = append(d.Mail, mailToJSON(m)) + } + entries, err := a.st.AuditFor(u.ID, 100) + if err != nil { + adminStoreFail(w, "list audit", u.ID, err) + return + } + for _, e := range entries { + d.Audit = append(d.Audit, auditJSON{ID: e.ID, At: rfc3339(e.At), ActorUserID: e.ActorUserID, + Action: e.Action, TargetUserID: e.TargetUserID, Detail: e.Detail}) + } + writeJSON(w, d) +} + +func (s *Server) handleAdminDisable(w http.ResponseWriter, r *http.Request, actor *users.User, _ *users.Session) { + t := s.adminTarget(w, r) + if t == nil || !s.guardRemoval(w, actor, t, false) { + return + } + if t.Status != users.StatusActive { + writeError(w, http.StatusConflict, "only active accounts can be disabled; delete or activate a pending account instead") + return + } + if err := s.auth.invalidateTokens("disable", t.ID, users.PurposeActivate, users.PurposeReset, users.PurposeEmailChange); err != nil { + writeError(w, http.StatusInternalServerError, "internal error") + return + } + if err := s.auth.st.SetStatus(t.ID, users.StatusDisabled); err != nil { + adminStoreFail(w, "disable", t.ID, err) + return + } + if err := s.auth.st.DeleteUserSessions(t.ID, 0); err != nil { + // currentUser rejects non-active users, so the sessions are dead + // anyway; keep going but leave a trace. + log.Printf("[users] disable: delete sessions for user #%d: %v", t.ID, err) + } + s.auth.audit(idPtr(actor.ID), "user.disable", idPtr(t.ID), nil) + s.writeAdminRow(w, t.ID) +} + +func (s *Server) handleAdminEnable(w http.ResponseWriter, r *http.Request, actor *users.User, _ *users.Session) { + t := s.adminTarget(w, r) + if t == nil { + return + } + if t.Status != users.StatusDisabled { + writeError(w, http.StatusConflict, "only disabled accounts can be enabled") + return + } + if err := s.auth.st.SetStatus(t.ID, users.StatusActive); err != nil { + adminStoreFail(w, "enable", t.ID, err) + return + } + s.auth.audit(idPtr(actor.ID), "user.enable", idPtr(t.ID), nil) + s.writeAdminRow(w, t.ID) +} + +func (s *Server) handleAdminDelete(w http.ResponseWriter, r *http.Request, actor *users.User, _ *users.Session) { + t := s.adminTarget(w, r) + if t == nil || !s.guardRemoval(w, actor, t, false) { + return + } + if err := s.auth.st.Delete(t.ID); err != nil { + adminStoreFail(w, "delete", t.ID, err) + return + } + s.auth.audit(idPtr(actor.ID), "user.delete", idPtr(t.ID), nil) + writeJSON(w, okResponse{OK: true}) +} + +func (s *Server) handleAdminRole(w http.ResponseWriter, r *http.Request, actor *users.User, _ *users.Session) { + t := s.adminTarget(w, r) + if t == nil { + return + } + var req roleRequest + if !decodeJSON(w, r, &req) { + return + } + if !req.Role.Valid() { + writeError(w, http.StatusBadRequest, "role must be user or admin") + return + } + // Activation sets the role from adminEmails, so a change now would be + // silently lost. + if t.Status == users.StatusPending { + writeError(w, http.StatusConflict, "activate the account first") + return + } + if req.Role == t.Role { + s.writeAdminRow(w, t.ID) + return + } + if req.Role == users.RoleUser && !s.guardRemoval(w, actor, t, true) { + return + } + if err := s.auth.st.SetRole(t.ID, req.Role); err != nil { + adminStoreFail(w, "set role", t.ID, err) + return + } + s.auth.audit(idPtr(actor.ID), "user.role", idPtr(t.ID), map[string]string{"role": string(req.Role)}) + s.writeAdminRow(w, t.ID) +} + +func (s *Server) handleAdminResendActivation(w http.ResponseWriter, r *http.Request, actor *users.User, _ *users.Session) { + a := s.auth + t := s.adminTarget(w, r) + if t == nil { + return + } + if t.Status != users.StatusPending { + writeError(w, http.StatusConflict, "only pending accounts need activation") + return + } + if err := a.mailToken(r.Context(), t, users.PurposeActivate, 48*time.Hour, "", "activate", + func(tok string) mailer.Message { return a.activationMail(t, tok) }); err != nil { + writeError(w, http.StatusServiceUnavailable, msgMailFailed) + return + } + a.audit(idPtr(actor.ID), "user.activation.resend", idPtr(t.ID), nil) + s.writeAdminRow(w, t.ID) +} + +// handleAdminActivate activates a pending user without the mail link, for +// when mail keeps failing. The address stays unverified; activated_by and +// the audit row keep that visible. +func (s *Server) handleAdminActivate(w http.ResponseWriter, r *http.Request, actor *users.User, _ *users.Session) { + a := s.auth + t := s.adminTarget(w, r) + if t == nil { + return + } + if t.Status != users.StatusPending { + writeError(w, http.StatusConflict, "only pending accounts can be activated") + return + } + if err := a.st.InvalidateTokens(t.ID, users.PurposeActivate); err != nil { + log.Printf("[users] manual activate: invalidate tokens for user #%d: %v", t.ID, err) + writeError(w, http.StatusInternalServerError, "internal error") + return + } + if err := a.st.Activate(t.ID, a.roleFor(t.Email), idPtr(actor.ID)); err != nil { + adminStoreFail(w, "activate", t.ID, err) + return + } + a.audit(idPtr(actor.ID), "user.activate.manual", idPtr(t.ID), nil) + s.writeAdminRow(w, t.ID) +} + +func (s *Server) handleAdminMailRefresh(w http.ResponseWriter, r *http.Request, actor *users.User, _ *users.Session) { + a := s.auth + t := s.adminTarget(w, r) + if t == nil { + return + } + mailID, err := strconv.ParseInt(mux.Vars(r)["mailId"], 10, 64) + if err != nil { + writeError(w, http.StatusBadRequest, "invalid mail id") + return + } + rec, err := a.st.MailByID(mailID) + if err != nil && !errors.Is(err, users.ErrNotFound) { + log.Printf("[users] admin mail refresh: load mail %d for user #%d: %v", mailID, t.ID, err) + writeError(w, http.StatusInternalServerError, "internal error") + return + } + if err != nil || rec.UserID == nil || *rec.UserID != t.ID { + writeError(w, http.StatusNotFound, "mail not found") + return + } + if rec.ProviderMessageID == "" { + writeError(w, http.StatusConflict, "this mail has no provider message id") + return + } + evs, err := a.mail.Events(r.Context(), rec.ProviderMessageID) + if err != nil { + log.Printf("[users] admin mail refresh: provider events for user #%d: %s", t.ID, redactAddrs(err)) + writeError(w, http.StatusBadGateway, "mail provider unavailable") + return + } + a.ingestMailEvents(evs) + a.audit(idPtr(actor.ID), "user.mail.refresh", idPtr(t.ID), nil) + rec, err = a.st.MailByID(mailID) + if err != nil { + log.Printf("[users] admin mail refresh: reload mail %d for user #%d: %v", mailID, t.ID, err) + writeError(w, http.StatusInternalServerError, "internal error") + return + } + writeJSON(w, mailToJSON(*rec)) +} diff --git a/cmd/server/admin_users_test.go b/cmd/server/admin_users_test.go new file mode 100644 index 000000000..a1b26f355 --- /dev/null +++ b/cmd/server/admin_users_test.go @@ -0,0 +1,235 @@ +package main + +import ( + "fmt" + "strings" + "testing" + + "github.com/meshcore-analyzer/mailer" + "github.com/meshcore-analyzer/users" +) + +// adminFixture: config admin "boss" plus regular user "uma". +func adminFixture(t *testing.T) (*authFixture, *client, *client) { + f := newAuthFixture(t, "boss@example.org") + return f, f.registerAndActivate(t, "boss@example.org", "Boss", pw), f.registerAndActivate(t, "uma@example.org", "Uma", pw) +} + +func userPath(id int64, suffix string) string { + return fmt.Sprintf("/api/admin/users/%d%s", id, suffix) +} + +func TestAdminUsersRequiresAdmin(t *testing.T) { + f, _, uma := adminFixture(t) + expectStatus(t, f.do("GET", "/api/admin/users", nil), 401) + expectStatus(t, f.do("GET", "/api/admin/users", nil, as(uma)), 403) +} + +func TestAdminListAndDetail(t *testing.T) { + f, boss, uma := adminFixture(t) + f.do("POST", "/api/auth/register", registerRequest{Email: "pending@example.org", DisplayName: "Pen", Password: pw}) + w := f.do("GET", "/api/admin/users?status=pending", nil, as(boss)) + expectStatus(t, w, 200) + rows := decode[[]adminUserJSON](t, w) + if len(rows) != 1 || rows[0].Email != "pending@example.org" || rows[0].LastMail == nil || rows[0].LastMail.Purpose != "activate" { + t.Fatalf("pending rows = %+v", rows) + } + expectStatus(t, f.do("GET", "/api/admin/users?status=bogus", nil, as(boss)), 400) + w = f.do("GET", userPath(uma.me.ID, ""), nil, as(boss)) + expectStatus(t, w, 200) + d := decode[adminUserDetailJSON](t, w) + if d.User.Email != "uma@example.org" || len(d.Sessions) != 1 || len(d.Mail) != 1 || len(d.Audit) == 0 { + t.Fatalf("detail = %+v", d) + } + all := decode[[]adminUserJSON](t, f.do("GET", "/api/admin/users", nil, as(boss))) + for _, r := range all { + if r.Email == "boss@example.org" && !r.ConfigAdmin { + t.Fatal("config admin not marked") + } + } +} + +func TestAdminListBouncingFilter(t *testing.T) { + f, boss, uma := adminFixture(t) + if err := f.st.SetEmailBouncing(uma.me.ID, true); err != nil { + t.Fatal(err) + } + w := f.do("GET", "/api/admin/users?bouncing=1", nil, as(boss)) + expectStatus(t, w, 200) + rows := decode[[]adminUserJSON](t, w) + if len(rows) != 1 || rows[0].Email != "uma@example.org" || !rows[0].EmailBouncing { + t.Fatalf("bouncing rows = %+v", rows) + } + if all := decode[[]adminUserJSON](t, f.do("GET", "/api/admin/users", nil, as(boss))); len(all) != 2 { + t.Fatalf("unfiltered rows = %d, want 2", len(all)) + } + for _, v := range []string{"0", "yes", "true"} { + expectStatus(t, f.do("GET", "/api/admin/users?bouncing="+v, nil, as(boss)), 400) + } +} + +func TestAdminDisableEnable(t *testing.T) { + f, boss, uma := adminFixture(t) + expectStatus(t, f.do("POST", userPath(uma.me.ID, "/disable"), nil, as(boss)), 200) + expectStatus(t, f.do("GET", "/api/auth/me", nil, as(uma)), 401) + expectStatus(t, f.do("POST", "/api/auth/login", loginRequest{Email: "uma@example.org", Password: pw}), 401) + expectStatus(t, f.do("POST", userPath(uma.me.ID, "/disable"), nil, as(boss)), 409) + expectStatus(t, f.do("POST", userPath(uma.me.ID, "/enable"), nil, as(boss)), 200) + f.login(t, "uma@example.org", pw) +} + +func TestAdminGuards(t *testing.T) { + f, boss, uma := adminFixture(t) + expectStatus(t, f.do("POST", userPath(boss.me.ID, "/disable"), nil, as(boss)), 409) // self + expectStatus(t, f.do("POST", userPath(uma.me.ID, "/role"), roleRequest{Role: users.RoleAdmin}, as(boss)), 200) + umaAdmin := f.login(t, "uma@example.org", pw) + expectStatus(t, f.do("POST", userPath(boss.me.ID, "/disable"), nil, as(umaAdmin)), 409) // config admin + expectStatus(t, f.do("DELETE", userPath(boss.me.ID, ""), nil, as(umaAdmin)), 409) // config admin + expectStatus(t, f.do("POST", userPath(boss.me.ID, "/role"), roleRequest{Role: users.RoleUser}, as(umaAdmin)), 409) // config admin + expectStatus(t, f.do("POST", userPath(uma.me.ID, "/role"), roleRequest{Role: "root"}, as(boss)), 400) + // uma may demote herself: boss remains. + expectStatus(t, f.do("POST", userPath(uma.me.ID, "/role"), roleRequest{Role: users.RoleUser}, as(umaAdmin)), 200) +} + +func TestAdminLastAdminCannotDemoteSelf(t *testing.T) { + f := newAuthFixture(t) + solo := f.registerAndActivate(t, "solo@example.org", "Solo", pw) + f.st.SetRole(solo.me.ID, users.RoleAdmin) // UI-promoted, not a config admin + expectStatus(t, f.do("POST", userPath(solo.me.ID, "/role"), roleRequest{Role: users.RoleUser}, as(solo)), 409) +} + +func TestAdminManualActivate(t *testing.T) { + f, boss, _ := adminFixture(t) + f.do("POST", "/api/auth/register", registerRequest{Email: "late@example.org", DisplayName: "Late", Password: pw}) + link := f.lastToken(t) + late, _ := f.st.GetByEmail("late@example.org") + expectStatus(t, f.do("POST", userPath(late.ID, "/activate"), nil, as(boss)), 200) + got, _ := f.st.GetByID(late.ID) + if got.Status != users.StatusActive || got.ActivatedBy == nil || *got.ActivatedBy != boss.me.ID { + t.Fatalf("after manual activate: %+v", got) + } + expectStatus(t, f.do("POST", "/api/auth/activate", activateRequest{Token: link, Password: pw}), 410) + expectStatus(t, f.do("POST", userPath(late.ID, "/activate"), nil, as(boss)), 409) + if !hasAudit(t, f, late.ID, "user.activate.manual") { + t.Fatal("no user.activate.manual audit row") + } + f.login(t, "late@example.org", pw) +} + +func TestAdminResendActivation(t *testing.T) { + f, boss, uma := adminFixture(t) + f.do("POST", "/api/auth/register", registerRequest{Email: "re@example.org", DisplayName: "Re", Password: pw}) + old := f.lastToken(t) + re, _ := f.st.GetByEmail("re@example.org") + expectStatus(t, f.do("POST", userPath(re.ID, "/resend-activation"), nil, as(boss)), 200) + fresh := f.lastToken(t) + if fresh == old { + t.Fatal("no new link sent") + } + expectStatus(t, f.do("POST", "/api/auth/activate", activateRequest{Token: old, Password: pw}), 410) + expectStatus(t, f.do("POST", "/api/auth/activate", activateRequest{Token: fresh, Password: pw}), 200) + expectStatus(t, f.do("POST", userPath(uma.me.ID, "/resend-activation"), nil, as(boss)), 409) +} + +func TestAdminDelete(t *testing.T) { + f, boss, uma := adminFixture(t) + expectStatus(t, f.do("DELETE", userPath(uma.me.ID, ""), nil, as(boss)), 200) + expectStatus(t, f.do("GET", userPath(uma.me.ID, ""), nil, as(boss)), 404) +} + +func TestAdminMailRefresh(t *testing.T) { + f, boss, uma := adminFixture(t) + mails, _ := f.st.MailForUser(uma.me.ID, 10) + m := mails[0] + f.fake.SetEvents(m.ProviderMessageID, []mailer.Event{ + {MessageID: m.ProviderMessageID, Event: mailer.EventHardBounce, Reason: "user unknown", At: m.SentAt.Add(60e9)}, + }) + w := f.do("POST", userPath(uma.me.ID, fmt.Sprintf("/mail/%d/refresh", m.ID)), nil, as(boss)) + expectStatus(t, w, 200) + if got := decode[mailJSON](t, w); got.LastEvent != mailer.EventHardBounce || got.LastReason != "user unknown" { + t.Fatalf("refreshed = %+v", got) + } + if !hasAudit(t, f, uma.me.ID, "user.mail.refresh") { + t.Fatal("no user.mail.refresh audit row") + } + if u, _ := f.st.GetByID(uma.me.ID); !u.EmailBouncing { + t.Fatal("hard bounce did not flag the address") + } + expectStatus(t, f.do("POST", userPath(boss.me.ID, fmt.Sprintf("/mail/%d/refresh", m.ID)), nil, as(boss)), 404) // wrong owner +} + +func hasAudit(t *testing.T, f *authFixture, userID int64, action string) bool { + t.Helper() + entries, err := f.st.AuditFor(userID, 50) + if err != nil { + t.Fatal(err) + } + for _, e := range entries { + if e.Action == action { + return true + } + } + return false +} + +// A pending row squatting on a config-admin address is not an admin yet and +// stays deletable. +func TestAdminDeletePendingSquatterOnConfigAddress(t *testing.T) { + f, boss, _ := adminFixture(t) + f.do("POST", "/api/auth/register", registerRequest{Email: "squat@example.org", DisplayName: "Sq", Password: pw}) + f.srv.auth.set.adminEmails["squat@example.org"] = true + sq, _ := f.st.GetByEmail("squat@example.org") + expectStatus(t, f.do("DELETE", userPath(sq.ID, ""), nil, as(boss)), 200) + if _, err := f.st.GetByID(sq.ID); err == nil { + t.Fatal("pending squatter still present") + } +} + +func TestAdminUnknownAndBadID(t *testing.T) { + f, boss, _ := adminFixture(t) + expectStatus(t, f.do("POST", userPath(9999, "/disable"), nil, as(boss)), 404) + expectStatus(t, f.do("GET", "/api/admin/users/abc", nil, as(boss)), 400) +} + +// A pending account must go through activation; disable then enable must not +// be a way around it. +func TestAdminCannotDisablePending(t *testing.T) { + f, boss, _ := adminFixture(t) + f.do("POST", "/api/auth/register", registerRequest{Email: "pend@example.org", DisplayName: "Pe", Password: pw}) + p, _ := f.st.GetByEmail("pend@example.org") + expectStatus(t, f.do("POST", userPath(p.ID, "/disable"), nil, as(boss)), 409) + got, _ := f.st.GetByID(p.ID) + if got.Status != users.StatusPending { + t.Fatalf("status = %s, want pending", got.Status) + } + expectStatus(t, f.do("POST", userPath(p.ID, "/enable"), nil, as(boss)), 409) +} + +func TestAdminDisableKillsPendingLinks(t *testing.T) { + f, boss, uma := adminFixture(t) + confirm, reset := pendingLinks(t, f, uma, "uma@example.org", "attacker@example.org") + expectStatus(t, f.do("POST", userPath(uma.me.ID, "/disable"), nil, as(boss)), 200) + expectStatus(t, f.do("POST", userPath(uma.me.ID, "/enable"), nil, as(boss)), 200) + expectStatus(t, f.do("POST", "/api/account/confirm-email", tokenRequest{Token: confirm}), 410) + expectStatus(t, f.do("POST", "/api/auth/reset", resetRequest{Token: reset, Password: "another new secret"}), 410) +} + +func TestAdminDisableTokenStoreFailureIs500(t *testing.T) { + f, boss, uma := adminFixture(t) + f.breakTable(t, "tokens") + expectStatus(t, f.do("POST", userPath(uma.me.ID, "/disable"), nil, as(boss)), 500) +} + +func TestAdminRoleChangeOnPendingIs409(t *testing.T) { + f, boss, _ := adminFixture(t) + f.do("POST", "/api/auth/register", registerRequest{Email: "pend@example.org", DisplayName: "Pend", Password: pw}) + p, _ := f.st.GetByEmail("pend@example.org") + w := f.do("POST", userPath(p.ID, "/role"), roleRequest{Role: users.RoleAdmin}, as(boss)) + expectStatus(t, w, 409) + if !strings.Contains(w.Body.String(), "activate the account first") { + t.Fatalf("body = %s", w.Body.String()) + } + if got, _ := f.st.GetByID(p.ID); got.Role != users.RoleUser { + t.Fatalf("role changed on a pending user: %+v", got) + } +} diff --git a/cmd/server/advert_evidence_scale_test.go b/cmd/server/advert_evidence_scale_test.go new file mode 100644 index 000000000..eed081e8d --- /dev/null +++ b/cmd/server/advert_evidence_scale_test.go @@ -0,0 +1,105 @@ +//go:build advert_evidence_scale + +package main + +import ( + "fmt" + "os" + "testing" + "time" +) + +// Run after the ingestor's TestAdvertBackfillScale with the same ADVERT_SCALE_DB. +// Models a server starting just before backfill: its cursor is zero and retained +// packets initially have no evidence. Uses the default 1-second poll cadence, +// with one real cached analytics request per tick and a 300K-packet retention. +func TestAdvertEvidenceCatchupScale(t *testing.T) { + path := os.Getenv("ADVERT_SCALE_DB") + if path == "" { + t.Fatal("ADVERT_SCALE_DB must name the completed ingestor scale fixture") + } + db, err := openFixtureReader(t, path) + if err != nil { + t.Fatal(err) + } + defer db.conn.Close() + var horizon int64 + var evidenceCount int + if err := db.conn.QueryRow(`SELECT MAX(id),COUNT(*) FROM advert_route_evidence`).Scan(&horizon, &evidenceCount); err != nil { + t.Fatal(err) + } + if evidenceCount < 200000 { + t.Fatal("fixture must contain the completed 11M-observation evidence set") + } + const size = 300000 + packets := make([]*StoreTx, size) + for i := range packets { + pt := PayloadGRP_TXT + if (i+1)%10 == 0 { + pt = PayloadADVERT + } + tx := makeRelayAirtimeTx(i+1, pt, 120, 0, fmt.Sprintf("scale-%d", i+1)) + tx.FirstSeen = time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC).Add(time.Duration(i) * (7 * 24 * time.Hour / size)).Format(time.RFC3339) + packets[i] = tx + } + s := newRelayAirtimeShareTestStore(packets) + s.db = db + s.rfCacheTTL = time.Hour + window := TimeWindow{Since: "2026-01-07T23:00:00Z", Until: "2026-01-08T00:00:00Z", Label: "1h"} + initial := s.GetRelayAirtimeShareWithWindow(window) + if initial["total_count"] != 1785 { + t.Fatalf("window fixture selected %v, want1785", initial["total_count"]) + } + var initialAdverts int + for _, row := range initial["rows"].([]map[string]interface{}) { + if row["advert_kind"] == "other" { + initialAdverts = row["count"].(int) + } + } + if initialAdverts != 179 { + t.Fatalf("initial unknown adverts=%d want179", initialAdverts) + } + started := time.Now() + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + ticks := 0 + var computeTotal, timeMax, retainedReady time.Duration + for s.advertEvidenceCursor < horizon { + <-ticker.C + ticks++ + if err := s.pollAdvertEvidence(500); err != nil { + t.Fatal(err) + } + if retainedReady == 0 && s.byTxID[size].AdvertRouteEvidence == 3 { + retainedReady = time.Since(started) + } + at := time.Now() + s.GetRelayAirtimeShareWithWindow(window) + d := time.Since(at) + computeTotal += d + if d > timeMax { + timeMax = d + } + } + mixed := 0 + for _, tx := range packets { + if tx.AdvertRouteEvidence == 3 { + mixed++ + } + } + final := s.GetRelayAirtimeShareWithWindow(window) + var finalAdverts int + for _, row := range final["rows"].([]map[string]interface{}) { + if row["advert_kind"] == "mixed" { + finalAdverts = row["count"].(int) + } + } + if finalAdverts != 179 || final["total_count"] != 1785 { + t.Fatalf("final classification/window changed: %v", final) + } + t.Logf("eligible=1785 eligible_adverts=179 final_cursor=%d horizon=%d", s.advertEvidenceCursor, horizon) + t.Logf("CATCHUP evidence_rows=%d retained_packets=%d adverts=30000 poll_rows=500 cadence=1s elapsed=%s ticks=%d cache_misses=%d revisions=%d request_total=%s request_max=%s retained_ready=%s mixed_retained=%d", evidenceCount, size, time.Since(started), ticks, s.cacheMisses, s.advertEvidenceRevision, computeTotal, timeMax, retainedReady, mixed) + if mixed != 30000 { + t.Fatalf("lost retained evidence: mixed=%d want30000", mixed) + } +} diff --git a/cmd/server/advert_route_evidence.go b/cmd/server/advert_route_evidence.go new file mode 100644 index 000000000..9858483d5 --- /dev/null +++ b/cmd/server/advert_route_evidence.go @@ -0,0 +1,170 @@ +package main + +import ( + "fmt" + "log" + "strings" +) + +// The table is optional for legacy read-only databases. Re-probe while absent +// so a concurrently starting ingestor can create it without a server restart. +func (db *DB) advertEvidencePresent() bool { + if db == nil || db.conn == nil { + return false + } + if db.advertEvidenceTable.Load() { + return true + } + var exists int + if err := db.conn.QueryRow(db.nativeSQL("SELECT 1 FROM sqlite_master WHERE type='table' AND name='advert_route_evidence'", `SELECT 1 FROM information_schema.tables WHERE table_schema=current_schema() AND table_name='advert_route_evidence'`)).Scan(&exists); err != nil { + return false + } + db.advertEvidenceTable.Store(true) + return true +} + +// At most two indexed evidence rows per selected transmission, fetched in +// batches rather than per-packet round trips. No raw frames enter the store. +func (db *DB) advertEvidenceForIDs(ids []int) (map[int]uint8, error) { + result := make(map[int]uint8, len(ids)) + if len(ids) == 0 || !db.advertEvidencePresent() { + return result, nil + } + for start := 0; start < len(ids); start += 500 { + end := start + 500 + if end > len(ids) { + end = len(ids) + } + args := make([]interface{}, end-start) + for i, id := range ids[start:end] { + args[i] = id + } + query := `SELECT tx_id,SUM(bit) FROM advert_route_evidence WHERE tx_id IN (` + db.sqlPlaceholders(len(args)) + `) GROUP BY tx_id` + if db.advertEvidenceReadHook != nil { + db.advertEvidenceReadHook() + } + rows, err := db.conn.Query(query, args...) + if err != nil { + return nil, err + } + for rows.Next() { + var id int + var mask uint8 + if err := rows.Scan(&id, &mask); err != nil { + rows.Close() + return nil, err + } + result[id] = mask + } + err = rows.Err() + rows.Close() + if err != nil { + return nil, err + } + } + return result, nil +} + +// Caller holds mu. Unioning evidence is independent of cache invalidation so +// loaders and polls can invalidate once per batch, not once per transmission. +func unionAdvertEvidence(tx *StoreTx, mask uint8) bool { + if tx == nil || tx.AdvertRouteEvidence|mask == tx.AdvertRouteEvidence { + return false + } + tx.AdvertRouteEvidence |= mask + return true +} + +// Caller holds mu. The revision prevents an in-flight old computation from +// repopulating an entry after the evidence that produced it has changed. +func (s *PacketStore) invalidateAdvertEvidence() { + s.cacheMu.Lock() + s.advertEvidenceRevision++ + for key := range s.rfCache { + if strings.HasPrefix(key, "relay-airtime-share|") { + delete(s.rfCache, key) + } + } + s.cacheMu.Unlock() +} + +func advertTxIDs(txs []*StoreTx) []int { + ids := make([]int, 0, len(txs)) + for _, tx := range txs { + if tx != nil && tx.PayloadType != nil && *tx.PayloadType == PayloadADVERT { + ids = append(ids, tx.ID) + } + } + return ids +} + +// Caller holds mu. A transmission evicted during the mask read is skipped; +// its persisted union will be fetched if it is loaded again. +func (s *PacketStore) mergeAdvertEvidence(masks map[int]uint8) { + changed := false + for id, mask := range masks { + if unionAdvertEvidence(s.byTxID[id], mask) { + changed = true + } + } + if changed { + s.invalidateAdvertEvidence() + } +} + +func (s *PacketStore) pollAdvertEvidence(limit int) error { + s.advertEvidenceMu.Lock() + defer s.advertEvidenceMu.Unlock() + if !s.db.advertEvidencePresent() { + return nil + } + if limit <= 0 || limit > 500 { + limit = 500 + } + rows, err := s.db.conn.Query(`SELECT id,tx_id,bit FROM advert_route_evidence WHERE id>`+s.db.parameter(1)+` ORDER BY id LIMIT `+s.db.parameter(2), s.advertEvidenceCursor, limit) + if err != nil { + return err + } + type event struct { + id int64 + txID int + bit uint8 + } + var batch []event + for rows.Next() { + var e event + if err := rows.Scan(&e.id, &e.txID, &e.bit); err != nil { + rows.Close() + return err + } + if e.bit != 1 && e.bit != 2 { + rows.Close() + return fmt.Errorf("invalid advert evidence bit %d", e.bit) + } + batch = append(batch, e) + } + err = rows.Err() + rows.Close() + if err != nil { + return err + } + s.mu.Lock() + defer s.mu.Unlock() + changed := false + for _, e := range batch { + if unionAdvertEvidence(s.byTxID[e.txID], e.bit) { + changed = true + } + s.advertEvidenceCursor = e.id + } + if changed { + s.invalidateAdvertEvidence() + } + return nil +} + +func (s *PacketStore) refreshAdvertEvidence() { + if err := s.pollAdvertEvidence(500); err != nil { + log.Printf("[store] advert evidence poll: %v", err) + } +} diff --git a/cmd/server/advert_route_evidence_test.go b/cmd/server/advert_route_evidence_test.go new file mode 100644 index 000000000..59041b3fc --- /dev/null +++ b/cmd/server/advert_route_evidence_test.go @@ -0,0 +1,500 @@ +package main + +import ( + "database/sql" + "fmt" + "testing" + "time" + + "github.com/meshcore-analyzer/lora" +) + +// Writer-side ingestion is exercised in cmd/ingestor. This fixture represents +// its durable output after F->Z->F (or Z->F->Z), where surviving raw frames +// alone cannot reconstruct the complete set of known route families. +func advertEvidenceFixture(t *testing.T, raw string, bits ...int) (*DB, *sql.DB) { + t.Helper() + path := createTestDBWithResolvedPath(t, 1, []string{"fixture-relay"}) + w, err := openFixtureSQL(path) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { w.Close() }) + for _, stmt := range []string{ + `ALTER TABLE transmissions ADD COLUMN from_pubkey TEXT`, + testNativeSQL(`CREATE TABLE advert_route_evidence (id INTEGER PRIMARY KEY AUTOINCREMENT, tx_id INTEGER NOT NULL REFERENCES transmissions(id) ON DELETE CASCADE, bit INTEGER NOT NULL CHECK (bit IN (1,2)), UNIQUE(tx_id,bit))`, `CREATE TABLE advert_route_evidence (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, tx_id INTEGER NOT NULL REFERENCES transmissions(id) ON DELETE CASCADE, bit INTEGER NOT NULL CHECK (bit IN (1,2)), UNIQUE(tx_id,bit))`), + } { + if _, err := w.Exec(stmt); err != nil { + t.Fatal(err) + } + } + now := time.Now().UTC().Add(-time.Minute).Format(time.RFC3339) + if _, err := w.Exec(`UPDATE transmissions SET raw_hex=$1, route_type=$2, first_seen=$3, from_pubkey='fixture-origin'`, raw, int(raw[1]-'0')&3, now); err != nil { + t.Fatal(err) + } + if _, err := w.Exec(`UPDATE observations SET raw_hex=$1, timestamp=$2`, raw, now); err != nil { + t.Fatal(err) + } + for _, bit := range bits { + if _, err := w.Exec(`INSERT INTO advert_route_evidence(tx_id,bit) VALUES(1,$1)`, bit); err != nil { + t.Fatal(err) + } + } + db, err := openFixtureReader(t, path) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { db.conn.Close() }) + return db, w +} + +func TestAdvertRouteEvidenceNodeAPIHonorsRequestedLimit(t *testing.T) { + db, w := advertEvidenceFixture(t, "1100aa", 1) + for id := 2; id <= 24; id++ { + if _, err := w.Exec(`INSERT INTO transmissions(id,hash,raw_hex,first_seen,payload_type,route_type,from_pubkey) SELECT $1,$2,'1100aa',first_seen,4,1,'fixture-origin' FROM transmissions WHERE id=1`, id, fmt.Sprintf("node-advert-%d", id)); err != nil { + t.Fatal(err) + } + mask := id % 4 + for _, bit := range []int{1, 2} { + if mask&bit != 0 { + if _, err := w.Exec(`INSERT INTO advert_route_evidence(tx_id,bit) VALUES($1,$2)`, id, bit); err != nil { + t.Fatal(err) + } + } + } + } + rows, err := db.GetRecentTransmissionsForNode("fixture-origin", 1000) + if err != nil { + t.Fatal(err) + } + if len(rows) != 24 { + t.Fatalf("node recent adverts returned %d, want requested 24 available rows", len(rows)) + } + wants := []string{"other", "flood", "zero_hop", "mixed"} + for _, row := range rows { + id := row["id"].(int) + if row["advert_kind"] != wants[id%4] { + t.Errorf("id=%d kind=%v want %s", id, row["advert_kind"], wants[id%4]) + } + } +} + +// Measures the added bulk-load work and an idle feed at a 30K-packet scale. +// Both paths touch indexed compact evidence, never observation raw frames. +func BenchmarkAdvertEvidence30K(b *testing.B) { + conn, err := openPostgresTestDB(b) + if err != nil { + b.Fatal(err) + } + conn.SetMaxOpenConns(1) + defer conn.Close() + if _, err := conn.Exec(testNativeSQL(`CREATE TABLE advert_route_evidence(id INTEGER PRIMARY KEY AUTOINCREMENT,tx_id INTEGER,bit INTEGER,UNIQUE(tx_id,bit)); + WITH RECURSIVE ids(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM ids WHERE n<30000) + INSERT INTO advert_route_evidence(tx_id,bit) SELECT n,1 FROM ids UNION ALL SELECT n,2 FROM ids`, `CREATE TABLE advert_route_evidence(id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,tx_id INTEGER,bit INTEGER,UNIQUE(tx_id,bit)); + WITH RECURSIVE ids(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM ids WHERE n<30000) + INSERT INTO advert_route_evidence(tx_id,bit) SELECT n,1 FROM ids UNION ALL SELECT n,2 FROM ids`)); err != nil { + b.Fatal(err) + } + db := &DB{backend: testBackendValue(), conn: conn} + db.advertEvidenceTable.Store(true) + ids := make([]int, 30000) + for i := range ids { + ids[i] = i + 1 + } + b.Run("bulk_load", func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + masks, err := db.advertEvidenceForIDs(ids) + if err != nil || len(masks) != 30000 { + b.Fatalf("mask load: %d %v", len(masks), err) + } + } + }) + b.Run("idle_poll", func(b *testing.B) { + s := NewPacketStore(db, &PacketStoreConfig{}) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if err := s.pollAdvertEvidence(500); err != nil { + b.Fatal(err) + } + } + }) +} + +func assertAdvertEvidenceViews(t *testing.T, s *PacketStore, want string) { + t.Helper() + result := s.GetRelayAirtimeShareWithWindow(TimeWindow{}) + rows := result["rows"].([]map[string]interface{}) + if len(rows) != 1 || rows[0]["advert_kind"] != want || rows[0]["count"] != 1 || result["total_count"] != 1 { + t.Errorf("relay airtime must count one hash as %s: %v", want, result) + } + if len(s.packets) != 1 { + t.Fatalf("loaded %d transmissions, want 1", len(s.packets)) + } + tx := s.packets[0] + // Classification must not change the established airtime formula, even + // when evidence includes zero-hop and the packet has a resolved relay. + wantScore := int64(lora.TimeOnAir(len(tx.RawHex)/2, defaultLoRaPreset())) * int64(s.distinctRelayCount(tx)) + if result["total_score"] != wantScore { + t.Errorf("airtime changed: %v, want %d", result["total_score"], wantScore) + } + for _, obs := range tx.Observations { + if obs.RawHex != "" { + t.Error("route evidence must not retain raw frames on StoreObs") + } + } + adverts, err := s.db.GetRecentTransmissionsForNode("fixture-origin", 20) + if err != nil { + t.Fatal(err) + } + if len(adverts) != 1 || adverts[0]["advert_kind"] != want { + t.Errorf("node API must agree on %s: %v", want, adverts) + } +} + +func TestAdvertRouteEvidenceLoadPaths(t *testing.T) { + for _, raw := range []string{"1100aa", "1200aa", "100102030400aa", "130102030400aa"} { + for _, mode := range []string{"cold", "chunk", "chunked_startup", "new_transmission"} { + t.Run(raw+"/"+mode, func(t *testing.T) { + db, _ := advertEvidenceFixture(t, raw, 1, 2) + s := NewPacketStore(db, &PacketStoreConfig{}) + s.useResolvedPathIndex = true + s.initResolvedPathIndex() + maskReads := 0 + db.advertEvidenceReadHook = func() { + maskReads++ + if !s.mu.TryRLock() { + t.Error("bulk evidence SQL must run outside the global store lock") + } else { + s.mu.RUnlock() + } + } + switch mode { + case "chunked_startup": + if err := s.LoadChunked(1); err != nil { + t.Fatal(err) + } + case "cold": + if err := s.Load(); err != nil { + t.Fatal(err) + } + case "chunk": + if err := s.loadChunk(time.Now().Add(-time.Hour), time.Now()); err != nil { + t.Fatal(err) + } + case "new_transmission": + s.IngestNewFromDB(0, 100) + } + if maskReads == 0 { + t.Fatal("load path never fetched durable evidence") + } + db.advertEvidenceReadHook = nil + assertAdvertEvidenceViews(t, s, "mixed") + }) + } + } +} + +func TestAdvertRouteEvidencePollRetryAndUnknownTransmission(t *testing.T) { + db, w := advertEvidenceFixture(t, "1100aa", 1) + s := NewPacketStore(db, &PacketStoreConfig{}) + if err := s.Load(); err != nil { + t.Fatal(err) + } + before := s.advertEvidenceCursor + if _, err := w.Exec(`ALTER TABLE advert_route_evidence RENAME TO evidence_unavailable`); err != nil { + t.Fatal(err) + } + if err := s.pollAdvertEvidence(1); err == nil { + t.Fatal("expected failed feed read") + } + if s.advertEvidenceCursor != before { + t.Fatal("failed feed read advanced cursor") + } + if _, err := w.Exec(`ALTER TABLE evidence_unavailable RENAME TO advert_route_evidence; + INSERT INTO transmissions(id,hash,raw_hex,first_seen,payload_type,route_type,from_pubkey) SELECT 2,'late-advert','1100aa',first_seen,4,1,'fixture-late' FROM transmissions WHERE id=1; + INSERT INTO advert_route_evidence(tx_id,bit) VALUES(2,1),(2,2),(1,2)`); err != nil { + t.Fatal(err) + } + if err := s.pollAdvertEvidence(1); err != nil { + t.Fatal(err) + } + if s.advertEvidenceCursor != before+1 { + t.Fatal("feed ignored batch limit") + } + if s.byTxID[2] != nil { + t.Fatal("feed should not retain unknown transmissions") + } + if err := s.pollAdvertEvidence(500); err != nil { + t.Fatal(err) + } + if s.byTxID[1].AdvertRouteEvidence != 3 { + t.Fatal("retry missed known transmission's late evidence") + } + s.IngestNewFromDB(1, 100) + if tx := s.byTxID[2]; tx == nil || tx.AdvertRouteEvidence != 3 { + t.Fatalf("load after feed consumed unknown event lost mask: %+v", tx) + } +} + +func TestAdvertRouteEvidenceStartupHandoffAndChunkMerge(t *testing.T) { + db, w := advertEvidenceFixture(t, "1100aa", 1) + s := NewPacketStore(db, &PacketStoreConfig{}) + // Arrives after the startup watermark, before the first packet load. + if _, err := w.Exec(`INSERT INTO advert_route_evidence(tx_id,bit) VALUES(1,2)`); err != nil { + t.Fatal(err) + } + if err := s.pollAdvertEvidence(500); err != nil { + t.Fatal(err) + } + if err := s.LoadChunked(1); err != nil { + t.Fatal(err) + } + if s.byTxID[1].AdvertRouteEvidence != 3 { + t.Fatal("startup handoff lost pre-load event") + } + if err := s.loadChunk(time.Now().Add(-time.Hour), time.Now()); err != nil { + t.Fatal(err) + } + for _, tx := range s.packets { + if tx.AdvertRouteEvidence != 3 { + t.Fatal("chunk merge discarded durable bits") + } + } + rows := s.computeRelayAirtimeShare(TimeWindow{})["rows"].([]map[string]interface{}) + if len(rows) != 1 || rows[0]["advert_kind"] != "mixed" || rows[0]["count"] != 1 { + t.Fatalf("chunk merge double-counted mixed hash: %v", rows) + } +} + +func TestAdvertRouteEvidenceEvictionReload(t *testing.T) { + db, _ := advertEvidenceFixture(t, "1200aa", 1, 2) + s := NewPacketStore(db, &PacketStoreConfig{RetentionHours: 1}) + if err := s.Load(); err != nil { + t.Fatal(err) + } + s.mu.Lock() + s.packets[0].FirstSeen = time.Now().Add(-48 * time.Hour).UTC().Format(time.RFC3339) + s.packets[0].LatestSeen = s.packets[0].FirstSeen + n := s.EvictStale() + s.mu.Unlock() + if n != 1 || s.byTxID[1] != nil { + t.Fatalf("eviction removed %d, want 1", n) + } + s.IngestNewFromDB(0, 100) + if tx := s.byTxID[1]; tx == nil || tx.AdvertRouteEvidence != 3 { + t.Fatalf("reload lost persisted mask: %+v", tx) + } +} + +func TestAdvertRouteEvidencePollSameObservationID(t *testing.T) { + for _, tc := range []struct { + name, first, second, kind string + firstBit, secondBit int + }{ + {"flood_zero_flood", "1100aa", "1200aa", "flood", 1, 2}, + {"zero_flood_zero", "1200aa", "1100aa", "zero_hop", 2, 1}, + } { + t.Run(tc.name, func(t *testing.T) { + db, w := advertEvidenceFixture(t, tc.first, tc.firstBit) + s := NewPacketStore(db, &PacketStoreConfig{}) + s.useResolvedPathIndex = true + s.initResolvedPathIndex() + s.rfCacheTTL = time.Hour + if err := s.Load(); err != nil { + t.Fatal(err) + } + initial := s.GetRelayAirtimeShareWithWindow(TimeWindow{}) + if rows := initial["rows"].([]map[string]interface{}); len(rows) != 1 || rows[0]["advert_kind"] != tc.kind { + t.Fatalf("initial fixture classification: %v", initial) + } + unrelated := &cachedResult{expiresAt: time.Now().Add(time.Hour)} + s.rfCache["unrelated-rf-result"] = unrelated + maxObs, maxTx := db.GetMaxObservationID(), db.GetMaxTransmissionID() + // This matches the ingestor's conflict update: neither observation + // ID nor timestamp advances, and the final raw matches the first. + tx, err := w.Begin() + if err != nil { + t.Fatal(err) + } + defer tx.Rollback() + if _, err := tx.Exec(`UPDATE observations SET raw_hex=$1 WHERE id=1`, tc.second); err != nil { + t.Fatal(err) + } + if _, err := tx.Exec(`INSERT INTO advert_route_evidence(tx_id,bit) VALUES(1,$1)`, tc.secondBit); err != nil { + t.Fatal(err) + } + if _, err := tx.Exec(`UPDATE observations SET raw_hex=$1 WHERE id=1`, tc.first); err != nil { + t.Fatal(err) + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + s.IngestNewFromDB(maxTx, 100) + s.IngestNewObservations(maxObs, 100) + assertAdvertEvidenceViews(t, s, "mixed") + if s.rfCache["unrelated-rf-result"] != unrelated { + t.Error("route evidence invalidated unrelated RF cache") + } + if db.GetMaxObservationID() != maxObs || db.GetMaxTransmissionID() != maxTx { + t.Fatal("fixture unexpectedly created a new observation/transmission") + } + // A fresh store must recover the same classification from disk. + restarted := NewPacketStore(db, &PacketStoreConfig{}) + restarted.useResolvedPathIndex = true + restarted.initResolvedPathIndex() + if err := restarted.Load(); err != nil { + t.Fatal(err) + } + assertAdvertEvidenceViews(t, restarted, "mixed") + }) + } +} + +func TestAdvertRouteEvidenceMissingTableIsUnknown(t *testing.T) { + db, w := advertEvidenceFixture(t, "1100aa") + if _, err := w.Exec(`DROP TABLE advert_route_evidence`); err != nil { + t.Fatal(err) + } + // Reopen after removing the table to exercise legacy read-only detection. + legacy, err := OpenDB(db.path) + if err != nil { + t.Fatal(err) + } + defer legacy.conn.Close() + s := NewPacketStore(legacy, &PacketStoreConfig{}) + s.useResolvedPathIndex = true + s.initResolvedPathIndex() + if err := s.Load(); err != nil { + t.Fatal(err) + } + assertAdvertEvidenceViews(t, s, "other") + var count int + if err := w.QueryRow(testNativeSQL(`SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='advert_route_evidence'`, `SELECT COUNT(*) FROM information_schema.tables WHERE table_schema=current_schema() AND table_name='advert_route_evidence'`)).Scan(&count); err != nil { + t.Fatal(err) + } + if count != 0 { + t.Fatal("read-only server created legacy evidence table") + } +} + +func TestAdvertRouteEvidenceTableAppearsAfterStartup(t *testing.T) { + db, w := advertEvidenceFixture(t, "1100aa") + var schema = testNativeSQL(`CREATE TABLE advert_route_evidence (id INTEGER PRIMARY KEY AUTOINCREMENT, tx_id INTEGER NOT NULL REFERENCES transmissions(id) ON DELETE CASCADE, bit INTEGER NOT NULL CHECK (bit IN (1,2)), UNIQUE(tx_id,bit))`, `CREATE TABLE advert_route_evidence (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, tx_id INTEGER NOT NULL REFERENCES transmissions(id) ON DELETE CASCADE, bit INTEGER NOT NULL CHECK (bit IN (1,2)), UNIQUE(tx_id,bit))`) + if _, err := w.Exec(`DROP TABLE advert_route_evidence`); err != nil { + t.Fatal(err) + } + s := NewPacketStore(db, &PacketStoreConfig{}) + if err := s.Load(); err != nil { + t.Fatal(err) + } + if s.byTxID[1].AdvertRouteEvidence != 0 { + t.Fatal("legacy load invented evidence") + } + s.GetRelayAirtimeShareWithWindow(TimeWindow{}) + if _, err := w.Exec(schema + `; INSERT INTO advert_route_evidence(tx_id,bit) VALUES(1,1),(1,2)`); err != nil { + t.Fatal(err) + } + if err := s.pollAdvertEvidence(500); err != nil { + t.Fatal(err) + } + rows := s.GetRelayAirtimeShareWithWindow(TimeWindow{})["rows"].([]map[string]interface{}) + if len(rows) != 1 || rows[0]["advert_kind"] != "mixed" { + t.Fatalf("late migration did not refresh legacy store: %v", rows) + } +} + +func TestAdvertRouteEvidencePollRejectsPartialBatch(t *testing.T) { + db, w := advertEvidenceFixture(t, "1100aa", 1) + w.SetMaxOpenConns(1) + s := NewPacketStore(db, &PacketStoreConfig{}) + if err := s.Load(); err != nil { + t.Fatal(err) + } + before := s.advertEvidenceCursor + if _, err := w.Exec(testNativeSQL(`PRAGMA ignore_check_constraints=ON; INSERT INTO advert_route_evidence(tx_id,bit) VALUES(1,2),(1,3)`, `ALTER TABLE advert_route_evidence DROP CONSTRAINT advert_route_evidence_bit_check; INSERT INTO advert_route_evidence(tx_id,bit) VALUES(1,2),(1,3)`)); err != nil { + t.Fatal(err) + } + if err := s.pollAdvertEvidence(500); err == nil { + t.Fatal("corrupt feed row accepted") + } + if s.advertEvidenceCursor != before || s.byTxID[1].AdvertRouteEvidence != 1 { + t.Fatal("partial failed batch changed evidence or cursor") + } + if _, err := w.Exec(`DELETE FROM advert_route_evidence WHERE bit=3; ALTER TABLE advert_route_evidence ADD CONSTRAINT advert_route_evidence_bit_check CHECK (bit IN (1,2))`); err != nil { + t.Fatal(err) + } + if err := s.pollAdvertEvidence(500); err != nil { + t.Fatal(err) + } + if s.byTxID[1].AdvertRouteEvidence != 3 { + t.Fatal("retry lost valid event preceding corrupt row") + } +} + +func TestAdvertRouteEvidenceInvalidatesOncePerBatch(t *testing.T) { + s := newRelayAirtimeShareTestStore([]*StoreTx{ + makeRelayAirtimeTx(1, PayloadADVERT, 3, 0, "batch-1"), + makeRelayAirtimeTx(2, PayloadADVERT, 3, 0, "batch-2"), + }) + other := &cachedResult{} + s.rfCache["unrelated-rf-result"] = other + s.rfCache["relay-airtime-share|"] = &cachedResult{} + s.rfCache["relay-airtime-share|7d"] = &cachedResult{} + masks := map[int]uint8{1: 3, 2: 3} + s.mu.Lock() + s.mergeAdvertEvidence(masks) + s.mergeAdvertEvidence(masks) + s.mu.Unlock() + if s.advertEvidenceRevision != 1 { + t.Fatalf("batch invalidated %d times, want one; unchanged replay must not invalidate", s.advertEvidenceRevision) + } + if len(s.rfCache) != 1 || s.rfCache["unrelated-rf-result"] != other { + t.Fatal("batch must invalidate only relay airtime entries") + } +} + +func TestRelayAirtimeShareAdvertDuplicateHashEvidenceUnion(t *testing.T) { + for _, reverse := range []bool{false, true} { + flood, direct := RouteFlood, RouteDirect + first := advertAirtimeTx(1, &flood, "1100aa") + second := advertAirtimeTx(2, &direct, "1200aa") + second.Hash = first.Hash + packets := []*StoreTx{first, second} + if reverse { + packets[0], packets[1] = packets[1], packets[0] + } + s := newRelayAirtimeShareTestStore(packets) + // Keep relay evidence identical so only classification changes. + s.addToResolvedPubkeyIndex(1, []string{"fixture-relay"}) + s.addToResolvedPubkeyIndex(2, []string{"fixture-relay"}) + result := s.computeRelayAirtimeShare(TimeWindow{}) + rows := result["rows"].([]map[string]interface{}) + if len(rows) != 1 || rows[0]["advert_kind"] != "mixed" || rows[0]["count"] != 1 || result["total_count"] != 1 { + t.Errorf("reverse=%v: duplicate hash must have one mixed bucket: %v", reverse, result) + } + if result["total_score"] != int64(lora.TimeOnAir(3, defaultLoRaPreset())) { + t.Errorf("reverse=%v: duplicate hash inflated score: %v", reverse, result) + } + } +} + +func TestRelayAirtimeShareAdvertEvidenceIndependentOfReportWindow(t *testing.T) { + flood, direct := RouteFlood, RouteDirect + inside := advertAirtimeTx(1, &direct, "1200aa") + outside := advertAirtimeTx(2, &flood, "1100aa") + outside.Hash = inside.Hash + outside.FirstSeen = "2025-12-01T00:00:00Z" + s := newRelayAirtimeShareTestStore([]*StoreTx{inside, outside}) + s.addToResolvedPubkeyIndex(inside.ID, []string{"fixture-relay"}) + s.addToResolvedPubkeyIndex(outside.ID, []string{"fixture-relay", "excluded-relay"}) + result := s.computeRelayAirtimeShare(TimeWindow{Since: "2026-01-01T00:00:00Z", Until: "2026-01-02T00:00:00Z"}) + rows := result["rows"].([]map[string]interface{}) + if len(rows) != 1 || rows[0]["advert_kind"] != "mixed" || result["total_count"] != 1 { + t.Fatalf("report window changed known hash evidence: %v", result) + } + if result["total_score"] != int64(lora.TimeOnAir(3, defaultLoRaPreset())) { + t.Fatalf("excluded record contributed airtime score: %v", result) + } +} diff --git a/cmd/server/advert_stats.go b/cmd/server/advert_stats.go index a67d790f6..10a14e0ba 100644 --- a/cmd/server/advert_stats.go +++ b/cmd/server/advert_stats.go @@ -1,30 +1,30 @@ package main -import "time" +import ( + "errors" + "time" -// advertRouteTypeFlood is ROUTE_TYPE_FLOOD from the MeshCore packet header. -// Named distinctly from the equivalent constant in the (still open) unscoped- -// relay PR so the two changes merge independently. -const advertRouteTypeFlood = 1 + "github.com/meshcore-analyzer/packetpath" +) // floodAdvertEntry is one advert transmission originated by a node, reduced to -// what the windowed flood-advert count needs: first-seen timestamp, route type +// what the windowed flood-advert count needs: first-seen timestamp, known route evidence // and packet hash (for dedup across re-ingests / multi-observer rows). type floodAdvertEntry struct { ts string - rt int + mask uint8 hash string } -// countFloodAdverts counts distinct flood adverts (route_type == -// advertRouteTypeFlood) whose first-seen lies within the past windowHours. Entries +// countFloodAdverts counts distinct adverts with known flood evidence whose +// first-seen lies within the past windowHours. Entries // with unparseable timestamps are skipped, matching relay-liveness behaviour; // entries without a hash fall back to their timestamp as the dedup key. func countFloodAdverts(entries []floodAdvertEntry, now time.Time, windowHours float64) int { cutoff := now.Add(-time.Duration(windowHours * float64(time.Hour))) seen := map[string]struct{}{} for _, e := range entries { - if e.rt != advertRouteTypeFlood { + if e.mask&packetpath.AdvertFlood == 0 { continue } t, ok := parseRelayTS(e.ts) @@ -40,16 +40,13 @@ func countFloodAdverts(entries []floodAdvertEntry, now time.Time, windowHours fl return len(seen) } -// CountFloodAdvertsForNode returns how many distinct FLOOD adverts pubkey -// originated in the last windowHours - the mesh-wide-airtime kind. Zero-hop -// adverts (route_type DIRECT) are excluded, so a nearby observer hearing a -// node's cheap local adverts does not inflate the number. +// CountFloodAdvertsForNode counts distinct adverts with recorded flood evidence, +// including mixed and transport-flood frames, independently of the first route. +// It is a lower bound while backfill is pending or evidence writes have failed. +// With no evidence table the count is unavailable, not a proven zero. // -// route_type is filtered in SQL so an advert-spamming node cannot truncate -// the flood count (review feedback on the earlier LIMIT approach). The time -// floor is a DATE-ONLY string with one day of slack: a date prefix compares -// lexically the same across every first_seen format parseRelayTS accepts -// ('T' and ' ' separators alike); the exact window check stays in Go. +// SQL filters known flood evidence before the row cap. The date-only floor has +// one day of slack for mixed timestamp formats; the exact window stays in Go. // // The row cap is a pure safety valve on per-request allocation: it applies to // flood adverts inside the floor window only, and 50000 in ~8 days is ~4 per @@ -61,10 +58,19 @@ func countFloodAdverts(entries []floodAdvertEntry, now time.Time, windowHours fl const floodAdvertRowCap = 50000 func (db *DB) CountFloodAdvertsForNode(pubkey string, windowHours float64, rowCap int) (int, error) { + if !db.advertEvidencePresent() { + return 0, errors.New("advert route evidence unavailable") + } floor := time.Now().UTC().Add(-time.Duration(windowHours*float64(time.Hour))).AddDate(0, 0, -1).Format("2006-01-02") rows, err := db.conn.Query( - "SELECT COALESCE(first_seen, ''), COALESCE(route_type, -1), COALESCE(hash, '') FROM transmissions WHERE from_pubkey = ? AND payload_type = ? AND route_type = ? AND first_seen >= ? ORDER BY id DESC LIMIT ?", - pubkey, payloadTypeAdvert, advertRouteTypeFlood, floor, rowCap) + db.nativeSQL(`SELECT COALESCE(first_seen, ''), CAST(?1 AS INTEGER), COALESCE(hash, '') FROM transmissions t + WHERE from_pubkey=?2 AND payload_type=?3 AND first_seen>=?4 + AND EXISTS(SELECT 1 FROM advert_route_evidence e WHERE e.tx_id=t.id AND e.bit=?5) + ORDER BY id DESC LIMIT ?6`, `SELECT COALESCE(first_seen, ''), `+db.parameter(1)+`::integer, COALESCE(hash, '') FROM transmissions t + WHERE from_pubkey=`+db.parameter(2)+` AND payload_type=`+db.parameter(3)+` AND first_seen>=`+db.parameter(4)+` + AND EXISTS(SELECT 1 FROM advert_route_evidence e WHERE e.tx_id=t.id AND e.bit=`+db.parameter(5)+`) + ORDER BY id DESC LIMIT `+db.parameter(6)), + packetpath.AdvertFlood, pubkey, payloadTypeAdvert, floor, packetpath.AdvertFlood, rowCap) if err != nil { return 0, err } @@ -72,10 +78,10 @@ func (db *DB) CountFloodAdvertsForNode(pubkey string, windowHours float64, rowCa var entries []floodAdvertEntry for rows.Next() { var e floodAdvertEntry - if err := rows.Scan(&e.ts, &e.rt, &e.hash); err != nil { + if err := rows.Scan(&e.ts, &e.mask, &e.hash); err != nil { return 0, err } entries = append(entries, e) } - return countFloodAdverts(entries, time.Now(), windowHours), nil + return countFloodAdverts(entries, time.Now(), windowHours), rows.Err() } diff --git a/cmd/server/advert_stats_test.go b/cmd/server/advert_stats_test.go index 099fac2cd..9ef3038fc 100644 --- a/cmd/server/advert_stats_test.go +++ b/cmd/server/advert_stats_test.go @@ -2,9 +2,12 @@ package main import ( "encoding/json" + "fmt" "net/http/httptest" "testing" "time" + + "github.com/meshcore-analyzer/packetpath" ) // insertAdvertTx seeds one advert transmission row - the single place that @@ -12,10 +15,19 @@ import ( func insertAdvertTx(t *testing.T, db *DB, pubkey, hash string, rt int, ts time.Time) { t.Helper() if _, err := db.conn.Exec( - `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, from_pubkey) VALUES ('00', ?, ?, ?, ?, ?)`, + `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, from_pubkey) VALUES ('00', $1, $2, $3, $4, $5)`, hash, ts.Format("2006-01-02T15:04:05.000Z"), rt, payloadTypeAdvert, pubkey); err != nil { t.Fatalf("insert transmission: %v", err) } + // Fixtures represent ingestor output: canonical routing alone is not evidence. + if _, err := db.conn.Exec(testNativeSQL(`CREATE TABLE IF NOT EXISTS advert_route_evidence(id INTEGER PRIMARY KEY AUTOINCREMENT,tx_id INTEGER,bit INTEGER,UNIQUE(tx_id,bit))`, `CREATE TABLE IF NOT EXISTS advert_route_evidence(id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,tx_id INTEGER,bit INTEGER,UNIQUE(tx_id,bit))`)); err != nil { + t.Fatal(err) + } + if rt == RouteFlood { + if _, err := db.conn.Exec(`INSERT INTO advert_route_evidence(tx_id,bit) SELECT id,1 FROM transmissions WHERE hash=$1`, hash); err != nil { + t.Fatal(err) + } + } } func advertTS(hoursAgo float64) string { @@ -27,21 +39,21 @@ func advertTS(hoursAgo float64) string { func TestCountFloodAdverts(t *testing.T) { now := time.Now() entries := []floodAdvertEntry{ - {ts: advertTS(1), rt: advertRouteTypeFlood, hash: "a1"}, - {ts: advertTS(2), rt: advertRouteTypeFlood, hash: "a1"}, // dup hash: one advert, two rows - {ts: advertTS(3), rt: advertRouteTypeFlood, hash: "a2"}, - {ts: advertTS(4), rt: 0, hash: "a3"}, // zero-hop (DIRECT): excluded - {ts: advertTS(9 * 24), rt: advertRouteTypeFlood, hash: "a4"}, // outside 7d window - {ts: "not-a-time", rt: advertRouteTypeFlood, hash: "a5"}, // unparseable: skipped - {ts: advertTS(5), rt: -1, hash: "a6"}, // route type absent: excluded + {ts: advertTS(1), mask: packetpath.AdvertFlood, hash: "a1"}, + {ts: advertTS(2), mask: packetpath.AdvertFlood, hash: "a1"}, // dup hash: one advert, two rows + {ts: advertTS(3), mask: packetpath.AdvertFlood, hash: "a2"}, + {ts: advertTS(4), mask: packetpath.AdvertDirectEmptyPath, hash: "a3"}, // direct-only evidence: excluded + {ts: advertTS(9 * 24), mask: packetpath.AdvertFlood, hash: "a4"}, // outside 7d window + {ts: "not-a-time", mask: packetpath.AdvertFlood, hash: "a5"}, // unparseable: skipped + {ts: advertTS(5), mask: 0, hash: "a6"}, // no known evidence: excluded } if got := countFloodAdverts(entries, now, 7*24); got != 2 { t.Fatalf("want 2 flood adverts in window, got %d", got) } // Hash-less entries dedup by timestamp instead of collapsing into one. hashless := []floodAdvertEntry{ - {ts: advertTS(1), rt: advertRouteTypeFlood}, - {ts: advertTS(2), rt: advertRouteTypeFlood}, + {ts: advertTS(1), mask: packetpath.AdvertFlood}, + {ts: advertTS(2), mask: packetpath.AdvertFlood}, } if got := countFloodAdverts(hashless, now, 7*24); got != 2 { t.Fatalf("want 2 hash-less flood adverts, got %d", got) @@ -52,6 +64,9 @@ func TestCountFloodAdverts(t *testing.T) { // counting recent flood adverts only (zero-hop and out-of-window excluded). func TestNodeDetailIncludesFloodAdvertCount(t *testing.T) { srv, router := setupTestServer(t) + if _, err := srv.db.conn.Exec(testNativeSQL(`CREATE TABLE advert_route_evidence(id INTEGER PRIMARY KEY AUTOINCREMENT,tx_id INTEGER,bit INTEGER,UNIQUE(tx_id,bit))`, `CREATE TABLE advert_route_evidence(id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,tx_id INTEGER,bit INTEGER,UNIQUE(tx_id,bit))`)); err != nil { + t.Fatal(err) + } now := time.Now().UTC() ins := func(hash string, rt int, ts time.Time) { insertAdvertTx(t, srv.db, "aabbccdd11223344", hash, rt, ts) @@ -82,13 +97,13 @@ func TestNodeDetailIncludesFloodAdvertCount(t *testing.T) { } before := fetch() - ins("fa1", advertRouteTypeFlood, now.Add(-2*time.Hour)) - ins("fa2", advertRouteTypeFlood, now.Add(-30*time.Hour)) - ins("za1", 0, now.Add(-1*time.Hour)) // zero-hop: excluded - ins("fa3", advertRouteTypeFlood, now.Add(-9*24*time.Hour)) // outside the 7d window + ins("fa1", RouteFlood, now.Add(-2*time.Hour)) + ins("fa2", RouteFlood, now.Add(-30*time.Hour)) + ins("za1", 0, now.Add(-1*time.Hour)) // zero-hop: excluded + ins("fa3", RouteFlood, now.Add(-9*24*time.Hour)) // outside the 7d window // Inside the SQL date floor (window + 1d slack) but outside the exact 7d // window - only the Go-side check rejects this one. - ins("fa4", advertRouteTypeFlood, now.Add(-time.Duration(7.5*24)*time.Hour)) + ins("fa4", RouteFlood, now.Add(-time.Duration(7.5*24)*time.Hour)) if got := fetch(); got != before+2 { t.Fatalf("want flood_advert_count_7d = %v+2, got %v", before, got) @@ -101,7 +116,7 @@ func TestCountFloodAdvertsForNode_RowCapSaturates(t *testing.T) { db := setupTestDB(t) now := time.Now().UTC() for i, h := range []string{"cap1", "cap2", "cap3"} { - insertAdvertTx(t, db, "capnode11223344", h, advertRouteTypeFlood, now.Add(-time.Duration(i+1)*time.Hour)) + insertAdvertTx(t, db, "capnode11223344", h, RouteFlood, now.Add(-time.Duration(i+1)*time.Hour)) } n, err := db.CountFloodAdvertsForNode("capnode11223344", 7*24, 2) if err != nil { @@ -111,3 +126,60 @@ func TestCountFloodAdvertsForNode_RowCapSaturates(t *testing.T) { t.Fatalf("want saturated count 2, got %d", n) } } + +func TestNodeDetailFloodCountAgreesWithRouteEvidence(t *testing.T) { + for _, firstRoute := range []int{RouteDirect, RouteFlood, RouteTransportFlood} { + t.Run(fmt.Sprint(firstRoute), func(t *testing.T) { + srv, router := setupTestServer(t) + if _, err := srv.db.conn.Exec(testNativeSQL(`CREATE TABLE IF NOT EXISTS advert_route_evidence(id INTEGER PRIMARY KEY AUTOINCREMENT, tx_id INTEGER, bit INTEGER, UNIQUE(tx_id,bit))`, `CREATE TABLE IF NOT EXISTS advert_route_evidence(id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, tx_id INTEGER, bit INTEGER, UNIQUE(tx_id,bit))`)); err != nil { + t.Fatal(err) + } + const pubkey = "aabbccdd11223344" + if _, err := srv.db.conn.Exec(`DELETE FROM observations WHERE transmission_id IN (SELECT id FROM transmissions WHERE from_pubkey=$1)`, pubkey); err != nil { + t.Fatal(err) + } + if _, err := srv.db.conn.Exec(`DELETE FROM transmissions WHERE from_pubkey=$1`, pubkey); err != nil { + t.Fatal(err) + } + insertAdvertTx(t, srv.db, pubkey, "mixed-advert", firstRoute, time.Now().Add(-time.Hour)) + if _, err := srv.db.conn.Exec(`INSERT INTO advert_route_evidence(tx_id,bit) SELECT id,1 FROM transmissions WHERE hash='mixed-advert' AND NOT EXISTS(SELECT 1 FROM advert_route_evidence WHERE tx_id=transmissions.id AND bit=1) UNION ALL SELECT id,2 FROM transmissions WHERE hash='mixed-advert'`); err != nil { + t.Fatal(err) + } + w := httptest.NewRecorder() + router.ServeHTTP(w, httptest.NewRequest("GET", "/api/nodes/"+pubkey, nil)) + var response NodeDetailResponse + if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil { + t.Fatal(err) + } + if w.Code != 200 || len(response.RecentAdverts) != 1 || response.RecentAdverts[0]["advert_kind"] != "mixed" { + t.Fatalf("unexpected node response: %d %+v", w.Code, response) + } + if response.Node["flood_advert_count_7d"] != float64(1) { + t.Errorf("mixed advert must contribute one flood count regardless of first route %d: %v", firstRoute, response.Node["flood_advert_count_7d"]) + } + }) + } +} + +func TestNodeDetailLegacyFloodCountIsUnavailable(t *testing.T) { + srv, router := setupTestServer(t) + if _, err := srv.db.conn.Exec(`DROP TABLE IF EXISTS advert_route_evidence`); err != nil { + t.Fatal(err) + } + insertAdvertTx(t, srv.db, "aabbccdd11223344", "legacy-flood", RouteFlood, time.Now().Add(-time.Hour)) + if _, err := srv.db.conn.Exec(`DROP TABLE advert_route_evidence`); err != nil { + t.Fatal(err) + } + w := httptest.NewRecorder() + router.ServeHTTP(w, httptest.NewRequest("GET", "/api/nodes/aabbccdd11223344", nil)) + var response NodeDetailResponse + if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil { + t.Fatal(err) + } + if w.Code != 200 { + t.Fatal(w.Code) + } + if _, ok := response.Node["flood_advert_count_7d"]; ok { + t.Errorf("missing evidence table must not report a proven count: %v", response.Node["flood_advert_count_7d"]) + } +} diff --git a/cmd/server/analytics_recompute_after_load_test.go b/cmd/server/analytics_recompute_after_load_test.go new file mode 100644 index 000000000..34ee530bf --- /dev/null +++ b/cmd/server/analytics_recompute_after_load_test.go @@ -0,0 +1,462 @@ +package main + +// Analytics recomputers after the startup load. +// +// main.go starts the recomputers as soon as the FIRST load chunk is in +// memory. Their initial compute therefore only sees that chunk (the +// oldest ids), and the next compute used to wait a full interval +// (5 min by default). These tests pin the contract: +// +// - the store exposes a signal that fires only after RunStartupLoad +// has finished, background fill included; +// - every recomputer recomputes as soon as that signal fires; +// - a pass that started before the signal never ends the #1659 +// warm-up, and a snapshot served after the force timeout is +// replaced as soon as the load is done. + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/gorilla/mux" +) + +func isClosedForTest(ch <-chan struct{}) bool { + select { + case <-ch: + return true + default: + return false + } +} + +func waitForTest(t *testing.T, what string, cond func() bool) { + t.Helper() + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + if cond() { + return + } + time.Sleep(5 * time.Millisecond) + } + t.Fatalf("timed out waiting for: %s", what) +} + +// The startup-load signal must stay open while the background fill is +// still running (LoadComplete is already true at that point) and close +// once RunStartupLoad returns. +func TestStartupLoadDone_ClosesOnlyAfterBackgroundFill(t *testing.T) { + dbPath := postgresTestDSN(t) + createTestDBSpreadOverDays(t, dbPath, 100, 14, time.Now().UTC().Unix()) + db, err := openFixtureReader(t, dbPath) + if err != nil { + t.Fatalf("OpenDB: %v", err) + } + defer db.conn.Close() + store := NewPacketStore(db, &PacketStoreConfig{RetentionHours: 14 * 24, HotStartupHours: 24}) + + var closedAtBgEntry, loadCompleteAtBgEntry atomic.Bool + store.bgLoaderEntryHook = func() { + closedAtBgEntry.Store(isClosedForTest(store.StartupLoadDone())) + loadCompleteAtBgEntry.Store(store.LoadComplete()) + } + if isClosedForTest(store.StartupLoadDone()) { + t.Fatal("StartupLoadDone closed before RunStartupLoad ran") + } + if err := store.RunStartupLoad(500); err != nil { + t.Fatalf("RunStartupLoad: %v", err) + } + if !loadCompleteAtBgEntry.Load() { + t.Fatal("fixture precondition: LoadComplete should already be true when the background fill starts") + } + if closedAtBgEntry.Load() { + t.Fatal("StartupLoadDone closed before the background fill ran") + } + if !isClosedForTest(store.StartupLoadDone()) { + t.Fatal("StartupLoadDone not closed after RunStartupLoad returned") + } +} + +// A failed load is terminal too: nothing more will be loaded, so the +// signal must close instead of leaving the recomputers waiting forever. +func TestStartupLoadDone_ClosesWhenLoadFails(t *testing.T) { + dbPath := postgresTestDSN(t) + createTestDBSpreadOverDays(t, dbPath, 10, 1, time.Now().UTC().Unix()) + db, err := openFixtureReader(t, dbPath) + if err != nil { + t.Fatalf("OpenDB: %v", err) + } + store := NewPacketStore(db, &PacketStoreConfig{RetentionHours: 24, HotStartupHours: 1}) + db.conn.Close() + if err := store.RunStartupLoad(500); err == nil { + t.Fatal("fixture precondition: RunStartupLoad on a closed conn should fail") + } + if !isClosedForTest(store.StartupLoadDone()) { + t.Fatal("StartupLoadDone not closed after a failed RunStartupLoad") + } +} + +// Derived caches with their own TTL (node hash-size info 15s, clock-skew +// engine 30s) may hold values computed from the partial store. The +// recomputes that follow the signal read them, so they must be dropped. +func TestStartupLoadDone_DropsDerivedCaches(t *testing.T) { + db := setupTestDB(t) + defer db.Close() + store := NewPacketStore(db, nil) + + store.hashSizeInfoMu.Lock() + store.hashSizeInfoCache = map[string]*hashSizeNodeInfo{"partial": {}} + store.hashSizeInfoAt = time.Now() + store.hashSizeInfoMu.Unlock() + store.clockSkew.mu.Lock() + store.clockSkew.lastComputed = time.Now() + store.clockSkew.mu.Unlock() + + store.signalStartupLoadDone() + + store.hashSizeInfoMu.Lock() + hashInfo := store.hashSizeInfoCache + store.hashSizeInfoMu.Unlock() + if hashInfo != nil { + t.Fatal("hash-size info cache survived the startup-load signal") + } + store.clockSkew.mu.RLock() + last := store.clockSkew.lastComputed + store.clockSkew.mu.RUnlock() + if !last.IsZero() { + t.Fatal("clock-skew engine still considers its pre-load result fresh") + } +} + +// The recomputer must not wait a full interval once the store is loaded. +func TestRecomputeWhenLoaded_RunsImmediately(t *testing.T) { + var loadedFlag atomic.Bool + rc := newAnalyticsRecomputer("t", time.Hour, func() interface{} { + if loadedFlag.Load() { + return "full" + } + return "partial" + }) + rc.Start() + defer rc.Stop() + if got := rc.Load(); got != "partial" { + t.Fatalf("initial snapshot = %v, want partial", got) + } + + loaded := make(chan struct{}) + stop := make(chan struct{}) + defer close(stop) + go recomputeWhenLoaded(loaded, stop, []*analyticsRecomputer{rc}) + + loadedFlag.Store(true) + close(loaded) + waitForTest(t, "snapshot recomputed after load", func() bool { return rc.Load() == "full" }) + if runs := rc.ComputeRuns(); runs != 2 { + t.Fatalf("ComputeRuns = %d, want 2 (initial + post-load)", runs) + } +} + +// Recomputers run one after the other, in slice order, so a recomputer +// that reads another one's snapshot (roles reads nodes-clock-skew) sees +// the post-load version. +func TestRecomputeWhenLoaded_RunsInOrder(t *testing.T) { + var loadedFlag atomic.Bool + var skewSnapshot atomic.Value + skew := newAnalyticsRecomputer("skew", time.Hour, func() interface{} { + v := "partial" + if loadedFlag.Load() { + v = "full" + } + time.Sleep(20 * time.Millisecond) + skewSnapshot.Store(v) + return v + }) + roles := newAnalyticsRecomputer("roles", time.Hour, func() interface{} { + return skewSnapshot.Load() + }) + skew.Start() + roles.Start() + defer skew.Stop() + defer roles.Stop() + + loaded := make(chan struct{}) + stop := make(chan struct{}) + defer close(stop) + done := make(chan struct{}) + go func() { + recomputeWhenLoaded(loaded, stop, []*analyticsRecomputer{skew, roles}) + close(done) + }() + loadedFlag.Store(true) + close(loaded) + select { + case <-done: + case <-time.After(3 * time.Second): + t.Fatal("recomputeWhenLoaded did not finish") + } + if got := roles.Load(); got != "full" { + t.Fatalf("roles snapshot = %v, want full (it ran before the skew recompute finished)", got) + } +} + +// The post-load order used by StartAnalyticsRecomputers: gated +// recomputers first (their 503 ends soonest), roles after +// nodes-clock-skew (roles reads that snapshot). +func TestAnalyticsRecomputers_PostLoadOrder(t *testing.T) { + db := setupTestDB(t) + defer db.Close() + store := NewPacketStore(db, nil) + stop := store.StartAnalyticsRecomputers(time.Hour) + defer stop() + + store.analyticsRecomputerMu.RLock() + list := store.analyticsRecomputersLocked() + store.analyticsRecomputerMu.RUnlock() + pos := map[string]int{} + for i, rc := range list { + pos[rc.name] = i + } + if len(list) != 11 || len(pos) != 11 { + t.Fatalf("want 11 distinct recomputers, got %d (%d distinct)", len(list), len(pos)) + } + for _, name := range []string{"rf", "topology", "channels"} { + if pos[name] > 2 { + t.Errorf("%s at position %d, want among the first three", name, pos[name]) + } + } + if pos["nodes-clock-skew"] > pos["roles"] { + t.Errorf("roles (%d) runs before nodes-clock-skew (%d)", pos["roles"], pos["nodes-clock-skew"]) + } +} + +// A pass that STARTED before the load finished must not end the warm-up, +// even when the load finishes while that pass is still running. +func TestWarmup_PassStartedBeforeLoadDoesNotOpenGate(t *testing.T) { + loaded := make(chan struct{}) + releaseSecond := make(chan struct{}) + var calls atomic.Int32 + rc := newAnalyticsRecomputer("t", time.Hour, func() interface{} { + switch calls.Add(1) { + case 1: + close(loaded) // load finishes while the first pass is running + return "partial" + default: + <-releaseSecond + return "full" + } + }) + rc.setWarmupReadyGate_1659(func() bool { return isClosedForTest(loaded) }) + rc.Start() + defer rc.Stop() + + if !rc.IsWarmingUp_1659() { + t.Fatal("gate opened on a pass that started before the store was loaded") + } + + stop := make(chan struct{}) + defer close(stop) + go recomputeWhenLoaded(loaded, stop, []*analyticsRecomputer{rc}) + waitForTest(t, "post-load pass started", func() bool { return calls.Load() == 2 }) + if !rc.IsWarmingUp_1659() { + t.Fatal("gate opened while the only post-load pass is still running") + } + close(releaseSecond) + waitForTest(t, "gate opens after the post-load pass", func() bool { return !rc.IsWarmingUp_1659() }) + if got := rc.Load(); got != "full" { + t.Fatalf("gate open but snapshot = %v, want full", got) + } +} + +// Force-open timeout followed by the load finishing: the first-chunk +// snapshot that was served after the timeout is replaced right away. +func TestWarmup_ForceOpenedSnapshotReplacedOnLoad(t *testing.T) { + prev := warmupForceTimeout + warmupForceTimeout = 30 * time.Millisecond + defer func() { warmupForceTimeout = prev }() + + loaded := make(chan struct{}) + rc := newAnalyticsRecomputer("t", time.Hour, func() interface{} { + if isClosedForTest(loaded) { + return "full" + } + return "partial" + }) + rc.setWarmupReadyGate_1659(func() bool { return isClosedForTest(loaded) }) + rc.Start() + defer rc.Stop() + stop := make(chan struct{}) + defer close(stop) + go recomputeWhenLoaded(loaded, stop, []*analyticsRecomputer{rc}) + + waitForTest(t, "force timeout opens the gate", func() bool { return !rc.IsWarmingUp_1659() }) + if got := rc.Load(); got != "partial" { + t.Fatalf("fixture precondition: forced-open snapshot = %v, want partial", got) + } + if !rc.FirstPassDoneAt_1659().IsZero() { + t.Fatal("a pre-load pass was recorded as the first full pass") + } + + close(loaded) + waitForTest(t, "snapshot replaced after load", func() bool { return rc.Load() == "full" }) + waitForTest(t, "first full pass recorded", func() bool { return !rc.FirstPassDoneAt_1659().IsZero() }) +} + +// The periodic ticker restarts from the post-load compute, so the next +// periodic pass is a full interval after it instead of a redundant pass +// on the original phase a moment later. +func TestRecomputeNow_RestartsTicker(t *testing.T) { + const interval = 400 * time.Millisecond + rc := newAnalyticsRecomputer("t", interval, func() interface{} { return 1 }) + rc.Start() // run 1 at t0; unreset ticker would fire at t0+400ms + defer rc.Stop() + + time.Sleep(250 * time.Millisecond) + rc.RecomputeNow() // run 2 at ~250ms; next tick now ~650ms + time.Sleep(300 * time.Millisecond) // ~550ms: past the old phase, before the new one + if runs := rc.ComputeRuns(); runs != 2 { + t.Fatalf("ComputeRuns = %d at ~550ms, want 2 (ticker not restarted by RecomputeNow)", runs) + } +} + +// RecomputeNow must not hang when the recomputer is stopped. +func TestRecomputeNow_ReturnsWhenStopped(t *testing.T) { + rc := newAnalyticsRecomputer("t", time.Hour, func() interface{} { return 1 }) + rc.Start() + rc.Stop() + done := make(chan struct{}) + go func() { + rc.RecomputeNow() + close(done) + }() + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("RecomputeNow blocked on a stopped recomputer") + } +} + +// After a lazy distance-index build, the distance recomputer snapshot +// must already reflect the new index when the handler stops answering 202. +func TestDistanceIndexBuild_RefreshesRecomputerBeforeReportingBuilt(t *testing.T) { + db := setupRichTestDB(t) + defer db.Close() + store := NewPacketStore(db, nil) + if err := store.Load(); err != nil { + t.Fatalf("Load(): %v", err) + } + stop := store.StartAnalyticsRecomputers(time.Hour) + defer stop() + before := store.recompDistance.ComputeRuns() + + store.TriggerDistanceIndexBuild() + waitForTest(t, "distance index built", store.DistanceIndexBuilt) + if runs := store.recompDistance.ComputeRuns(); runs <= before { + t.Fatalf("distance index reported built while recomputer still serves the pre-build snapshot (runs %d -> %d)", before, runs) + } +} + +// End to end: recomputers started at the first chunk, background fill +// still pending. Gated endpoints answer 503 until the load is done and +// then serve data from the whole store without waiting for the interval. +// Ungated recomputers keep answering (no new 503) and are recomputed +// right after the load. +func TestAnalyticsRecomputers_FullDataRightAfterStartupLoad(t *testing.T) { + const totalRows = 100 + dbPath := postgresTestDSN(t) + createTestDBSpreadOverDays(t, dbPath, totalRows, 14, time.Now().UTC().Unix()) + db, err := openFixtureReader(t, dbPath) + if err != nil { + t.Fatalf("OpenDB: %v", err) + } + defer db.conn.Close() + store := NewPacketStore(db, &PacketStoreConfig{RetentionHours: 14 * 24, HotStartupHours: 24}) + + bgEntered := make(chan struct{}) + releaseBg := make(chan struct{}) + store.bgLoaderEntryHook = func() { + close(bgEntered) + <-releaseBg + } + loadErr := make(chan error, 1) + go func() { loadErr <- store.RunStartupLoad(500) }() + <-bgEntered + + stop := store.StartAnalyticsRecomputers(time.Hour) + defer stop() + + srv := NewServer(db, &Config{Port: 3000}, NewHub()) + srv.store = store + router := mux.NewRouter() + srv.RegisterRoutes(router) + get := func(path string) *httptest.ResponseRecorder { + w := httptest.NewRecorder() + router.ServeHTTP(w, httptest.NewRequest("GET", path, nil)) + return w + } + + for _, p := range []string{"/api/analytics/rf", "/api/analytics/topology", "/api/analytics/channels"} { + if w := get(p); w.Code != http.StatusServiceUnavailable { + t.Fatalf("%s before full load: got %d, want 503", p, w.Code) + } + } + for _, p := range []string{"/api/analytics/hash-sizes", "/api/analytics/hash-collisions", "/api/analytics/roles"} { + if w := get(p); w.Code != http.StatusOK { + t.Fatalf("%s before full load: got %d, want 200 (no new warm-up 503)", p, w.Code) + } + } + + store.analyticsRecomputerMu.RLock() + all := map[string]*analyticsRecomputer{ + "topology": store.recompTopology, "rf": store.recompRF, "distance": store.recompDistance, + "channels": store.recompChannels, "hash-collisions": store.recompHashCollisions, + "hash-sizes": store.recompHashSizes, "roles": store.recompRoles, + "observers-clock-skew": store.recompObserversClockSkew, "nodes-clock-skew": store.recompNodesClockSkew, + } + store.analyticsRecomputerMu.RUnlock() + runsBefore := map[string]int64{} + for name, rc := range all { + runsBefore[name] = rc.ComputeRuns() + } + + close(releaseBg) + if err := <-loadErr; err != nil { + t.Fatalf("RunStartupLoad: %v", err) + } + store.mu.RLock() + inMemory := len(store.packets) + store.mu.RUnlock() + if inMemory != totalRows { + t.Fatalf("fixture precondition: %d packets in memory after load, want %d", inMemory, totalRows) + } + + // Post-load passes run sequentially: RF completing does not imply that + // topology or channels has published its full snapshot yet. + for _, name := range []string{"rf", "topology", "channels"} { + rc := all[name] + waitForTest(t, name+" first full pass finishes", func() bool { return !rc.FirstPassDoneAt_1659().IsZero() }) + } + rfResponse := get("/api/analytics/rf") + if rfResponse.Code != http.StatusOK { + t.Fatalf("rf after full load: got %d, want 200", rfResponse.Code) + } + var rf map[string]interface{} + if err := json.Unmarshal(rfResponse.Body.Bytes(), &rf); err != nil { + t.Fatalf("rf body: %v", err) + } + if got, _ := rf["totalTransmissions"].(float64); int(got) != totalRows { + t.Fatalf("rf totalTransmissions = %v right after full load, want %d (first-chunk snapshot still served)", rf["totalTransmissions"], totalRows) + } + for _, p := range []string{"/api/analytics/topology", "/api/analytics/channels"} { + if w := get(p); w.Code != http.StatusOK { + t.Fatalf("%s after full load: got %d, want 200", p, w.Code) + } + } + for name, rc := range all { + name, rc := name, rc + waitForTest(t, name+" recomputed after full load", func() bool { return rc.ComputeRuns() > runsBefore[name] }) + } +} diff --git a/cmd/server/analytics_recomputer.go b/cmd/server/analytics_recomputer.go index 57ff5f764..db6b5dbfd 100644 --- a/cmd/server/analytics_recomputer.go +++ b/cmd/server/analytics_recomputer.go @@ -10,6 +10,9 @@ package main import ( + "fmt" + "log" + "strings" "sync" "sync/atomic" "time" @@ -34,9 +37,10 @@ type analyticsRecomputer struct { interval time.Duration compute func() interface{} - cache atomic.Value // holds interface{} — the latest snapshot - stop chan struct{} - done chan struct{} + cache atomic.Value // holds interface{} — the latest snapshot + stop chan struct{} + done chan struct{} + recomputeReq chan chan struct{} // RecomputeNow → loop; the loop closes the inner channel when done startOnce sync.Once stopOnce sync.Once @@ -61,11 +65,12 @@ func newAnalyticsRecomputer(name string, interval time.Duration, compute func() interval = 5 * time.Minute } return &analyticsRecomputer{ - name: name, - interval: interval, - compute: compute, - stop: make(chan struct{}), - done: make(chan struct{}), + name: name, + interval: interval, + compute: compute, + stop: make(chan struct{}), + done: make(chan struct{}), + recomputeReq: make(chan chan struct{}), } } @@ -96,6 +101,10 @@ func (r *analyticsRecomputer) loop() { select { case <-t.C: r.runOnce() + case ack := <-r.recomputeReq: + r.runOnce() + t.Reset(r.interval) + close(ack) case <-r.stop: return } @@ -114,6 +123,10 @@ func (r *analyticsRecomputer) runOnce() { // reach markFirstPassDone otherwise). _ = recover() }() + // Sample the #1659 readiness gate BEFORE computing: a pass that + // started on a partially loaded store must not end the warm-up, + // even if the load finishes while it runs. + ready := r.warmupReadyGateOpen_1659() t0 := time.Now() result := r.compute() r.lastComputeNs.Store(int64(time.Since(t0))) @@ -128,9 +141,53 @@ func (r *analyticsRecomputer) runOnce() { // PR #1688 r1: called on EVERY successful pass (even nil // result) so a compute that returns nil but doesn't panic // still lifts the gate — banner-stuck-forever fix (munger #2). - // The markFirstPassDone helper is idempotent and additionally - // consults the chunked-loader readiness gate (munger #5). - r.markFirstPassDone_1659() + if ready { + r.markFirstPassDone_1659() + } +} + +// RecomputeNow has the loop goroutine run a compute that starts after +// this call, and waits for it to finish. The periodic ticker restarts +// from that compute, so the next periodic pass is a full interval +// later. Returns early if the recomputer is stopped; blocks until Start +// if it has not started yet. +func (r *analyticsRecomputer) RecomputeNow() { + ack := make(chan struct{}) + select { + case r.recomputeReq <- ack: + case <-r.stop: + return + } + select { + case <-ack: + case <-r.stop: + } +} + +// recomputeWhenLoaded waits for loaded to close, then recomputes each +// recomputer once, one at a time and in slice order. Sequential so the +// post-load passes do not all hold the store read lock at once, and so +// a recomputer that reads another one's snapshot can be placed after +// it. Returns when done or when stop closes. +func recomputeWhenLoaded(loaded, stop <-chan struct{}, rcs []*analyticsRecomputer) { + select { + case <-loaded: + case <-stop: + return + } + t0 := time.Now() + parts := make([]string, 0, len(rcs)) + for _, rc := range rcs { + select { + case <-stop: + return + default: + } + rc.RecomputeNow() + parts = append(parts, fmt.Sprintf("%s=%s", rc.name, rc.LastComputeDuration().Round(time.Millisecond))) + } + log.Printf("[analytics-recompute] startup load done: recomputed %d snapshots in %s (%s)", + len(rcs), time.Since(t0).Round(time.Millisecond), strings.Join(parts, " ")) } // Load returns the most recently computed snapshot, or nil if Start @@ -190,6 +247,22 @@ func pickInterval(override, def time.Duration) time.Duration { return def } +// analyticsRecomputersLocked lists the analytics recomputers in the +// order they start and recompute after the startup load: the three +// warm-up-gated ones first, and roles after nodes-clock-skew because +// computeAnalyticsRoles reads that recomputer's snapshot +// (GetFleetClockSkew). Caller holds analyticsRecomputerMu. +func (s *PacketStore) analyticsRecomputersLocked() []*analyticsRecomputer { + return []*analyticsRecomputer{ + s.recompRF, s.recompTopology, s.recompChannels, + s.recompDistance, s.recompHashCollisions, s.recompHashSizes, + s.recompObserversClockSkew, s.recompNodesClockSkew, + s.recompRoles, + s.recompRetransmissions, + s.recompDirectHeard, + } +} + // StartAnalyticsRecomputers wires each analytics endpoint to a // background recompute goroutine. Each runs an initial compute // synchronously (so the first read after startup is a cache hit, never @@ -260,34 +333,69 @@ func (s *PacketStore) StartAnalyticsRecomputers(defaultInterval time.Duration, o "nodes-clock-skew", pickInterval(ov.NodesClockSkew, defaultInterval), func() interface{} { return s.computeFleetClockSkew() }, ) - all := []*analyticsRecomputer{ - s.recompTopology, s.recompRF, s.recompDistance, - s.recompChannels, s.recompHashCollisions, s.recompHashSizes, - s.recompRoles, - s.recompObserversClockSkew, s.recompNodesClockSkew, - } + s.recompRetransmissions = newAnalyticsRecomputer( + "retransmissions", defaultInterval, + func() interface{} { + return s.computeRetransmissionPressure("", TimeWindow{}, retransmissionDefaultBucket) + }, + ) + // Feeds the node-health "Heard By" card. Not an analytics endpoint, + // but it has the same shape: one full pass over the store that no + // request can afford, served from an atomic snapshot. See + // direct_heard.go. + s.recompDirectHeard = newAnalyticsRecomputer( + "direct-heard", defaultInterval, + func() interface{} { + idx := s.computeDirectHeard() + s.publishDirectHeard(idx) + return idx + }, + ) + all := s.analyticsRecomputersLocked() s.analyticsRecomputerMu.Unlock() - // Issue #1659 (PR #1688 r1, munger #5): wire the chunked-loader - // readiness gate on the three warmup-gated recomputers (RF, - // Topology, Channels). markFirstPassDone_1659 will refuse to - // flip first-pass-done until s.LoadComplete() reports true — - // i.e. the cold-load has populated all observations. Otherwise - // the FIRST recomputer pass runs against the post-restart in-RAM - // slice and the gate opens on partial data (the original #1659 - // bug class). - loadCompleteGate := s.LoadComplete - s.recompRF.setWarmupReadyGate_1659(loadCompleteGate) - s.recompTopology.setWarmupReadyGate_1659(loadCompleteGate) - s.recompChannels.setWarmupReadyGate_1659(loadCompleteGate) + // Issue #1659 (PR #1688 r1, munger #5): wire the loader readiness + // gate on the three warmup-gated recomputers (RF, Topology, + // Channels). Only a pass that STARTS after the whole startup load + // (hot window AND background fill) ends the warm-up. LoadComplete() + // is not enough: it flips at the end of the hot window, so the gate + // used to open on a snapshot that missed the background fill. + loaded := s.StartupLoadDone() + loadedGate := func() bool { + select { + case <-loaded: + return true + default: + return false + } + } + s.recompRF.setWarmupReadyGate_1659(loadedGate) + s.recompTopology.setWarmupReadyGate_1659(loadedGate) + s.recompChannels.setWarmupReadyGate_1659(loadedGate) + s.recompRetransmissions.setWarmupReadyGate_1659(loadedGate) for _, rc := range all { rc.Start() } + // main.go starts the recomputers at the first load chunk, so the + // initial computes above only saw part of the data. Recompute as + // soon as the load is done instead of a full interval later. + stopPostLoad := make(chan struct{}) + postLoadDone := make(chan struct{}) + go func() { + defer close(postLoadDone) + recomputeWhenLoaded(loaded, stopPostLoad, all) + }() + + var stopOnce sync.Once return func() { - for _, rc := range all { - rc.Stop() - } + stopOnce.Do(func() { + close(stopPostLoad) + for _, rc := range all { + rc.Stop() + } + <-postLoadDone + }) } } diff --git a/cmd/server/analytics_warmup_1659.go b/cmd/server/analytics_warmup_1659.go index 37585c12c..8812d11fa 100644 --- a/cmd/server/analytics_warmup_1659.go +++ b/cmd/server/analytics_warmup_1659.go @@ -21,12 +21,14 @@ // // 1. (#1688 munger #5) Only mark first-pass-done when BOTH: // a. a recomputer pass has completed, AND -// b. the chunked loader has finished (s.LoadComplete()). +// b. that pass started after the startup load finished, hot +// window and background fill (store.StartupLoadDone()). // The gate's `readyGate` callback is wired by -// StartAnalyticsRecomputers to `store.LoadComplete`. Passes that -// complete while loadComplete is still false leave the gate in -// the warming-up state; the NEXT pass after loadComplete flips -// true is the one that opens the gate. +// StartAnalyticsRecomputers to that signal. Passes that start +// before it leave the gate in the warming-up state; the loop +// recomputes as soon as the signal fires, and that pass opens +// the gate. (The gate used to be store.LoadComplete, which flips +// at the end of the hot window, before the background fill.) // // 2. (#1688 munger #2 + kent-beck #2) The gate MUST lift in bounded // time. If compute() panics on every pass, hangs indefinitely, @@ -40,7 +42,8 @@ // default) elapsed since the recomputer was constructed // forces IsWarmingUp_1659() to false — degraded mode // (serve whatever cache exists, possibly empty) is -// strictly better than a permanent 503. +// strictly better than a permanent 503. The partial snapshot +// served that way is replaced by the post-load recompute. // // Concurrency (#1688 munger #3): // @@ -104,24 +107,23 @@ func (r *analyticsRecomputer) loadWarmupReadyGate_1659() func() bool { return *p } +// warmupReadyGateOpen_1659 reports whether the readyGate (when set) +// is open. runOnce samples it BEFORE computing and only calls +// markFirstPassDone_1659 when it was open: first-pass-done requires a +// pass that started after the store finished loading (munger #5). A +// check after the compute would accept a pass that began on partial +// data and merely finished after the load. +func (r *analyticsRecomputer) warmupReadyGateOpen_1659() bool { + gate := r.loadWarmupReadyGate_1659() + return gate == nil || gate() +} + // markFirstPassDone_1659 is called from analyticsRecomputer.runOnce() -// after every compute attempt (success OR nil result; panics are -// caught upstream and never reach here). +// after a compute attempt (success OR nil result; panics are caught +// upstream and never reach here) that started with the readyGate open. // -// The gate flip is conditional on the readyGate (when set) reporting -// true — this implements the munger #5 fix: first-pass-done must -// require BOTH a recomputer pass complete AND the chunked loader to -// have finished populating the in-RAM observation set. -// -// Idempotent: only the FIRST successful flip wins; subsequent calls -// observe a non-zero firstPassDoneNs and return immediately. +// Idempotent: only the FIRST successful flip wins. func (r *analyticsRecomputer) markFirstPassDone_1659() { - if r.firstPassDoneNs.Load() != 0 { - return - } - if gate := r.loadWarmupReadyGate_1659(); gate != nil && !gate() { - return - } r.firstPassDoneNs.CompareAndSwap(0, time.Now().UnixNano()) } diff --git a/cmd/server/analytics_warmup_1659_test.go b/cmd/server/analytics_warmup_1659_test.go index e78c71d98..3a72130a6 100644 --- a/cmd/server/analytics_warmup_1659_test.go +++ b/cmd/server/analytics_warmup_1659_test.go @@ -82,11 +82,12 @@ func TestAnalyticsRF_AfterFirstPassReturns200(t *testing.T) { db := setupTestDB(t) defer db.Close() store := NewPacketStore(db, nil) - // #1688 r1: the warmup gate now ALSO requires LoadComplete() to be - // true before first-pass-done flips (munger #5). Tests that don't - // exercise the chunked loader must flip it manually to model a - // production server that has finished cold-loading. - store.loadComplete.Store(true) + // #1688 r1: the warmup gate ALSO requires the startup load (hot + // window and background fill) to be done before first-pass-done + // flips (munger #5). Tests that don't exercise the loader must + // signal it manually to model a production server that has + // finished cold-loading. + store.signalStartupLoadDone() stop := store.StartAnalyticsRecomputers(50 * time.Millisecond) defer stop() diff --git a/cmd/server/auth_e2e.go b/cmd/server/auth_e2e.go new file mode 100644 index 000000000..63387d847 --- /dev/null +++ b/cmd/server/auth_e2e.go @@ -0,0 +1,60 @@ +//go:build e2etest + +package main + +// E2E-only hooks, compiled only with -tags e2etest (CI's corescope-server-e2e): +// the "fake" mail provider and GET /__e2e/last-mail, which returns the newest +// fake mail so Playwright can follow its link, and GET /__e2e/unsubscribe-link, +// which returns a user's notification unsubscribe link. Never part of a +// release build. + +import ( + "net/http" + + "github.com/gorilla/mux" + "github.com/meshcore-analyzer/mailer" +) + +type e2eMail struct { + To string `json:"to"` + Subject string `json:"subject"` + Text string `json:"text"` +} + +type e2eLink struct { + Link string `json:"link"` +} + +func init() { + fakeMailerAllowed = true + e2eRoutes = func(s *Server, r *mux.Router) { + r.HandleFunc("/__e2e/last-mail", func(w http.ResponseWriter, _ *http.Request) { + f, ok := s.auth.mail.(*mailer.Fake) + if !ok { + writeError(w, http.StatusNotFound, "not using the fake mailer") + return + } + m, _, ok := f.Last() + if !ok { + writeError(w, http.StatusNotFound, "no mail sent yet") + return + } + writeJSON(w, e2eMail{To: m.To, Subject: m.Subject, Text: m.Text}) + }).Methods("GET") + // The unsubscribe link a notification mail would carry, so the E2E + // can follow it without waiting for a real state change. + r.HandleFunc("/__e2e/unsubscribe-link", func(w http.ResponseWriter, req *http.Request) { + u, err := s.auth.st.GetByEmail(req.URL.Query().Get("email")) + if err != nil { + writeError(w, http.StatusNotFound, "no such user") + return + } + p, err := s.auth.st.NotifyPrefsFor(u.ID) + if err != nil { + writeError(w, http.StatusInternalServerError, "internal error") + return + } + writeJSON(w, e2eLink{Link: s.auth.link("unsubscribe", p.UnsubToken)}) + }).Methods("GET") + } +} diff --git a/cmd/server/auth_e2e_off_test.go b/cmd/server/auth_e2e_off_test.go new file mode 100644 index 000000000..2ea5ce438 --- /dev/null +++ b/cmd/server/auth_e2e_off_test.go @@ -0,0 +1,14 @@ +//go:build !e2etest + +package main + +import "testing" + +// The e2e probe must never exist in a normal build. +func TestE2ELastMailAbsentInNormalBuild(t *testing.T) { + f := newAuthFixture(t) + expectStatus(t, f.do("GET", "/__e2e/last-mail", nil), 404) + if e2eRoutes != nil || fakeMailerAllowed { + t.Fatal("e2e hooks are set in a normal build") + } +} diff --git a/cmd/server/auth_e2e_on_test.go b/cmd/server/auth_e2e_on_test.go new file mode 100644 index 000000000..3b6b0df8d --- /dev/null +++ b/cmd/server/auth_e2e_on_test.go @@ -0,0 +1,32 @@ +//go:build e2etest + +package main + +import "testing" + +func TestE2ELastMailReturnsNewestFakeMail(t *testing.T) { + f := newAuthFixture(t) // registerAuthRoutes must add the hook itself + expectStatus(t, f.do("GET", "/__e2e/last-mail", nil), 404) + f.registerAndActivate(t, "e2e@example.test", "E2E", "correct horse battery") + w := f.do("GET", "/__e2e/last-mail", nil) + expectStatus(t, w, 200) + m := decode[e2eMail](t, w) + if m.To != "e2e@example.test" || m.Text == "" { + t.Fatalf("unexpected mail: %+v", m) + } +} + +func TestE2EUnsubscribeLink(t *testing.T) { + f := newAuthFixture(t) + expectStatus(t, f.do("GET", "/__e2e/unsubscribe-link?email=nobody@example.test", nil), 404) + c := f.registerAndActivate(t, "e2e@example.test", "E2E", "correct horse battery") + w := f.do("GET", "/__e2e/unsubscribe-link?email=e2e@example.test", nil) + expectStatus(t, w, 200) + p, err := f.st.NotifyPrefsFor(c.me.ID) + if err != nil { + t.Fatal(err) + } + if l := decode[e2eLink](t, w); l.Link != testBase+"/#/account/unsubscribe?token="+p.UnsubToken { + t.Fatalf("link = %q", l.Link) + } +} diff --git a/cmd/server/auth_fixture_test.go b/cmd/server/auth_fixture_test.go new file mode 100644 index 000000000..fe87e3641 --- /dev/null +++ b/cmd/server/auth_fixture_test.go @@ -0,0 +1,303 @@ +package main + +import ( + "bytes" + "database/sql" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "net/url" + "regexp" + "strings" + "testing" + "time" + + "github.com/gorilla/mux" + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/mailer" + "github.com/meshcore-analyzer/pgutil/pgtest" + "github.com/meshcore-analyzer/users" + "modernc.org/sqlite" +) + +const ( + testBase = "https://scope.example.org" + testAPIKey = "test-secret-key-strong-enough" + testHook = "webhook-secret-0123456789" +) + +type authFixture struct { + ownerURL string + srv *Server + router *mux.Router + fake *mailer.Fake + st *users.Store +} + +// client is a browser: its session cookie and CSRF token. +type client struct { + cookie *http.Cookie + csrf string + me meResponse +} + +// Account owners must use the same SQLite library as users.Store, not the +// telemetry driver: https://sqlite.org/howtocorrupt.html#multiple_copies_of_sqlite_linked_into_the_same_application +func openAccountFixtureSQL(target string) (*sql.DB, error) { + if strings.HasPrefix(target, "postgres://") || strings.HasPrefix(target, "postgresql://") { + return openFixtureSQL(target) + } + uri, err := dbconfig.SQLiteURI(target, url.Values{"mode": {"rw"}, "_pragma": {"busy_timeout(5000)"}}) + if err != nil { + return nil, err + } + db, err := sql.Open("sqlite", uri) + if err == nil { + db.SetMaxOpenConns(1) + } + return db, err +} + +// Account fixture owners share the live users.Store file. Mixing SQLite +// implementations in one process bypasses their per-library lock bookkeeping. +func TestAccountFixtureUsesAccountSQLiteDriver(t *testing.T) { + if testBackend(t) != dbconfig.SQLite { + t.Skip("SQLite account driver boundary") + } + f := newAuthFixture(t) + db, err := openAccountFixtureSQL(f.ownerURL) + if err != nil { + t.Fatal(err) + } + defer db.Close() + if _, ok := db.Driver().(*sqlite.Driver); !ok { + t.Fatalf("account fixture uses %T; users.Store uses modernc.org/sqlite", db.Driver()) + } + if err := db.Ping(); err != nil { + t.Fatal(err) + } +} + +func newTestAuthService(t *testing.T, adminEmails ...string) (*authService, *mailer.Fake) { + a, fake, _ := newTestAuthServiceWithURL(t, postgresTestDSN(t), adminEmails...) + return a, fake +} + +func newTestBackupAuthService(t *testing.T, adminEmails ...string) (*authService, *mailer.Fake) { + a, fake, _ := newTestAuthServiceWithURL(t, testDatabaseDSN(t), adminEmails...) + return a, fake +} + +func newTestAuthServiceWithURL(t *testing.T, ownerURL string, adminEmails ...string) (*authService, *mailer.Fake, string) { + t.Helper() + runtimeURL := ownerURL + if testBackend(t) == dbconfig.Postgres { + owner, err := openAccountFixtureSQL(ownerURL) + if err != nil { + t.Fatal(err) + } + defer owner.Close() + if err := users.Apply(owner); err != nil { + t.Fatal(err) + } + runtimeURL = pgtest.Writer(t, ownerURL) + u, _ := url.Parse(runtimeURL) + if _, err := owner.Exec(`REVOKE INSERT,UPDATE,DELETE ON corescope_schema,schema_version FROM "` + u.User.Username() + `"`); err != nil { + t.Fatal(err) + } + } + set := &userMgmtSettings{ + databaseURL: runtimeURL, adminEmails: map[string]bool{}, + sessionTTL: 30 * 24 * time.Hour, provider: "fake", + fromEmail: "noreply@example.org", fromName: "CoreScope", webhookSecret: testHook, + } + set.baseURL, _ = url.Parse(testBase) + set.secureCookie = true + for _, e := range adminEmails { + set.adminEmails[e] = true + } + st, err := users.Open(set.databaseURL) + if err != nil { + t.Fatal(err) + } + fake := &mailer.Fake{} + a := newAuthService(set, st, fake) + t.Cleanup(func() { + a.waitAudits() + st.Close() + }) + return a, fake, ownerURL +} + +// newAuthFixture builds a Server with auth on and only the auth routes. +func newAuthFixture(t *testing.T, adminEmails ...string) *authFixture { + return newAuthFixtureWithURL(t, postgresTestDSN(t), adminEmails...) +} + +func newBackupAuthFixture(t *testing.T, adminEmails ...string) *authFixture { + return newAuthFixtureWithURL(t, testDatabaseDSN(t), adminEmails...) +} + +func newAuthFixtureWithURL(t *testing.T, databaseURL string, adminEmails ...string) *authFixture { + t.Helper() + a, fake, ownerURL := newTestAuthServiceWithURL(t, databaseURL, adminEmails...) + srv := &Server{cfg: &Config{APIKey: testAPIKey}, perfStats: NewPerfStats(), auth: a} + r := mux.NewRouter() + srv.registerAuthRoutes(r) + return &authFixture{srv: srv, router: r, fake: fake, st: a.st, ownerURL: ownerURL} +} + +type reqMod func(*http.Request) + +func as(c *client) reqMod { + return func(r *http.Request) { + if c.cookie != nil { + r.AddCookie(c.cookie) + } + if c.csrf != "" { + r.Header.Set(csrfHeader, c.csrf) + } + } +} + +func header(k, v string) reqMod { return func(r *http.Request) { r.Header.Set(k, v) } } +func fromIP(ip string) reqMod { return func(r *http.Request) { r.RemoteAddr = ip + ":5555" } } + +// do serves one request and then waits for its background audit writes, +// so tests see audit rows in request order. +func (f *authFixture) do(method, path string, body any, mods ...reqMod) *httptest.ResponseRecorder { + w := f.serve(method, path, body, mods...) + f.srv.auth.waitAudits() + return w +} + +// serve serves one request and returns without waiting for audit writes. +func (f *authFixture) serve(method, path string, body any, mods ...reqMod) *httptest.ResponseRecorder { + var rd io.Reader + if body != nil { + b, _ := json.Marshal(body) + rd = bytes.NewReader(b) + } + req := httptest.NewRequest(method, path, rd) + req.RemoteAddr = "203.0.113.10:5555" + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + if !isSafeMethod(method) { + req.Header.Set("Origin", testBase) + } + for _, m := range mods { + m(req) + } + w := httptest.NewRecorder() + f.router.ServeHTTP(w, req) + return w +} + +func decode[T any](t *testing.T, w *httptest.ResponseRecorder) T { + t.Helper() + var v T + if err := json.Unmarshal(w.Body.Bytes(), &v); err != nil { + t.Fatalf("decode %T from %q: %v", v, w.Body.String(), err) + } + return v +} + +func expectStatus(t *testing.T, w *httptest.ResponseRecorder, code int) { + t.Helper() + if w.Code != code { + t.Fatalf("status = %d, want %d; body: %s", w.Code, code, w.Body.String()) + } +} + +var tokenRE = regexp.MustCompile(`token=([A-Za-z0-9_%\-]+)`) + +// lastToken extracts the token from the newest fake mail's link. +func (f *authFixture) lastToken(t *testing.T) string { + t.Helper() + m, _, ok := f.fake.Last() + if !ok { + t.Fatal("no mail was sent") + } + sm := tokenRE.FindStringSubmatch(m.Text) + if sm == nil { + t.Fatalf("no token in mail text: %q", m.Text) + } + tok, _ := url.QueryUnescape(sm[1]) + return tok +} + +func sessionFrom(t *testing.T, w *httptest.ResponseRecorder) *http.Cookie { + t.Helper() + for _, c := range w.Result().Cookies() { + if c.Name == sessionCookieName && c.Value != "" { + return c + } + } + t.Fatalf("no %s cookie in response", sessionCookieName) + return nil +} + +// registerAndActivate runs the link flow and returns the logged-in client. +func (f *authFixture) registerAndActivate(t *testing.T, email, name, password string) *client { + t.Helper() + w := f.do("POST", "/api/auth/register", registerRequest{Email: email, DisplayName: name, Password: password}) + expectStatus(t, w, 200) + w = f.do("POST", "/api/auth/activate", activateRequest{Token: f.lastToken(t), Password: password}) + expectStatus(t, w, 200) + me := decode[meResponse](t, w) + return &client{cookie: sessionFrom(t, w), csrf: me.CSRFToken, me: me} +} + +func (f *authFixture) login(t *testing.T, email, password string) *client { + t.Helper() + w := f.do("POST", "/api/auth/login", loginRequest{Email: email, Password: password}) + expectStatus(t, w, 200) + me := decode[meResponse](t, w) + return &client{cookie: sessionFrom(t, w), csrf: me.CSRFToken, me: me} +} + +// breakTable renames a users.db table behind the store's back, so the next +// store call that touches it fails with a DB error (not ErrNotFound). +func (f *authFixture) breakTable(t *testing.T, table string) { + t.Helper() + db, err := openAccountFixtureSQL(f.ownerURL) + if err != nil { + t.Fatal(err) + } + defer db.Close() + if _, err := db.Exec(`ALTER TABLE ` + table + ` RENAME TO ` + table + `_broken`); err != nil { + t.Fatal(err) + } +} + +// unusedTokens counts uid's outstanding links of purpose p, read straight +// from users.db (the raw tokens are not observable when no mail left). +func (f *authFixture) unusedTokens(t *testing.T, uid int64, p users.Purpose) int { + t.Helper() + db, err := openAccountFixtureSQL(f.ownerURL) + if err != nil { + t.Fatal(err) + } + defer db.Close() + var n int + if err := db.QueryRow(`SELECT COUNT(*) FROM tokens WHERE user_id = $1 AND purpose = $2 AND used_at IS NULL`, uid, string(p)).Scan(&n); err != nil { + t.Fatal(err) + } + return n +} + +// execDB runs raw SQL on users.db behind the store's back (triggers that +// simulate a concurrent writer or a failing statement). +func (f *authFixture) execDB(t *testing.T, stmt string) { + t.Helper() + db, err := openAccountFixtureSQL(f.ownerURL) + if err != nil { + t.Fatal(err) + } + defer db.Close() + if _, err := db.Exec(stmt); err != nil { + t.Fatal(err) + } +} diff --git a/cmd/server/auth_flow_test.go b/cmd/server/auth_flow_test.go new file mode 100644 index 000000000..ef270c2ef --- /dev/null +++ b/cmd/server/auth_flow_test.go @@ -0,0 +1,244 @@ +package main + +import ( + "errors" + "fmt" + "net/http" + "strings" + "testing" + + "github.com/meshcore-analyzer/users" +) + +const pw = "correct horse battery" + +func TestAuthRegisterActivateLoginLogout(t *testing.T) { + f := newAuthFixture(t) + w := f.do("POST", "/api/auth/register", registerRequest{Email: "Alice@Example.org", DisplayName: "Alice", Password: pw}) + expectStatus(t, w, 200) + if m, _, _ := f.fake.Last(); m.To != "alice@example.org" || !strings.Contains(m.Subject, "Activate") { + t.Fatalf("activation mail = %+v", m) + } + // Pending accounts cannot log in. + expectStatus(t, f.do("POST", "/api/auth/login", loginRequest{Email: "alice@example.org", Password: pw}), 401) + + w = f.do("POST", "/api/auth/activate", activateRequest{Token: f.lastToken(t), Password: pw}) + expectStatus(t, w, 200) + me := decode[meResponse](t, w) + ck := sessionFrom(t, w) + if me.Role != users.RoleUser || me.CSRFToken == "" || !ck.HttpOnly || !ck.Secure { + t.Fatalf("activate: me=%+v cookie=%+v", me, ck) + } + c := &client{cookie: ck, csrf: me.CSRFToken} + expectStatus(t, f.do("GET", "/api/auth/me", nil, as(c)), 200) + expectStatus(t, f.do("POST", "/api/auth/logout", nil, as(c)), 200) + expectStatus(t, f.do("GET", "/api/auth/me", nil, as(c)), 401) + f.login(t, "alice@example.org", pw) +} + +func TestAuthRegisterIsEnumerationSafe(t *testing.T) { + f := newAuthFixture(t) + f.registerAndActivate(t, "bob@example.org", "Bob", pw) + first := f.do("POST", "/api/auth/register", registerRequest{Email: "carol@example.org", DisplayName: "Carol", Password: pw}) + again := f.do("POST", "/api/auth/register", registerRequest{Email: "bob@example.org", DisplayName: "Bobby", Password: pw}) + if first.Code != again.Code || first.Body.String() != again.Body.String() { + t.Fatalf("responses differ: %d %q vs %d %q", first.Code, first.Body, again.Code, again.Body) + } + if m, _, _ := f.fake.Last(); m.To != "bob@example.org" || !strings.Contains(m.Subject, "Registration attempt") { + t.Fatalf("existing-account notice = %+v", m) + } + // Re-registering a still-pending address re-sends the activation link. + f.do("POST", "/api/auth/register", registerRequest{Email: "carol@example.org", DisplayName: "Carol", Password: pw}) + if m, _, _ := f.fake.Last(); m.To != "carol@example.org" || !strings.Contains(m.Subject, "Activate") { + t.Fatalf("pending re-register mail = %+v", m) + } +} + +func TestAuthValidationErrors(t *testing.T) { + f := newAuthFixture(t) + w := f.do("POST", "/api/auth/register", registerRequest{Email: "x@example.org", DisplayName: "X", Password: pw}) + expectStatus(t, w, 400) + w = f.do("POST", "/api/auth/register", registerRequest{Email: "x@example.org", DisplayName: "Xx", Password: "short"}) + expectStatus(t, w, 400) + w = f.do("POST", "/api/auth/register", map[string]string{"email": "x@example.org", "bogus": "1"}) + expectStatus(t, w, 400) +} + +func TestAuthActivateAdminFromConfig(t *testing.T) { + f := newAuthFixture(t, "boss@example.org") + c := f.registerAndActivate(t, "boss@example.org", "Boss", pw) + if c.me.Role != users.RoleAdmin { + t.Fatalf("config admin activated as %q", c.me.Role) + } + // Activation links are single-use. + expectStatus(t, f.do("POST", "/api/auth/activate", activateRequest{Token: "bogus", Password: pw}), 410) +} + +func TestAuthLoginGenericFailure(t *testing.T) { + f := newAuthFixture(t) + f.registerAndActivate(t, "dave@example.org", "Dave", pw) + unknown := f.do("POST", "/api/auth/login", loginRequest{Email: "nobody@example.org", Password: pw}) + wrong := f.do("POST", "/api/auth/login", loginRequest{Email: "dave@example.org", Password: "wrong password!"}) + if unknown.Code != 401 || wrong.Code != 401 || unknown.Body.String() != wrong.Body.String() { + t.Fatalf("unknown=%d %q wrong=%d %q", unknown.Code, unknown.Body, wrong.Code, wrong.Body) + } +} + +func TestAuthConfigAdminRoleRestoredOnLogin(t *testing.T) { + f := newAuthFixture(t, "boss@example.org") + c := f.registerAndActivate(t, "boss@example.org", "Boss", pw) + f.st.SetRole(c.me.ID, users.RoleUser) // e.g. demoted by direct DB edit + if again := f.login(t, "boss@example.org", pw); again.me.Role != users.RoleAdmin { + t.Fatalf("config admin logged in as %q", again.me.Role) + } +} + +func TestAuthForgotAndReset(t *testing.T) { + f := newAuthFixture(t) + c := f.registerAndActivate(t, "erin@example.org", "Erin", pw) + sent := len(f.fake.Sent()) + expectStatus(t, f.do("POST", "/api/auth/forgot", emailRequest{Email: "nobody@example.org"}), 200) + if len(f.fake.Sent()) != sent { + t.Fatal("forgot for an unknown address sent mail") + } + expectStatus(t, f.do("POST", "/api/auth/forgot", emailRequest{Email: "erin@example.org"}), 200) + tok := f.lastToken(t) + expectStatus(t, f.do("POST", "/api/auth/reset", resetRequest{Token: tok, Password: "a brand new secret"}), 200) + expectStatus(t, f.do("GET", "/api/auth/me", nil, as(c)), 401) // all sessions ended + f.login(t, "erin@example.org", "a brand new secret") + expectStatus(t, f.do("POST", "/api/auth/reset", resetRequest{Token: tok, Password: "another new secret"}), 410) +} + +func TestAuthForgotIsEnumerationSafe(t *testing.T) { + f := newAuthFixture(t) + f.registerAndActivate(t, "gina@example.org", "Gina", pw) + unknown := f.do("POST", "/api/auth/forgot", emailRequest{Email: "nobody@example.org"}) + known := f.do("POST", "/api/auth/forgot", emailRequest{Email: "gina@example.org"}) + if unknown.Code != known.Code || unknown.Body.String() != known.Body.String() { + t.Fatalf("responses differ: %d %q vs %d %q", unknown.Code, unknown.Body, known.Code, known.Body) + } +} + +func TestAuthWrongCSRFTokenRejected(t *testing.T) { + f := newAuthFixture(t) + c := f.registerAndActivate(t, "hank@example.org", "Hank", pw) + f.router.HandleFunc("/api/test/mutate", f.srv.withUser(func(w http.ResponseWriter, r *http.Request, u *users.User, sess *users.Session) { + writeJSON(w, okResponse{OK: true}) + })).Methods("POST") + expectStatus(t, f.do("POST", "/api/test/mutate", nil, as(c)), 200) + expectStatus(t, f.do("POST", "/api/test/mutate", nil, as(c), header(csrfHeader, "not-the-token")), 403) +} + +func TestAuthLoginRateLimit(t *testing.T) { + f := newAuthFixture(t) + for i := 0; i < 10; i++ { + expectStatus(t, f.do("POST", "/api/auth/login", loginRequest{Email: "x@example.org", Password: "nope nope nope"}), 401) + } + w := f.do("POST", "/api/auth/login", loginRequest{Email: "x@example.org", Password: "nope nope nope"}) + expectStatus(t, w, 429) + if w.Header().Get("Retry-After") == "" { + t.Fatal("429 without Retry-After") + } + // Per-address bucket: another IP is still limited for the same address. + expectStatus(t, f.do("POST", "/api/auth/login", loginRequest{Email: "x@example.org", Password: "nope"}, fromIP("198.51.100.7")), 429) +} + +func TestAuthOriginRequiredForLogin(t *testing.T) { + f := newAuthFixture(t) + w := f.do("POST", "/api/auth/login", loginRequest{Email: "x@example.org", Password: pw}, header("Origin", "https://evil.example")) + expectStatus(t, w, 403) +} + +func TestAuthMailFailureRollsBackRegistration(t *testing.T) { + f := newAuthFixture(t) + f.fake.SetSendErr(errors.New("brevo down")) + expectStatus(t, f.do("POST", "/api/auth/register", registerRequest{Email: "fay@example.org", DisplayName: "Fay", Password: pw}), 503) + if _, err := f.st.GetByEmail("fay@example.org"); !errors.Is(err, users.ErrNotFound) { + t.Fatalf("user kept after failed mail: %v", err) + } + f.fake.SetSendErr(nil) + expectStatus(t, f.do("POST", "/api/auth/register", registerRequest{Email: "fay@example.org", DisplayName: "Fay", Password: pw}), 200) +} + +func TestAuthPendingReRegisterNewestWins(t *testing.T) { + f := newAuthFixture(t) + const p1, p2 = "first password here", "second password here" + expectStatus(t, f.do("POST", "/api/auth/register", registerRequest{Email: "vic@example.org", DisplayName: "Squatter", Password: p1}), 200) + expectStatus(t, f.do("POST", "/api/auth/register", registerRequest{Email: "vic@example.org", DisplayName: "Victim", Password: p2}), 200) + link := f.lastToken(t) + // The squatter cannot use the victim's link: it needs the newest password. + w := f.do("POST", "/api/auth/activate", activateRequest{Token: link, Password: p1}) + expectStatus(t, w, 401) + w = f.do("POST", "/api/auth/activate", activateRequest{Token: link, Password: p2}) + expectStatus(t, w, 200) + if me := decode[meResponse](t, w); me.DisplayName != "Victim" { + t.Fatalf("displayName = %q, want the newest registration's", me.DisplayName) + } + expectStatus(t, f.do("POST", "/api/auth/login", loginRequest{Email: "vic@example.org", Password: p1}), 401) + f.login(t, "vic@example.org", p2) +} + +func TestAuthActivateWrongPasswordKeepsToken(t *testing.T) { + f := newAuthFixture(t) + expectStatus(t, f.do("POST", "/api/auth/register", registerRequest{Email: "kim@example.org", DisplayName: "Kim", Password: pw}), 200) + tok := f.lastToken(t) + w := f.do("POST", "/api/auth/activate", activateRequest{Token: tok, Password: "not the password"}) + expectStatus(t, w, 401) + if !strings.Contains(w.Body.String(), "wrong password for this account") { + t.Fatalf("body = %s", w.Body.String()) + } + if u, _ := f.st.GetByEmail("kim@example.org"); u.Status != users.StatusPending { + t.Fatalf("activated with a wrong password: %+v", u) + } + expectStatus(t, f.do("POST", "/api/auth/activate", activateRequest{Token: tok, Password: pw}), 200) +} + +func TestAuthActivatePasswordRateLimited(t *testing.T) { + f := newAuthFixture(t) + expectStatus(t, f.do("POST", "/api/auth/register", registerRequest{Email: "lou@example.org", DisplayName: "Lou", Password: pw}), 200) + tok := f.lastToken(t) + for i := 0; i < 10; i++ { + ip := fromIP(fmt.Sprintf("198.51.100.%d", i+1)) + expectStatus(t, f.do("POST", "/api/auth/activate", activateRequest{Token: tok, Password: "guess guess guess"}, ip), 401) + } + // Per-account bucket: a fresh IP is still limited for this account. + w := f.do("POST", "/api/auth/activate", activateRequest{Token: tok, Password: pw}, fromIP("198.51.100.99")) + expectStatus(t, w, 429) + if w.Header().Get("Retry-After") == "" { + t.Fatal("429 without Retry-After") + } +} + +func TestAuthActivateDBErrorIs500(t *testing.T) { + f := newAuthFixture(t) + expectStatus(t, f.do("POST", "/api/auth/register", registerRequest{Email: "max@example.org", DisplayName: "Max", Password: pw}), 200) + tok := f.lastToken(t) + f.breakTable(t, "users") + expectStatus(t, f.do("POST", "/api/auth/activate", activateRequest{Token: tok, Password: pw}), 500) +} + +// A re-register that lands between the password check and the activation +// replaces the hash; the activation must not go through on the old one. +func TestAuthActivateHashChangedMidwayIs409(t *testing.T) { + f := newAuthFixture(t) + expectStatus(t, f.do("POST", "/api/auth/register", registerRequest{Email: "ned@example.org", DisplayName: "Ned", Password: pw}), 200) + tok := f.lastToken(t) + f.execDB(t, testNativeSQL(`CREATE TRIGGER race AFTER UPDATE OF used_at ON tokens BEGIN UPDATE users SET password_hash='replaced-by-a-re-register' WHERE id=NEW.user_id; END`, `CREATE FUNCTION race() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN UPDATE users SET password_hash='replaced-by-a-re-register' WHERE id=NEW.user_id; RETURN NEW; END $$; CREATE TRIGGER race AFTER UPDATE OF used_at ON tokens FOR EACH ROW EXECUTE FUNCTION race()`)) + w := f.do("POST", "/api/auth/activate", activateRequest{Token: tok, Password: pw}) + expectStatus(t, w, 409) + if !strings.Contains(w.Body.String(), "account changed, try again") { + t.Fatalf("body = %s", w.Body.String()) + } + for _, c := range w.Result().Cookies() { + if c.Name == sessionCookieName && c.Value != "" { + t.Fatal("session cookie set on a refused activation") + } + } + u, _ := f.st.GetByEmail("ned@example.org") + if u.Status != users.StatusPending { + t.Fatalf("activated on a replaced hash: %+v", u) + } + if n := f.unusedTokens(t, u.ID, users.PurposeActivate); n != 1 { + t.Fatalf("%d usable activation links after a refused activation, want 1", n) + } +} diff --git a/cmd/server/auth_handlers.go b/cmd/server/auth_handlers.go new file mode 100644 index 000000000..f559978f3 --- /dev/null +++ b/cmd/server/auth_handlers.go @@ -0,0 +1,302 @@ +package main + +import ( + "errors" + "log" + "net/http" + "strconv" + "time" + + "github.com/meshcore-analyzer/mailer" + "github.com/meshcore-analyzer/users" +) + +const ( + msgCheckMail = "If the address can receive mail, a message is on its way. Check your inbox." + msgBadLogin = "incorrect email or password" + msgMailFailed = "mail could not be sent, try again later" +) + +func (s *Server) handleRegister(w http.ResponseWriter, r *http.Request) { + a := s.auth + var req registerRequest + if !decodeJSON(w, r, &req) { + return + } + email, err := users.NormalizeEmail(req.Email) + if writeValidation(w, err) { + return + } + name, err := users.ValidateDisplayName(req.DisplayName) + if writeValidation(w, err) { + return + } + if writeValidation(w, users.ValidatePassword(req.Password)) { + return + } + if !a.allow(w, r, a.signup, "email:"+email) { + return + } + // Hash before the lookup so new and existing addresses cost the same. + hash, err := users.HashPassword(req.Password) + if err != nil { + writeError(w, http.StatusInternalServerError, "internal error") + return + } + if existing, err := a.st.GetByEmail(email); err == nil { + // Identical response either way (no account enumeration). + if existing.Status == users.StatusPending { + // Newest registration wins: the account takes the password and + // name of the latest request, and activation demands that + // password, so a squatter cannot plant credentials and the owner + // can recover a squatted address. Failures stay invisible to the + // client (same response as every other branch). + if err := a.st.SetPassword(existing.ID, hash); err != nil { + log.Printf("[users] register: update pending user #%d: %v", existing.ID, err) + } else if err := a.st.SetDisplayName(existing.ID, name); err != nil { + log.Printf("[users] register: update pending user #%d: %v", existing.ID, err) + } else { + existing.DisplayName = name + _ = a.mailToken(r.Context(), existing, users.PurposeActivate, 48*time.Hour, "", "activate", + func(tok string) mailer.Message { return a.activationMail(existing, tok) }) + } + } else { + _ = a.sendMail(r.Context(), existing, "register-notice", a.registerNoticeMail(existing)) + } + writeJSON(w, okResponse{OK: true, Message: msgCheckMail}) + return + } + u, err := a.st.CreatePending(email, name, hash) + if errors.Is(err, users.ErrEmailTaken) { // lost a race with a concurrent register + writeJSON(w, okResponse{OK: true, Message: msgCheckMail}) + return + } + if err != nil { + writeError(w, http.StatusInternalServerError, "internal error") + return + } + err = a.mailToken(r.Context(), u, users.PurposeActivate, 48*time.Hour, "", "activate", + func(tok string) mailer.Message { return a.activationMail(u, tok) }) + if err != nil { + if derr := a.st.Delete(u.ID); derr != nil { + log.Printf("[users] rollback of user #%d failed: %v", u.ID, derr) + } + writeError(w, http.StatusServiceUnavailable, msgMailFailed) + return + } + a.audit(nil, "user.register", idPtr(u.ID), nil) + writeJSON(w, okResponse{OK: true, Message: msgCheckMail}) +} + +// handleActivate needs the token and the account password: a re-register +// of a pending address replaces the password, so whoever clicks must prove +// they know the newest one (a squatter's link cannot log the owner into an +// account carrying the squatter's password). +func (s *Server) handleActivate(w http.ResponseWriter, r *http.Request) { + a := s.auth + var req activateRequest + if !decodeJSON(w, r, &req) { + return + } + uid, err := a.st.TokenUser(req.Token, users.PurposeActivate) + if err != nil { + writeTokenError(w, err) + return + } + if !a.allow(w, r, a.login, "activate:#"+strconv.FormatInt(uid, 10)) { + return + } + u, err := a.st.GetByID(uid) + if err != nil && !errors.Is(err, users.ErrNotFound) { + log.Printf("[users] activate: load user #%d: %v", uid, err) + writeError(w, http.StatusInternalServerError, "internal error") + return + } + if err != nil || u.Status != users.StatusPending { + writeError(w, http.StatusGone, "this account is already activated, log in instead") + return + } + ok, err := users.VerifyPassword(u.PasswordHash, req.Password) + if err != nil { + log.Printf("[users] activate: verify password for user #%d: %v", u.ID, err) + writeError(w, http.StatusInternalServerError, "internal error") + return + } + if !ok { + writeError(w, http.StatusUnauthorized, "wrong password for this account") + return + } + // Activate only on the hash that was just verified: a re-register in + // between replaces it, and the newest password must be proven. + switch err := a.st.ActivateWithToken(req.Token, u.ID, a.roleFor(u.Email), u.PasswordHash); { + case err == nil: + case errors.Is(err, users.ErrAccountChanged): + writeError(w, http.StatusConflict, "account changed, try again") + return + case errors.Is(err, users.ErrTokenInvalid), errors.Is(err, users.ErrTokenExpired): + writeTokenError(w, err) + return + default: + log.Printf("[users] activate user #%d: %v", u.ID, err) + writeError(w, http.StatusInternalServerError, "internal error") + return + } + a.audit(nil, "user.activate", idPtr(u.ID), nil) + if u, err = a.st.GetByID(u.ID); err != nil { + writeError(w, http.StatusInternalServerError, "internal error") + return + } + a.startSession(w, r, u) +} + +// startSession creates a session, sets the cookie and answers with /me. +// It reports whether the session was started. +func (a *authService) startSession(w http.ResponseWriter, r *http.Request, u *users.User) bool { + raw, sess, err := a.st.CreateSession(u.ID, a.set.sessionTTL, r.UserAgent()) + if err != nil { + writeError(w, http.StatusInternalServerError, "internal error") + return false + } + _ = a.st.TouchLogin(u.ID) + a.setSessionCookie(w, raw, sess.ExpiresAt) + writeJSON(w, meFrom(u, sess)) + return true +} + +// loginFailReason is the audit reason of a refused login for an existing +// account. A wrong password wins over the account state. +func loginFailReason(passwordOK bool, st users.Status) string { + if !passwordOK { + return "wrong_password" + } + if st == users.StatusPending { + return "pending" + } + return "disabled" +} + +func (s *Server) handleLogin(w http.ResponseWriter, r *http.Request) { + a := s.auth + var req loginRequest + if !decodeJSON(w, r, &req) { + return + } + email, emailErr := users.NormalizeEmail(req.Email) + key := "email:" + email + if emailErr != nil { + key = "email:invalid" + } + if !a.allow(w, r, a.login, key) { + return + } + var u *users.User + if emailErr == nil { + u, _ = a.st.GetByEmail(email) + } + if u == nil { + users.BurnPasswordCheck(req.Password) + writeError(w, http.StatusUnauthorized, msgBadLogin) + return + } + ok, err := users.VerifyPassword(u.PasswordHash, req.Password) + if err != nil || !ok || u.Status != users.StatusActive { + writeError(w, http.StatusUnauthorized, msgBadLogin) + // In the background after the answer is decided, so the write never + // changes response timing. + a.auditAsync(nil, "user.login.failed", idPtr(u.ID), map[string]string{"reason": loginFailReason(err == nil && ok, u.Status)}) + return + } + // Config wins: an address in adminEmails is always admin. + if a.isConfigAdmin(u.Email) && u.Role != users.RoleAdmin { + if err := a.st.SetRole(u.ID, users.RoleAdmin); err == nil { + u.Role = users.RoleAdmin + a.audit(nil, "user.role.config", idPtr(u.ID), map[string]string{"role": "admin"}) + } + } + if a.startSession(w, r, u) { + a.auditAsync(nil, "user.login", idPtr(u.ID), nil) + } +} + +func (s *Server) handleLogout(w http.ResponseWriter, r *http.Request) { + if c, err := r.Cookie(sessionCookieName); err == nil && c.Value != "" { + _ = s.auth.st.DeleteSessionByToken(c.Value) + } + s.auth.clearSessionCookie(w) + writeJSON(w, okResponse{OK: true}) +} + +func (s *Server) handleMe(w http.ResponseWriter, r *http.Request) { + u, sess := s.auth.currentUser(w, r) + if u == nil { + writeError(w, http.StatusUnauthorized, "not logged in") + return + } + writeJSON(w, meFrom(u, sess)) +} + +func (s *Server) handleForgot(w http.ResponseWriter, r *http.Request) { + a := s.auth + var req emailRequest + if !decodeJSON(w, r, &req) { + return + } + email, err := users.NormalizeEmail(req.Email) + if err != nil { + writeJSON(w, okResponse{OK: true, Message: msgCheckMail}) + return + } + if !a.allow(w, r, a.signup, "email:"+email) { + return + } + u, err := a.st.GetByEmail(email) + if err != nil || u.Status != users.StatusActive { + writeJSON(w, okResponse{OK: true, Message: msgCheckMail}) + return + } + err = a.mailToken(r.Context(), u, users.PurposeReset, time.Hour, "", "reset", + func(tok string) mailer.Message { return a.resetMail(u, tok) }) + if err != nil { + writeError(w, http.StatusServiceUnavailable, msgMailFailed) + return + } + writeJSON(w, okResponse{OK: true, Message: msgCheckMail}) +} + +func (s *Server) handleReset(w http.ResponseWriter, r *http.Request) { + a := s.auth + var req resetRequest + if !decodeJSON(w, r, &req) { + return + } + if writeValidation(w, users.ValidatePassword(req.Password)) { + return + } + uid, _, err := a.st.ConsumeToken(req.Token, users.PurposeReset) + if err != nil { + writeTokenError(w, err) + return + } + hash, err := users.HashPassword(req.Password) + if err == nil { + err = a.st.SetPassword(uid, hash) + } + if err != nil { + writeError(w, http.StatusInternalServerError, "internal error") + return + } + // Sessions first: a token-store failure below must not leave an + // attacker's session alive. + if err := a.st.DeleteUserSessions(uid, 0); err != nil { + log.Printf("[users] reset: end sessions for user #%d: %v", uid, err) + writeError(w, http.StatusInternalServerError, "internal error") + return + } + // A pending email change must not survive the owner's recovery. + if err := a.invalidateTokens("reset", uid, users.PurposeEmailChange); err != nil { + writeError(w, http.StatusInternalServerError, "internal error") + return + } + a.audit(idPtr(uid), "user.password.reset", idPtr(uid), nil) + writeJSON(w, okResponse{OK: true, Message: "Password changed. Log in with your new password."}) +} diff --git a/cmd/server/auth_login_audit_test.go b/cmd/server/auth_login_audit_test.go new file mode 100644 index 000000000..29f6076ca --- /dev/null +++ b/cmd/server/auth_login_audit_test.go @@ -0,0 +1,165 @@ +package main + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/meshcore-analyzer/users" +) + +func TestLoginIsAudited(t *testing.T) { + f := newAuthFixture(t) + dave := f.registerAndActivate(t, "dave@example.org", "Dave", pw) + expectStatus(t, f.do("POST", "/api/auth/register", registerRequest{Email: "pen@example.org", DisplayName: "Pen", Password: pw}), 200) + pen, err := f.st.GetByEmail("pen@example.org") + if err != nil { + t.Fatal(err) + } + + f.login(t, "dave@example.org", pw) + expectStatus(t, f.do("POST", "/api/auth/login", loginRequest{Email: "dave@example.org", Password: "wrong password!"}), 401) + expectStatus(t, f.do("POST", "/api/auth/login", loginRequest{Email: "nobody@example.org", Password: pw}), 401) + expectStatus(t, f.do("POST", "/api/auth/login", loginRequest{Email: "pen@example.org", Password: pw}), 401) + if err := f.st.SetStatus(dave.me.ID, users.StatusDisabled); err != nil { + t.Fatal(err) + } + expectStatus(t, f.do("POST", "/api/auth/login", loginRequest{Email: "dave@example.org", Password: pw}), 401) + + type row struct { + action string + target int64 + reason string + } + want := []row{ + {"user.login.failed", dave.me.ID, "disabled"}, + {"user.login.failed", pen.ID, "pending"}, + {"user.login.failed", dave.me.ID, "wrong_password"}, + {"user.login", dave.me.ID, ""}, + } + got, err := f.st.AuditList(users.AuditFilter{Actions: []string{"user.login.*"}}) + if err != nil { + t.Fatal(err) + } + if len(got) != len(want) { + t.Fatalf("login rows = %+v; want %d (the unknown address writes none)", got, len(want)) + } + for i, e := range got { + if e.Action != want[i].action || e.ActorUserID != nil || e.TargetUserID == nil || + *e.TargetUserID != want[i].target || e.Detail["reason"] != want[i].reason { + t.Errorf("row %d = %+v; want %+v", i, e, want[i]) + } + } + all, _ := f.st.AuditList(users.AuditFilter{}) + for _, e := range all { + for _, v := range e.Detail { + if strings.Contains(v, "nobody") { + t.Fatalf("an unknown address reached the audit log: %+v", e) + } + } + } +} + +func TestLoginFailureResponsesAreIdentical(t *testing.T) { + f := newAuthFixture(t) + dis := f.registerAndActivate(t, "dis@example.org", "Dis", pw) + if err := f.st.SetStatus(dis.me.ID, users.StatusDisabled); err != nil { + t.Fatal(err) + } + expectStatus(t, f.do("POST", "/api/auth/register", registerRequest{Email: "pen@example.org", DisplayName: "Pen", Password: pw}), 200) + bodies := map[string]string{} + for name, req := range map[string]loginRequest{ + "unknown": {Email: "nobody@example.org", Password: pw}, + "wrong": {Email: "dis@example.org", Password: "wrong password!"}, + "pending": {Email: "pen@example.org", Password: pw}, + "disabled": {Email: "dis@example.org", Password: pw}, + } { + w := f.do("POST", "/api/auth/login", req) + if w.Code != 401 { + t.Errorf("%s: status %d; want 401", name, w.Code) + } + bodies[name] = w.Body.String() + } + for name, b := range bodies { + if b != bodies["unknown"] { + t.Errorf("%s body %q differs from the unknown-address body %q", name, b, bodies["unknown"]) + } + } +} + +func TestLoginAuditFailureDoesNotChangeTheAnswer(t *testing.T) { + f := newAuthFixture(t) + f.registerAndActivate(t, "dave@example.org", "Dave", pw) + ref := f.do("POST", "/api/auth/login", loginRequest{Email: "nobody@example.org", Password: pw}).Body.String() + f.breakTable(t, "audit_log") + w := f.do("POST", "/api/auth/login", loginRequest{Email: "dave@example.org", Password: "wrong password!"}) + if w.Code != 401 || w.Body.String() != ref { + t.Fatalf("with a broken audit log: %d %q; want 401 %q", w.Code, w.Body.String(), ref) + } + f.login(t, "dave@example.org", pw) +} + +func TestJanitorPrunesOldLoginAudit(t *testing.T) { + f := newAuthFixture(t) + dave := f.registerAndActivate(t, "dave@example.org", "Dave", pw) + id := dave.me.ID + old := time.Now().Add(-91 * 24 * time.Hour) + f.st.SetClock(func() time.Time { return old }) + for _, a := range []string{"user.login", "user.login.failed", "user.password.change"} { + if err := f.st.Audit(nil, a, &id, nil); err != nil { + t.Fatal(err) + } + } + f.st.SetClock(time.Now) + f.srv.auth.prune() + list, err := f.st.AuditList(users.AuditFilter{UserID: &id}) + if err != nil { + t.Fatal(err) + } + var acts []string + for _, e := range list { + acts = append(acts, e.Action) + } + if got := strings.Join(acts, ","); got != "user.password.change,user.activate,user.register" { + t.Fatalf("after prune: %s", got) + } +} + +// A locked users.db must not hold the login answer: the audit row is +// written in the background and lands once the lock is gone. A synchronous +// write would block on the lock and then fail, leaving no row. +func TestLoginAnswersBeforeTheAuditWrite(t *testing.T) { + f := newAuthFixture(t) + dave := f.registerAndActivate(t, "dave@example.org", "Dave", pw) + + db, err := openAccountFixtureSQL(f.ownerURL) + if err != nil { + t.Fatal(err) + } + defer db.Close() + conn, err := db.Conn(context.Background()) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + if _, err := conn.ExecContext(context.Background(), testNativeSQL("BEGIN IMMEDIATE", "BEGIN; LOCK TABLE audit_log IN SHARE MODE")); err != nil { + t.Fatal(err) + } + + w := f.serve("POST", "/api/auth/login", loginRequest{Email: "dave@example.org", Password: "wrong password!"}) + if w.Code != 401 { + t.Fatalf("status %d; want 401", w.Code) + } + if _, err := conn.ExecContext(context.Background(), "ROLLBACK"); err != nil { + t.Fatal(err) + } + f.srv.auth.waitAudits() + got, err := f.st.AuditList(users.AuditFilter{Actions: []string{"user.login.failed"}}) + if err != nil { + t.Fatal(err) + } + if len(got) != 1 || got[0].TargetUserID == nil || *got[0].TargetUserID != dave.me.ID { + t.Fatalf("login rows after the lock = %+v; want one for dave", got) + } +} diff --git a/cmd/server/auth_mail.go b/cmd/server/auth_mail.go new file mode 100644 index 000000000..7b99245ef --- /dev/null +++ b/cmd/server/auth_mail.go @@ -0,0 +1,187 @@ +package main + +import ( + "context" + "html" + "log" + "net/url" + "regexp" + "strings" + "time" + "unicode" + + "github.com/meshcore-analyzer/mailer" + "github.com/meshcore-analyzer/users" +) + +// mailLine is one linked line of a mail: a list item in HTML, the text and +// the URL on their own lines in plain text. +type mailLine struct{ text, url string } + +type mailContent struct { + subject string + greeting string + paragraphs []string + lines []mailLine // after the paragraphs + actionLabel string + actionURL string + footer *mailLine // replaces the default footer when set +} + +// link builds {publicBaseUrl}/#/account/{page}?token=… (never from the +// request Host, see the spec's Configuration section). +func (a *authService) link(page, token string) string { + return a.set.baseURL.String() + "/#/account/" + page + "?token=" + url.QueryEscape(token) +} + +// mailSafeText makes a name chosen by someone else safe for one line of a +// mail: control characters (line breaks and tabs included) and bidi +// overrides and isolates (U+202A-U+202E, U+2066-U+2069) become a space, +// runs of spaces collapse and the ends are trimmed. The HTML part escapes +// the result as usual. +func mailSafeText(s string) string { + return strings.Join(strings.Fields(strings.Map(func(r rune) rune { + if unicode.IsControl(r) || (r >= 0x202A && r <= 0x202E) || (r >= 0x2066 && r <= 0x2069) { + return ' ' + } + return r + }, s)), " ") +} + +func (a *authService) render(to, toName, tag string, c mailContent) mailer.Message { + var text, h strings.Builder + text.WriteString(c.greeting + "\n\n") + h.WriteString("

" + html.EscapeString(c.greeting) + "

") + for _, p := range c.paragraphs { + text.WriteString(p + "\n\n") + h.WriteString("

" + html.EscapeString(p) + "

") + } + if len(c.lines) > 0 { + h.WriteString("
    ") + for _, l := range c.lines { + l.text = mailSafeText(l.text) + text.WriteString("- " + l.text + "\n " + l.url + "\n") + h.WriteString(`
  • ` + html.EscapeString(l.text) + `
    • `) + } + h.WriteString("
") + text.WriteString("\n") + } + if c.actionURL != "" { + text.WriteString(c.actionLabel + ":\n" + c.actionURL + "\n\n") + h.WriteString(`

` + html.EscapeString(c.actionLabel) + `

`) + } + if c.footer != nil { + text.WriteString(c.footer.text + "\n" + c.footer.url + "\n") + h.WriteString(`

` + html.EscapeString(c.footer.text) + ` ` + html.EscapeString(c.footer.url) + `

`) + } else { + footer := "You received this because this address was used on " + a.set.baseURL.Host + + ". If that was not you, you can ignore this mail." + text.WriteString(footer + "\n") + h.WriteString(`

` + html.EscapeString(footer) + `

`) + } + return mailer.Message{To: to, ToName: toName, Subject: "[" + a.set.fromName + "] " + c.subject, + HTML: h.String(), Text: text.String(), Tag: tag} +} + +func (a *authService) activationMail(u *users.User, token string) mailer.Message { + return a.render(u.Email, u.DisplayName, "activate", mailContent{ + subject: "Activate your account", greeting: "Hello " + u.DisplayName + ",", + paragraphs: []string{"Confirm your address to activate your account. The link works once and expires in 48 hours.", + "When you open the link, enter the password you chose when you registered."}, + actionLabel: "Activate my account", actionURL: a.link("activate", token)}) +} + +func (a *authService) resetMail(u *users.User, token string) mailer.Message { + return a.render(u.Email, u.DisplayName, "reset", mailContent{ + subject: "Reset your password", greeting: "Hello " + u.DisplayName + ",", + paragraphs: []string{"Someone asked to reset the password of your account. The link works once and expires in 1 hour. Using it logs out all your devices."}, + actionLabel: "Choose a new password", actionURL: a.link("reset", token)}) +} + +func (a *authService) registerNoticeMail(u *users.User) mailer.Message { + return a.render(u.Email, u.DisplayName, "register-notice", mailContent{ + subject: "Registration attempt with your address", greeting: "Hello " + u.DisplayName + ",", + paragraphs: []string{"Someone tried to register a new account with your address, but you already have one.", + "If it was you, log in, or use \"forgot password\" if you lost it."}, + actionLabel: "Log in", actionURL: a.set.baseURL.String() + "/#/account/login"}) +} + +func (a *authService) emailChangeConfirmMail(u *users.User, newEmail, token string) mailer.Message { + return a.render(newEmail, u.DisplayName, "email-change", mailContent{ + subject: "Confirm your new address", greeting: "Hello " + u.DisplayName + ",", + paragraphs: []string{"Confirm that this address should replace the one on your account. The link expires in 24 hours."}, + actionLabel: "Confirm new address", actionURL: a.link("confirm-email", token)}) +} + +func (a *authService) emailChangeNoticeMail(u *users.User, newEmail string) mailer.Message { + return a.render(u.Email, u.DisplayName, "email-change-notice", mailContent{ + subject: "Address change requested", greeting: "Hello " + u.DisplayName + ",", + paragraphs: []string{"A change of your account address to " + newEmail + " was requested. It takes effect only when confirmed from the new address.", + "If this was not you, change your password now."}}) +} + +// addrRE matches anything around an @ up to whitespace or a character that +// cannot appear unquoted in an address, so RFC 5322 atext specials and +// UTF-8 local parts and domains are covered. +var addrRE = regexp.MustCompile(`[^\s"<>()\[\],;:]+@[^\s"<>()\[\],;:]+`) + +// redactAddrs renders a mailer/provider error for the server log with every +// email address replaced by : provider messages can echo the +// recipient, and addresses belong only in the audit and mail tables. +func redactAddrs(err error) string { + return addrRE.ReplaceAllString(err.Error(), "") +} + +// sendMail sends msg and records it in the mail log. The error is logged +// (without tokens) and returned so the caller can answer 503. +func (a *authService) sendMail(ctx context.Context, u *users.User, purpose string, msg mailer.Message) error { + id, err := a.mail.Send(ctx, msg) + if err != nil { + log.Printf("[users] mail %s for user #%d failed: %s", purpose, u.ID, redactAddrs(err)) + return err + } + if _, err := a.st.LogMail(idPtr(u.ID), msg.To, purpose, id); err != nil { + log.Printf("[users] mail log for user #%d: %v", u.ID, err) + } + return nil +} + +// ingestMailEvents records provider events and flags undeliverable addresses. +// Shared by the webhook and the admin "refresh" pull. +func (a *authService) ingestMailEvents(evs []mailer.Event) { + for _, ev := range evs { + uid, found, err := a.st.RecordMailEvent(ev.MessageID, ev.Event, ev.At, ev.Reason) + if err != nil { + log.Printf("[users] record mail event: %v", err) + continue + } + if found && uid != nil && mailer.IsUndeliverable(ev.Event) { + if err := a.st.SetEmailBouncing(*uid, true); err != nil { + log.Printf("[users] flag bouncing for user #%d: %v", *uid, err) + } + } + } +} + +// mailToken issues a one-time token for u, builds the mail around it and +// sends it. If issuing or sending fails the purpose's outstanding tokens are +// invalidated, so no usable link exists for a mail that never left; the +// caller answers 503. newEmail is stored only for users.PurposeEmailChange. +// mailPurpose is the label recorded in the mail log. +func (a *authService) mailToken(ctx context.Context, u *users.User, tokenPurpose users.Purpose, + ttl time.Duration, newEmail, mailPurpose string, build func(token string) mailer.Message) error { + tok, err := a.st.IssueToken(u.ID, tokenPurpose, ttl, newEmail) + if err != nil { + log.Printf("[users] issue %s token for user #%d: %v", tokenPurpose, u.ID, err) + } else { + err = a.sendMail(ctx, u, mailPurpose, build(tok)) + } + if err != nil { + if ierr := a.st.InvalidateTokens(u.ID, tokenPurpose); ierr != nil { + log.Printf("[users] invalidate %s tokens for user #%d: %v", tokenPurpose, u.ID, ierr) + } + return err + } + return nil +} diff --git a/cmd/server/auth_mail_test.go b/cmd/server/auth_mail_test.go new file mode 100644 index 000000000..23677dba0 --- /dev/null +++ b/cmd/server/auth_mail_test.go @@ -0,0 +1,87 @@ +package main + +import ( + "bytes" + "context" + "errors" + "log" + "strings" + "testing" + + "github.com/meshcore-analyzer/users" +) + +func TestRedactAddrs(t *testing.T) { + cases := map[string]string{ + `brevo 400 invalid_parameter: email "Bob.Smith+x@Mail.Example.org" is not valid`: `brevo 400 invalid_parameter: email "" is not valid`, + "to a@b.co and c_d@e-f.example.net failed": "to and failed", + "brevo 503 service unavailable": "brevo 503 service unavailable", + "rejected john.smith&co@example.org": "rejected ", + "rejected jörg.müller@x.de": "rejected ", + "rejected bob@bücher.de": "rejected ", + "rejected bob+newsletter@example.org": "rejected ", + "rejected o'neil!#$*/=?^{|}~@example.org, retry": "rejected , retry", + "recipient ; cc (bea@example.org)": "recipient <>; cc ()", + } + for in, want := range cases { + if got := redactAddrs(errors.New(in)); got != want { + t.Errorf("redactAddrs(%q) = %q, want %q", in, got, want) + } + } +} + +func TestSendMailLogHasNoAddress(t *testing.T) { + a, fake := newTestAuthService(t) + fake.SetSendErr(errors.New(`brevo 400: invalid email victim@example.org`)) + var buf bytes.Buffer + prev := log.Writer() + log.SetOutput(&buf) + defer log.SetOutput(prev) + u := &users.User{ID: 7, Email: "victim@example.org", DisplayName: "Vic"} + if err := a.sendMail(context.Background(), u, "reset", a.resetMail(u, "tok")); err == nil { + t.Fatal("sendMail returned nil on a send error") + } + out := buf.String() + if strings.Contains(out, "victim@example.org") || !strings.Contains(out, "") || !strings.Contains(out, "#7") { + t.Fatalf("log line = %q", out) + } +} + +// Activation needs the account password (handleActivate), so the mail says so. +func TestActivationMailMentionsPassword(t *testing.T) { + a, _ := newTestAuthService(t) + u := &users.User{ID: 1, Email: "new@example.org", DisplayName: "New"} + m := a.activationMail(u, "tok") + for _, body := range []string{m.Text, m.HTML} { + if !strings.Contains(body, "enter the password you chose") { + t.Fatalf("activation mail does not mention the password: %q", body) + } + } +} + +func TestLogStartupRedactsDatabaseURL(t *testing.T) { + a, _ := newTestAuthService(t) + a.set.databaseURL = "postgresql://private:secret@host/accounts?password=hidden" + var buf bytes.Buffer + prev := log.Writer() + log.SetOutput(&buf) + defer log.SetOutput(prev) + a.logStartup() + if !strings.Contains(buf.String(), string(a.st.Backend())) || strings.Contains(buf.String(), "secret") || strings.Contains(buf.String(), "hidden") { + t.Fatalf("unsafe startup log: %q", buf.String()) + } +} + +func TestRenderLinesAndFooter(t *testing.T) { + a, _ := newTestAuthService(t) + m := a.render("x@example.org", "X", "notify", mailContent{subject: "S", greeting: "Hi", + lines: []mailLine{{text: "n: offline", url: "https://e.org/#/nodes/ab"}}, + footer: &mailLine{text: "Stop these mails:", url: "https://e.org/#/account/unsubscribe?token=t"}}) + if !strings.Contains(m.HTML, ``) || + !strings.Contains(m.Text, "- n: offline\n https://e.org/#/nodes/ab\n") || + !strings.Contains(m.HTML, `Stop these mails: `) || + !strings.Contains(m.Text, "Stop these mails:\nhttps://e.org/#/account/unsubscribe?token=t\n") || + strings.Contains(m.Text, "If that was not you") { + t.Fatalf("rendered = %q / %q", m.HTML, m.Text) + } +} diff --git a/cmd/server/auth_ratelimit.go b/cmd/server/auth_ratelimit.go new file mode 100644 index 000000000..b1cd0835a --- /dev/null +++ b/cmd/server/auth_ratelimit.go @@ -0,0 +1,112 @@ +package main + +import ( + "log" + "math" + "net/http" + "strconv" + "sync" + "time" +) + +// rateLimiter is a keyed token bucket: n requests per period, refilled +// continuously. In-memory, so limits reset on restart. That is acceptable +// for brute-force damping, not a quota system. +type rateLimiter struct { + mu sync.Mutex + rate float64 // tokens per second + burst float64 + buckets map[string]*rlBucket + now func() time.Time + + maxBuckets int // cap on len(buckets); a field so tests can lower it + warnFull sync.Once +} + +const rateLimiterMaxBuckets = 100000 + +type rlBucket struct { + tokens float64 + last time.Time +} + +func newRateLimiter(n int, per time.Duration) *rateLimiter { + return &rateLimiter{rate: float64(n) / per.Seconds(), burst: float64(n), + buckets: map[string]*rlBucket{}, now: time.Now, maxBuckets: rateLimiterMaxBuckets} +} + +// take consumes one token for key. When none is left it reports how long +// until the next one. +func (l *rateLimiter) take(key string) (bool, time.Duration) { + l.mu.Lock() + defer l.mu.Unlock() + now := l.now() + b := l.buckets[key] + if b == nil { + if len(l.buckets) >= l.maxBuckets { + l.sweepLocked(now) + if len(l.buckets) >= l.maxBuckets { + l.warnFull.Do(func() { + log.Printf("[users] rate limiter full (%d keys), refusing new keys until buckets refill", l.maxBuckets) + }) + return false, time.Duration(1 / l.rate * float64(time.Second)) + } + } + b = &rlBucket{tokens: l.burst, last: now} + l.buckets[key] = b + } + b.tokens = math.Min(l.burst, b.tokens+now.Sub(b.last).Seconds()*l.rate) + b.last = now + if b.tokens >= 1 { + b.tokens-- + return true, 0 + } + return false, time.Duration((1 - b.tokens) / l.rate * float64(time.Second)) +} + +// gc drops buckets that have refilled completely (they carry no state). +func (l *rateLimiter) gc() { + l.mu.Lock() + defer l.mu.Unlock() + l.sweepLocked(l.now()) +} + +// sweepLocked deletes fully refilled buckets. The caller holds l.mu. +func (l *rateLimiter) sweepLocked(now time.Time) { + for k, b := range l.buckets { + if b.tokens+now.Sub(b.last).Seconds()*l.rate >= l.burst { + delete(l.buckets, k) + } + } +} + +// allow applies l to the client IP (only when IPs can be told apart; same +// rule as the /ws limiter, see ws_limits.go) and to each extra key. It +// writes 429 + Retry-After and returns false when any bucket is empty. +func (a *authService) allow(w http.ResponseWriter, r *http.Request, l *rateLimiter, extraKeys ...string) bool { + keys := extraKeys + if ip, distinct := a.ipr.clientIP(r); distinct && ip != nil { + keys = append([]string{"ip:" + ip.String()}, extraKeys...) + } else { + a.warnIndistinct.Do(func() { + log.Printf("[users] rate limits by IP are off: requests arrive from a proxy and userManagement.trustedProxies is empty; per-address limits still apply, and the webhook limiter is off too") + }) + } + for _, k := range keys { + if ok, wait := l.take(k); !ok { + writeTooManyRequests(w, wait) + return false + } + } + return true +} + +// writeTooManyRequests answers 429 with a Retry-After of at least 1 s. +func writeTooManyRequests(w http.ResponseWriter, wait time.Duration) { + secs := int(math.Ceil(wait.Seconds())) + if secs < 1 { + secs = 1 + } + w.Header().Set("Retry-After", strconv.Itoa(secs)) + writeError(w, http.StatusTooManyRequests, "too many attempts, try again later") +} diff --git a/cmd/server/auth_ratelimit_test.go b/cmd/server/auth_ratelimit_test.go new file mode 100644 index 000000000..6e7db12e6 --- /dev/null +++ b/cmd/server/auth_ratelimit_test.go @@ -0,0 +1,125 @@ +package main + +import ( + "net/http/httptest" + "testing" + "time" + + "github.com/meshcore-analyzer/users" +) + +func TestRateLimiterBurstAndRefill(t *testing.T) { + now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC) + l := newRateLimiter(3, time.Minute) + l.now = func() time.Time { return now } + for i := 0; i < 3; i++ { + if ok, _ := l.take("k"); !ok { + t.Fatalf("take %d refused within burst", i) + } + } + ok, wait := l.take("k") + if ok || wait <= 0 || wait > 20*time.Second { + t.Fatalf("4th take = %v, wait %v", ok, wait) + } + if ok, _ := l.take("other"); !ok { + t.Fatal("keys are not independent") + } + now = now.Add(21 * time.Second) // one token per 20s + if ok, _ := l.take("k"); !ok { + t.Fatal("no refill after 21s") + } + now = now.Add(time.Hour) + l.gc() + l.mu.Lock() + n := len(l.buckets) + l.mu.Unlock() + if n != 0 { + t.Fatalf("gc kept %d full buckets", n) + } +} + +func TestRateLimiterBucketCap(t *testing.T) { + now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC) + l := newRateLimiter(3, time.Minute) + l.now = func() time.Time { return now } + l.maxBuckets = 2 + l.take("a") + l.take("b") + // both buckets are partly drained, so the inline sweep frees nothing + ok, wait := l.take("c") + if ok || wait <= 0 { + t.Fatalf("new key at the cap = %v, wait %v; want refused", ok, wait) + } + if ok, _ := l.take("a"); !ok { + t.Fatal("existing key refused at the cap") + } + // once the old buckets have refilled, the sweep makes room + now = now.Add(time.Hour) + if ok, _ := l.take("c"); !ok { + t.Fatal("new key refused after buckets refilled") + } + l.mu.Lock() + n := len(l.buckets) + l.mu.Unlock() + if n > 2 { + t.Fatalf("buckets = %d, cap 2", n) + } +} + +func TestAllowKeysOnForwardedIP(t *testing.T) { + a, _ := newTestAuthService(t) + a.ipr.trustedProxies = parseCIDRList([]string{"10.0.0.0/8"}, "test") + l := newRateLimiter(1, time.Hour) + try := func(xff string) bool { + r := httptest.NewRequest("POST", "/x", nil) + r.RemoteAddr = "10.0.0.1:4000" + r.Header.Set("X-Forwarded-For", xff) + return a.allow(httptest.NewRecorder(), r, l) + } + if !try("198.51.100.7") { + t.Fatal("first request refused") + } + if try("198.51.100.7") { + t.Fatal("same forwarded IP not limited") + } + if !try("198.51.100.8") { + t.Fatal("different forwarded IP shares a bucket") + } +} + +func TestCloseUserManagementWaitsForJanitor(t *testing.T) { + a, _ := newTestAuthService(t) + srv := &Server{auth: a} + release := make(chan struct{}) + a.wg.Add(1) + go func() { + defer a.wg.Done() + <-a.stop + <-release // janitor still busy after stop was signalled + }() + done := make(chan struct{}) + go func() { srv.closeUserManagement(); close(done) }() + select { + case <-done: + t.Fatal("close returned while the janitor was still running") + case <-time.After(100 * time.Millisecond): + } + close(release) + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("close did not return after the janitor exited") + } +} + +func TestAdminRowConfigAdminNeedsAdminRole(t *testing.T) { + a, _ := newTestAuthService(t, "boss@example.org") + pending := users.User{Email: "boss@example.org", Role: users.RoleUser, Status: users.StatusPending} + if a.adminRow(pending, nil).ConfigAdmin { + t.Fatal("pending squatter on a config address reported as config admin") + } + admin := users.User{Email: "boss@example.org", Role: users.RoleAdmin, Status: users.StatusActive} + if !a.adminRow(admin, nil).ConfigAdmin { + t.Fatal("real config admin not reported") + } +} diff --git a/cmd/server/auth_routes.go b/cmd/server/auth_routes.go new file mode 100644 index 000000000..354c21023 --- /dev/null +++ b/cmd/server/auth_routes.go @@ -0,0 +1,56 @@ +package main + +import "github.com/gorilla/mux" + +// e2eRoutes is set only by the e2etest build (auth_e2e.go). +var e2eRoutes func(s *Server, r *mux.Router) + +// registerAuthRoutes adds every user-management route. Called by +// RegisterRoutes only when s.auth != nil, so with the feature off these +// paths are not registered and fall through to the SPA handler like any +// unknown path. +func (s *Server) registerAuthRoutes(r *mux.Router) { + r.HandleFunc("/api/auth/register", s.requireOrigin(s.handleRegister)).Methods("POST") + r.HandleFunc("/api/auth/activate", s.requireOrigin(s.handleActivate)).Methods("POST") + r.HandleFunc("/api/auth/login", s.requireOrigin(s.handleLogin)).Methods("POST") + r.HandleFunc("/api/auth/logout", s.requireOrigin(s.handleLogout)).Methods("POST") + r.HandleFunc("/api/auth/me", s.handleMe).Methods("GET") + r.HandleFunc("/api/auth/forgot", s.requireOrigin(s.handleForgot)).Methods("POST") + r.HandleFunc("/api/auth/reset", s.requireOrigin(s.handleReset)).Methods("POST") + r.HandleFunc("/api/account", s.withUser(s.handleAccountPatch)).Methods("PATCH") + r.HandleFunc("/api/account", s.withUser(s.handleAccountDelete)).Methods("DELETE") + r.HandleFunc("/api/account/password", s.withUser(s.handleAccountPassword)).Methods("POST") + r.HandleFunc("/api/account/email", s.withUser(s.handleAccountEmail)).Methods("POST") + r.HandleFunc("/api/account/confirm-email", s.requireOrigin(s.handleConfirmEmail)).Methods("POST") + r.HandleFunc("/api/account/sessions", s.withUser(s.handleAccountSessions)).Methods("GET") + r.HandleFunc("/api/account/sessions/{id}", s.withUser(s.handleAccountSessionDelete)).Methods("DELETE") + r.HandleFunc("/api/account/settings", s.withUser(s.handleSettingsGet)).Methods("GET") + r.HandleFunc("/api/account/settings", s.withUser(s.handleSettingsPut)).Methods("PUT") + r.HandleFunc("/api/account/settings", s.withUser(s.handleSettingsDelete)).Methods("DELETE") + r.HandleFunc("/api/account/export", s.withUser(s.handleAccountExport)).Methods("GET") + r.HandleFunc("/api/admin/users", s.withAdmin(s.handleAdminUsers)).Methods("GET") + r.HandleFunc("/api/admin/users/{id}", s.withAdmin(s.handleAdminUserDetail)).Methods("GET") + r.HandleFunc("/api/admin/users/{id}", s.withAdmin(s.handleAdminDelete)).Methods("DELETE") + r.HandleFunc("/api/admin/users/{id}/disable", s.withAdmin(s.handleAdminDisable)).Methods("POST") + r.HandleFunc("/api/admin/users/{id}/enable", s.withAdmin(s.handleAdminEnable)).Methods("POST") + r.HandleFunc("/api/admin/users/{id}/role", s.withAdmin(s.handleAdminRole)).Methods("POST") + r.HandleFunc("/api/admin/users/{id}/resend-activation", s.withAdmin(s.handleAdminResendActivation)).Methods("POST") + r.HandleFunc("/api/admin/users/{id}/activate", s.withAdmin(s.handleAdminActivate)).Methods("POST") + r.HandleFunc("/api/admin/users/{id}/mail/{mailId}/refresh", s.withAdmin(s.handleAdminMailRefresh)).Methods("POST") + r.HandleFunc("/api/admin/audit", s.withAdmin(s.handleAdminAudit)).Methods("GET") + r.HandleFunc("/api/admin/stats", s.withAdmin(s.handleAdminStats)).Methods("GET") + r.HandleFunc("/api/admin/users-backup", s.withAdmin(s.handleAdminUsersBackup)).Methods("GET") + if s.auth.set.proposals.enabled { + s.registerProposalRoutes(r) + } + if s.auth.notify != nil { + s.registerNotifyRoutes(r) + } + // The webhook exists only when a secret is configured. + if s.auth.set.webhookSecret != "" { + r.HandleFunc("/api/mail/brevo/webhook", s.handleBrevoWebhook).Methods("POST") + } + if e2eRoutes != nil { + e2eRoutes(s, r) + } +} diff --git a/cmd/server/auth_service.go b/cmd/server/auth_service.go new file mode 100644 index 000000000..cd5e04549 --- /dev/null +++ b/cmd/server/auth_service.go @@ -0,0 +1,265 @@ +package main + +import ( + "github.com/meshcore-analyzer/dbconfig" + "log" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/meshcore-analyzer/mailer" + "github.com/meshcore-analyzer/users" +) + +// authService is the optional user-management layer. Server.auth is nil +// unless userManagement.enabled, and nothing in this file runs then. +type authService struct { + st *users.Store + mail mailer.Mailer + set *userMgmtSettings + ipr *wsLimiter // only its clientIP rule is used + + login *rateLimiter + signup *rateLimiter // register, forgot, self-service resend + hook *rateLimiter + settingsPut *rateLimiter // PUT /api/account/settings, per user + + // approved is the snapshot of approved hashtag channel names, filled at + // construction and reloaded from users.db after every proposal decision. + // approvedMu serialises the reloads so an older read never replaces a + // newer one. + approvedMu sync.Mutex + approved atomic.Pointer[[]string] + + // notify evaluates node notifications; nil unless + // userManagement.notifications.enabled. + notify *notifier + + warnIndistinct sync.Once + stop chan struct{} + stopOnce sync.Once + wg sync.WaitGroup // the janitor and the notifier + auditWG sync.WaitGroup // in-flight auditAsync writes +} + +func newAuthService(set *userMgmtSettings, st *users.Store, m mailer.Mailer) *authService { + a := &authService{ + st: st, mail: m, set: set, + ipr: &wsLimiter{trustedProxies: set.trustedProxies}, + login: newRateLimiter(10, 15*time.Minute), + signup: newRateLimiter(5, time.Hour), + hook: newRateLimiter(600, time.Minute), + settingsPut: newRateLimiter(60, time.Hour), + stop: make(chan struct{}), + } + a.refreshApproved() + return a +} + +// initUserManagement builds s.auth when the feature is on. Call it after +// NewServer and before RegisterRoutes. measurementDatabaseURL is passed to +// users.Open as a forbidden database, so accounts cannot share telemetry storage. +func (s *Server) initUserManagement(measurementTarget string) error { + if !s.cfg.UserManagementEnabled() { + return nil + } + set, err := resolveUserManagement(s.cfg.UserManagement, runtimeStateDir, os.Getenv) + if err != nil { + return err + } + storage := dbconfig.Storage{Backend: dbconfig.SQLite, DBPath: measurementTarget, UsersDBPath: set.databaseURL, StateDir: runtimeStateDir} + if strings.HasPrefix(measurementTarget, "postgres://") || strings.HasPrefix(measurementTarget, "postgresql://") { + storage = dbconfig.Storage{Backend: dbconfig.Postgres, ReaderDatabaseURL: measurementTarget, UsersDatabaseURL: set.databaseURL, StateDir: runtimeStateDir} + } + return s.initUserManagementStorage(storage) +} +func (s *Server) initUserManagementStorage(storage dbconfig.Storage) error { + if !s.cfg.UserManagementEnabled() { + return nil + } + set, err := resolveUserManagement(s.cfg.UserManagement, storage.StateDir, os.Getenv) + if err != nil { + return err + } + st, err := users.OpenStorage(storage) + if err != nil { + return err + } + var m mailer.Mailer + if set.provider == "fake" { + m = &mailer.Fake{} + } else { + m = mailer.NewBrevo(set.brevoAPIKey, set.fromEmail, set.fromName) + } + s.auth = newAuthService(set, st, m) + s.auth.logStartup() + s.auth.wg.Add(1) + go func() { + defer s.auth.wg.Done() + s.auth.janitor(time.Hour) + }() + if set.notify.enabled { + s.auth.notify = newNotifier(s.auth, serverNotifySource{s: s}, time.Now) + s.auth.wg.Add(1) + go func() { + defer s.auth.wg.Done() + s.auth.notify.loop(s.auth.stop, set.notify.interval) + }() + log.Printf("[notify] node notifications enabled: every %s, at most %d mails per user and %d in total per 24 hours", + set.notify.interval, set.notify.perUserPerDay, set.notify.maxMailsPerDay) + } + return nil +} + +// closeUserManagement stops the janitor and closes the account connection pool. Safe when off. +func (s *Server) closeUserManagement() { + if s.auth == nil { + return + } + s.auth.stopOnce.Do(func() { + close(s.auth.stop) + s.auth.wg.Wait() + s.auth.waitAudits() + if err := s.auth.st.Close(); err != nil { + log.Printf("[users] close: %v", err) + } + }) +} + +func (a *authService) logStartup() { + log.Printf("[users] user management enabled: %s, %d config admin(s), webhook=%v", + a.st.Backend(), len(a.set.adminEmails), a.set.webhookSecret != "") + admins, err := a.st.List(users.ListFilter{Role: users.RoleAdmin}) + if err != nil { + log.Printf("[users] list admins: %v", err) + return + } + for _, u := range admins { + if !a.isConfigAdmin(u.Email) { + log.Printf("[users] note: admin #%d is not in adminEmails (promoted in the UI, or removed from config)", u.ID) + } + } +} + +func (a *authService) janitor(every time.Duration) { + t := time.NewTicker(every) + defer t.Stop() + for { + a.prune() + a.maybeBackup(time.Now()) + select { + case <-a.stop: + return + case <-t.C: + } + } +} + +func (a *authService) prune() { + if _, err := a.st.PruneStalePending(48 * time.Hour); err != nil { + log.Printf("[users] prune pending: %v", err) + } + if _, err := a.st.PruneExpiredSessions(); err != nil { + log.Printf("[users] prune sessions: %v", err) + } + if _, err := a.st.PruneTokens(7 * 24 * time.Hour); err != nil { + log.Printf("[users] prune tokens: %v", err) + } + if _, err := a.st.PruneMail(90 * 24 * time.Hour); err != nil { + log.Printf("[users] prune mail log: %v", err) + } + // Login rows only; every other audit action is kept (admin-dashboard spec). + if _, err := a.st.PruneAudit([]string{"user.login", "user.login.failed"}, 90*24*time.Hour); err != nil { + log.Printf("[users] prune login audit: %v", err) + } + if _, err := a.st.PruneProposals(proposalRetention); err != nil { + log.Printf("[users] prune proposals: %v", err) + } + a.login.gc() + a.signup.gc() + a.hook.gc() + a.settingsPut.gc() +} + +func (a *authService) isConfigAdmin(email string) bool { return a.set.adminEmails[email] } + +// refreshApproved reloads the approved hashtag channel names from users.db, +// oldest approval first and capped at maxApproved like the ingestor. The +// mutex spans the read and the store, so concurrent decisions cannot leave +// an older read in place. A failure is logged and keeps the previous +// snapshot. No-op with proposals off. +func (a *authService) refreshApproved() { + if !a.set.proposals.enabled { + return + } + a.approvedMu.Lock() + defer a.approvedMu.Unlock() + list, err := a.st.ApprovedSubjects(users.KindHashtagChannel, a.set.proposals.maxApproved) + if err != nil { + log.Printf("[users] load approved channels: %v", err) + return + } + a.approved.Store(&list) +} + +// approvedChannels returns the current snapshot; never nil. +func (a *authService) approvedChannels() []string { + if p := a.approved.Load(); p != nil { + return *p + } + return []string{} +} + +// roleFor is the role an address gets on activation. +func (a *authService) roleFor(email string) users.Role { + if a.isConfigAdmin(email) { + return users.RoleAdmin + } + return users.RoleUser +} + +func (a *authService) audit(actor *int64, action string, target *int64, detail map[string]string) { + if err := a.st.Audit(actor, action, target, detail); err != nil { + log.Printf("[users] audit %s: %v", action, err) + } +} + +// auditAsync writes an audit row in the background, so neither the write +// nor a locked users.db delays the response. Best-effort: a failure is +// logged by a.audit, and a row is lost if the process stops first. +func (a *authService) auditAsync(actor *int64, action string, target *int64, detail map[string]string) { + a.auditWG.Add(1) + go func() { + defer a.auditWG.Done() + a.audit(actor, action, target, detail) + }() +} + +// waitAudits blocks until every auditAsync write has finished. +func (a *authService) waitAudits() { a.auditWG.Wait() } + +func idPtr(id int64) *int64 { return &id } + +// invalidateTokens burns uid's outstanding links of each purpose. A failure +// is logged (user id only) and returned; callers answer 500. +func (a *authService) invalidateTokens(op string, uid int64, ps ...users.Purpose) error { + for _, p := range ps { + if err := a.st.InvalidateTokens(uid, p); err != nil { + log.Printf("[users] %s: invalidate %s tokens for user #%d: %v", op, p, uid, err) + return err + } + } + return nil +} + +// absForLog resolves path for a log line, so a relative users.db path shows +// which file it means; the path as given when it cannot be resolved. +func absForLog(path string) string { + if abs, err := filepath.Abs(path); err == nil { + return abs + } + return path +} diff --git a/cmd/server/auth_session.go b/cmd/server/auth_session.go new file mode 100644 index 000000000..ce8f21875 --- /dev/null +++ b/cmd/server/auth_session.go @@ -0,0 +1,132 @@ +package main + +import ( + "net/http" + "net/url" + "strings" + "time" + + "github.com/meshcore-analyzer/users" +) + +const ( + sessionCookieName = "cs_session" + csrfHeader = "X-CS-CSRF" +) + +func (a *authService) setSessionCookie(w http.ResponseWriter, raw string, expires time.Time) { + http.SetCookie(w, &http.Cookie{ + Name: sessionCookieName, Value: raw, Path: "/", + Expires: expires, MaxAge: int(time.Until(expires).Seconds()), + HttpOnly: true, Secure: a.set.secureCookie, SameSite: http.SameSiteLaxMode, + }) +} + +func (a *authService) clearSessionCookie(w http.ResponseWriter) { + http.SetCookie(w, &http.Cookie{Name: sessionCookieName, Value: "", Path: "/", MaxAge: -1, + HttpOnly: true, Secure: a.set.secureCookie, SameSite: http.SameSiteLaxMode}) +} + +// currentUser returns the active user behind the session cookie, or nils. +// A session last seen more than a day ago is extended (sliding expiry) +// and its cookie re-issued. +func (a *authService) currentUser(w http.ResponseWriter, r *http.Request) (*users.User, *users.Session) { + c, err := r.Cookie(sessionCookieName) + if err != nil || c.Value == "" { + return nil, nil + } + sess, err := a.st.LookupSession(c.Value) + if err != nil { + return nil, nil + } + u, err := a.st.GetByID(sess.UserID) + if err != nil || u.Status != users.StatusActive { + return nil, nil + } + if time.Since(sess.LastSeenAt) > 24*time.Hour { + if err := a.st.ExtendSession(sess.ID, a.set.sessionTTL); err == nil { + a.setSessionCookie(w, c.Value, time.Now().Add(a.set.sessionTTL)) + } + } + return u, sess +} + +// originOK requires the request's Origin (or, if absent, Referer) to be the +// configured publicBaseUrl's origin. +func (a *authService) originOK(r *http.Request) bool { + want := a.set.baseURL.Scheme + "://" + a.set.baseURL.Host + if o := r.Header.Get("Origin"); o != "" { + return strings.EqualFold(o, want) + } + if ref := r.Header.Get("Referer"); ref != "" { + u, err := url.Parse(ref) + return err == nil && strings.EqualFold(u.Scheme+"://"+u.Host, want) + } + return false +} + +func (a *authService) csrfOK(r *http.Request, sess *users.Session) bool { + return a.originOK(r) && constantTimeEqual(r.Header.Get(csrfHeader), sess.CSRFToken) +} + +func isSafeMethod(m string) bool { return m == http.MethodGet || m == http.MethodHead } + +type authedHandler func(w http.ResponseWriter, r *http.Request, u *users.User, sess *users.Session) + +// withUser requires a logged-in active user; state-changing methods must +// also pass the origin + CSRF-token check. +func (s *Server) withUser(h authedHandler) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + u, sess := s.auth.currentUser(w, r) + if u == nil { + writeError(w, http.StatusUnauthorized, "not logged in") + return + } + if !isSafeMethod(r.Method) && !s.auth.csrfOK(r, sess) { + writeError(w, http.StatusForbidden, "CSRF check failed") + return + } + h(w, r, u, sess) + } +} + +// adminSessionOK is the one place that decides whether a session user may +// act as admin: state-changing methods must pass the origin + CSRF check, +// and the role must be admin. It writes the 403 itself when not. +func (a *authService) adminSessionOK(w http.ResponseWriter, r *http.Request, u *users.User, sess *users.Session) bool { + if !isSafeMethod(r.Method) && !a.csrfOK(r, sess) { + writeError(w, http.StatusForbidden, "CSRF check failed") + return false + } + if u.Role != users.RoleAdmin { + writeError(w, http.StatusForbidden, "admin role required") + return false + } + return true +} + +// withAdmin requires a logged-in admin (see adminSessionOK). +func (s *Server) withAdmin(h authedHandler) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + u, sess := s.auth.currentUser(w, r) + if u == nil { + writeError(w, http.StatusUnauthorized, "not logged in") + return + } + if !s.auth.adminSessionOK(w, r, u, sess) { + return + } + h(w, r, u, sess) + } +} + +// requireOrigin guards unauthenticated state-changing endpoints (login CSRF). +func (s *Server) requireOrigin(h http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if !s.auth.originOK(r) { + writeError(w, http.StatusForbidden, "request origin not allowed") + return + } + h(w, r) + } +} diff --git a/cmd/server/auth_types.go b/cmd/server/auth_types.go new file mode 100644 index 000000000..97b9a2a7c --- /dev/null +++ b/cmd/server/auth_types.go @@ -0,0 +1,217 @@ +package main + +import ( + "encoding/json" + "errors" + "net/http" + "time" + + "github.com/meshcore-analyzer/users" +) + +type okResponse struct { + OK bool `json:"ok"` + Message string `json:"message,omitempty"` +} + +type meResponse struct { + ID int64 `json:"id"` + Email string `json:"email"` + DisplayName string `json:"displayName"` + Role users.Role `json:"role"` + CSRFToken string `json:"csrfToken"` +} + +type registerRequest struct { + Email string `json:"email"` + DisplayName string `json:"displayName"` + Password string `json:"password"` +} + +type loginRequest struct { + Email string `json:"email"` + Password string `json:"password"` +} + +type emailRequest struct { + Email string `json:"email"` +} + +type tokenRequest struct { + Token string `json:"token"` +} + +type activateRequest struct { + Token string `json:"token"` + Password string `json:"password"` +} + +type resetRequest struct { + Token string `json:"token"` + Password string `json:"password"` +} + +type profileRequest struct { + DisplayName string `json:"displayName"` +} + +type passwordChangeRequest struct { + CurrentPassword string `json:"currentPassword"` + NewPassword string `json:"newPassword"` +} + +type emailChangeRequest struct { + NewEmail string `json:"newEmail"` + CurrentPassword string `json:"currentPassword"` +} + +type passwordConfirmRequest struct { + CurrentPassword string `json:"currentPassword"` +} + +type roleRequest struct { + Role users.Role `json:"role"` +} + +type sessionJSON struct { + ID int64 `json:"id"` + CreatedAt string `json:"createdAt"` + LastSeenAt string `json:"lastSeenAt"` + ExpiresAt string `json:"expiresAt"` + UserAgent string `json:"userAgent"` + Current bool `json:"current"` +} + +type mailEventJSON struct { + Event string `json:"event"` + At string `json:"at"` + Reason string `json:"reason,omitempty"` +} + +type mailJSON struct { + ID int64 `json:"id"` + Purpose string `json:"purpose"` + To string `json:"to"` + SentAt string `json:"sentAt"` + LastEvent string `json:"lastEvent"` + LastEventAt string `json:"lastEventAt"` + LastReason string `json:"lastReason,omitempty"` + Events []mailEventJSON `json:"events,omitempty"` +} + +type adminUserJSON struct { + ID int64 `json:"id"` + Email string `json:"email"` + DisplayName string `json:"displayName"` + Role users.Role `json:"role"` + Status users.Status `json:"status"` + CreatedAt string `json:"createdAt"` + ActivatedAt *string `json:"activatedAt"` + ActivatedManually bool `json:"activatedManually"` + ActivatedBy *int64 `json:"activatedBy"` + LastLoginAt *string `json:"lastLoginAt"` + EmailBouncing bool `json:"emailBouncing"` + ConfigAdmin bool `json:"configAdmin"` + LastMail *mailJSON `json:"lastMail"` +} + +type auditJSON struct { + ID int64 `json:"id"` + At string `json:"at"` + ActorUserID *int64 `json:"actorUserId"` + Action string `json:"action"` + TargetUserID *int64 `json:"targetUserId"` + Detail map[string]string `json:"detail"` +} + +type adminUserDetailJSON struct { + User adminUserJSON `json:"user"` + Sessions []sessionJSON `json:"sessions"` + Mail []mailJSON `json:"mail"` + Audit []auditJSON `json:"audit"` +} + +func rfc3339(t time.Time) string { return t.UTC().Format(time.RFC3339) } + +func rfc3339Ptr(t *time.Time) *string { + if t == nil { + return nil + } + s := rfc3339(*t) + return &s +} + +func meFrom(u *users.User, sess *users.Session) meResponse { + return meResponse{ID: u.ID, Email: u.Email, DisplayName: u.DisplayName, Role: u.Role, CSRFToken: sess.CSRFToken} +} + +func sessionToJSON(s users.Session, currentID int64) sessionJSON { + return sessionJSON{ID: s.ID, CreatedAt: rfc3339(s.CreatedAt), LastSeenAt: rfc3339(s.LastSeenAt), + ExpiresAt: rfc3339(s.ExpiresAt), UserAgent: s.UserAgent, Current: s.ID == currentID} +} + +func mailToJSON(m users.MailRecord) mailJSON { + out := mailJSON{ID: m.ID, Purpose: m.Purpose, To: m.ToEmail, SentAt: rfc3339(m.SentAt), + LastEvent: m.LastEvent, LastEventAt: rfc3339(m.LastEventAt), LastReason: m.LastReason} + for _, e := range m.Events { + out.Events = append(out.Events, mailEventJSON{Event: e.Event, At: rfc3339(e.At), Reason: e.Reason}) + } + return out +} + +func (a *authService) adminRow(u users.User, last map[int64]users.MailRecord) adminUserJSON { + row := adminUserJSON{ID: u.ID, Email: u.Email, DisplayName: u.DisplayName, Role: u.Role, Status: u.Status, + CreatedAt: rfc3339(u.CreatedAt), ActivatedAt: rfc3339Ptr(u.ActivatedAt), ActivatedManually: u.ActivatedBy != nil, + ActivatedBy: u.ActivatedBy, LastLoginAt: rfc3339Ptr(u.LastLoginAt), EmailBouncing: u.EmailBouncing, + ConfigAdmin: u.Role == users.RoleAdmin && a.isConfigAdmin(u.Email)} + if m, ok := last[u.ID]; ok { + mj := mailToJSON(m) + row.LastMail = &mj + } + return row +} + +// decodeJSON reads a small JSON body into dst, rejecting unknown fields. +func decodeJSON(w http.ResponseWriter, r *http.Request, dst any) bool { + return decodeJSONMax(w, r, dst, 16<<10, http.StatusBadRequest) +} + +// decodeJSONMax is decodeJSON with a body cap of limit bytes. A body over +// the cap answers tooLarge (decodeJSON keeps its historical 400). +func decodeJSONMax(w http.ResponseWriter, r *http.Request, dst any, limit int64, tooLarge int) bool { + r.Body = http.MaxBytesReader(w, r.Body, limit) + dec := json.NewDecoder(r.Body) + dec.DisallowUnknownFields() + if err := dec.Decode(dst); err != nil { + var mbe *http.MaxBytesError + if errors.As(err, &mbe) && tooLarge != http.StatusBadRequest { + writeError(w, tooLarge, "request body too large") + return false + } + writeError(w, http.StatusBadRequest, "invalid request body") + return false + } + return true +} + +// writeValidation writes 400 for a users.ValidationError and reports +// whether it did. +func writeValidation(w http.ResponseWriter, err error) bool { + var ve *users.ValidationError + if errors.As(err, &ve) { + writeError(w, http.StatusBadRequest, ve.Msg) + return true + } + return false +} + +func writeTokenError(w http.ResponseWriter, err error) { + switch { + case errors.Is(err, users.ErrTokenExpired): + writeError(w, http.StatusGone, "this link has expired") + case errors.Is(err, users.ErrTokenInvalid): + writeError(w, http.StatusGone, "this link is invalid or was already used") + default: + writeError(w, http.StatusInternalServerError, "internal error") + } +} diff --git a/cmd/server/backup.go b/cmd/server/backup.go index 91915ed1a..a362118da 100644 --- a/cmd/server/backup.go +++ b/cmd/server/backup.go @@ -7,29 +7,16 @@ import ( "net/http" "os" "path/filepath" - "strings" "time" + + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/pgutil" + "github.com/meshcore-analyzer/sqliteutil" ) -// handleBackup streams a consistent SQLite snapshot of the analyzer DB. -// -// Requires API-key authentication (mounted via requireAPIKey in routes.go). -// -// Strategy: SQLite's `VACUUM INTO 'path'` produces an atomic, defragmented -// copy of the current database into a new file. It runs at READ ISOLATION -// against the source DB (works on our read-only connection) and never -// blocks concurrent writers — the ingestor keeps writing to the WAL while -// the snapshot is taken from a consistent read transaction. -// -// Response: -// -// 200 OK -// Content-Type: application/octet-stream -// Content-Disposition: attachment; filename="corescope-backup-.db" -// -// -// The temp file is removed after the response is fully written, regardless -// of whether the client successfully consumed the stream. +// handleBackup stages a consistent native SQLite file or PostgreSQL archive. +// pgutil passes credentials through the child environment, bounds its lifetime, +// removes partial output, and returns redacted errors. func (s *Server) handleBackup(w http.ResponseWriter, r *http.Request) { if s.db == nil || s.db.conn == nil { writeError(w, http.StatusServiceUnavailable, "database unavailable") @@ -56,12 +43,18 @@ func (s *Server) handleBackup(w http.ResponseWriter, r *http.Request) { } }() - snapshotPath := filepath.Join(tmpDir, fmt.Sprintf("corescope-backup-%d.db", ts)) + extension := ".dump" + if s.db.Backend() == dbconfig.SQLite { + extension = ".db" + } + snapshotPath := filepath.Join(tmpDir, fmt.Sprintf("corescope-backup-%d%s", ts, extension)) - // SQLite parses the path literal — escape any single quotes defensively. - // (mkdtemp output won't contain quotes, but be paranoid for future-proofing.) - escaped := strings.ReplaceAll(snapshotPath, "'", "''") - if _, err := s.db.conn.ExecContext(r.Context(), fmt.Sprintf("VACUUM INTO '%s'", escaped)); err != nil { + if s.db.Backend() == dbconfig.SQLite { + err = sqliteutil.Snapshot(r.Context(), s.db.path, snapshotPath) + } else { + err = pgutil.Dump(r.Context(), s.db.path, snapshotPath) + } + if err != nil { writeError(w, http.StatusInternalServerError, "snapshot failed: "+err.Error()) return } @@ -78,8 +71,9 @@ func (s *Server) handleBackup(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Length", fmt.Sprintf("%d", stat.Size())) } w.Header().Set("Content-Type", "application/octet-stream") - w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"corescope-backup-%d.db\"", ts)) + w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"corescope-backup-%d%s\"", ts, extension)) w.Header().Set("X-Content-Type-Options", "nosniff") + w.Header().Set("Cache-Control", "no-store") w.WriteHeader(http.StatusOK) if _, err := io.Copy(w, f); err != nil { diff --git a/cmd/server/backup_test.go b/cmd/server/backup_test.go index 024ef3cb1..d4c4f0c8f 100644 --- a/cmd/server/backup_test.go +++ b/cmd/server/backup_test.go @@ -3,13 +3,32 @@ package main import ( "net/http" "net/http/httptest" + "net/url" + "os" + "path/filepath" "strings" "testing" ) -// sqliteMagic is the 16-byte file header identifying a valid SQLite 3 database. -// See https://www.sqlite.org/fileformat.html#magic_header_string -const sqliteMagic = "SQLite format 3\x00" +// postgresMagic is the signature of a PostgreSQL custom-format archive. +var postgresMagic = testNativeSQL("SQLite format 3\x00", "PGDMP") + +// Installed PostgreSQL selections pin even the default schema explicitly. +func selectedBackupTestDSN(t *testing.T) string { + t.Helper() + dsn := testDatabaseDSN(t) + if testBackendValue() == "postgres" { + u, err := url.Parse(dsn) + if err != nil { + t.Fatal("invalid fixture URL") + } + q := u.Query() + q.Set("search_path", `"public"`) + u.RawQuery = q.Encode() + dsn = u.String() + } + return dsn +} func TestBackupRequiresAPIKey(t *testing.T) { _, router := setupTestServerWithAPIKey(t, "test-secret-key-strong-enough") @@ -22,9 +41,18 @@ func TestBackupRequiresAPIKey(t *testing.T) { } } -func TestBackupReturnsValidSQLiteSnapshot(t *testing.T) { +func TestBackupRestoresNativePostgresSnapshot(t *testing.T) { const apiKey = "test-secret-key-strong-enough" - _, router := setupTestServerWithAPIKey(t, apiKey) + srv, router := setupTestServerWithAPIKey(t, apiKey) + srv.db = setupTestDBAtURL(t, selectedBackupTestDSN(t)) + seedTestData(t, srv.db) + writerURL := srv.db.path + reader, err := openFixtureReader(t, writerURL) + if err != nil { + t.Fatal(err) + } + defer reader.Close() + srv.db = reader req := httptest.NewRequest("GET", "/api/backup", nil) req.Header.Set("X-API-Key", apiKey) @@ -41,15 +69,31 @@ func TestBackupReturnsValidSQLiteSnapshot(t *testing.T) { } cd := w.Header().Get("Content-Disposition") - if !strings.HasPrefix(cd, "attachment;") || !strings.Contains(cd, "filename=\"corescope-backup-") || !strings.HasSuffix(cd, ".db\"") { - t.Errorf("expected Content-Disposition attachment with corescope-backup-.db filename, got %q", cd) + if !strings.HasPrefix(cd, "attachment;") || !strings.Contains(cd, "filename=\"corescope-backup-") || !strings.HasSuffix(cd, testNativeSQL(".db\"", ".dump\"")) { + t.Errorf("expected Content-Disposition attachment with corescope-backup-.dump filename, got %q", cd) } body := w.Body.Bytes() - if len(body) < len(sqliteMagic) { - t.Fatalf("backup body too short (%d bytes) — expected SQLite file", len(body)) + if len(body) < len(postgresMagic) { + t.Fatalf("backup body too short (%d bytes) — expected PostgreSQL file", len(body)) + } + if got := string(body[:len(postgresMagic)]); got != postgresMagic { + t.Fatalf("expected PostgreSQL magic header %q, got %q", postgresMagic, got) } - if got := string(body[:len(sqliteMagic)]); got != sqliteMagic { - t.Fatalf("expected SQLite magic header %q, got %q", sqliteMagic, got) + path := filepath.Join(t.TempDir(), "telemetry.dump") + if err := os.WriteFile(path, body, 0o600); err != nil { + t.Fatal(err) } + restored := restorePostgresTestBackup(t, path) + var transmissions, observations int + if err := restored.QueryRow(`SELECT COUNT(*) FROM transmissions`).Scan(&transmissions); err != nil { + t.Fatal(err) + } + if err := restored.QueryRow(`SELECT COUNT(*) FROM observations`).Scan(&observations); err != nil { + t.Fatal(err) + } + if transmissions != 3 || observations != 4 { + t.Fatalf("restored counts = %d/%d", transmissions, observations) + } + } diff --git a/cmd/server/body_limits_test.go b/cmd/server/body_limits_test.go new file mode 100644 index 000000000..47741f980 --- /dev/null +++ b/cmd/server/body_limits_test.go @@ -0,0 +1,55 @@ +package main + +import ( + "bytes" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +// The two unauthenticated POST endpoints must cap their request bodies. +// Before the cap, json.Decoder buffered the whole body and /api/decode +// hex-decoded it before the 184-byte payload check ran, so one request +// could hold a multi-hundred-MB buffer. +func TestUnauthenticatedPostBodyLimits(t *testing.T) { + _, router := setupTestServerWithAPIKey(t, "") + + post := func(path, body string) *httptest.ResponseRecorder { + req := httptest.NewRequest("POST", path, bytes.NewBufferString(body)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + return w + } + + t.Run("decode: normal body still works", func(t *testing.T) { + w := post("/api/decode", `{"hex":"0200"}`) + if w.Code != http.StatusOK { + t.Fatalf("expected 200, got %d (body: %s)", w.Code, w.Body.String()) + } + }) + + t.Run("decode: oversized body is rejected with 413", func(t *testing.T) { + big := `{"hex":"` + strings.Repeat("ab", decodeBodyLimit) + `"}` + w := post("/api/decode", big) + if w.Code != http.StatusRequestEntityTooLarge { + t.Fatalf("expected 413, got %d (body: %s)", w.Code, w.Body.String()) + } + }) + + t.Run("observations: normal body still works", func(t *testing.T) { + w := post("/api/packets/observations", `{"hashes":["abc123"]}`) + if w.Code != http.StatusOK { + t.Fatalf("expected 200, got %d (body: %s)", w.Code, w.Body.String()) + } + }) + + t.Run("observations: oversized body is rejected with 413", func(t *testing.T) { + big := `{"hashes":["` + strings.Repeat("a", batchObservationsBodyLimit) + `"]}` + w := post("/api/packets/observations", big) + if w.Code != http.StatusRequestEntityTooLarge { + t.Fatalf("expected 413, got %d (body: %s)", w.Code, w.Body.String()) + } + }) +} diff --git a/cmd/server/bounded_load_test.go b/cmd/server/bounded_load_test.go index 6da1d4586..1c5379429 100644 --- a/cmd/server/bounded_load_test.go +++ b/cmd/server/bounded_load_test.go @@ -1,21 +1,15 @@ package main import ( - "database/sql" "fmt" - "os" - "path/filepath" "testing" "time" - - _ "modernc.org/sqlite" ) // createTestDB creates a temporary SQLite database with N transmissions (1 obs each). func createTestDB(t *testing.T, numTx int) string { t.Helper() - dir := t.TempDir() - dbPath := filepath.Join(dir, "test.db") + dbPath := postgresTestDSN(t) createTestDBAt(t, dbPath, numTx) return dbPath } @@ -23,7 +17,7 @@ func createTestDB(t *testing.T, numTx int) string { // loadStore creates a PacketStore from a test DB with given maxMemoryMB. func loadStore(t *testing.T, dbPath string, maxMemMB int) *PacketStore { t.Helper() - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } @@ -37,7 +31,6 @@ func loadStore(t *testing.T, dbPath string, maxMemMB int) *PacketStore { func TestBoundedLoad_LimitedMemory(t *testing.T) { dbPath := createTestDB(t, 5000) - defer os.RemoveAll(filepath.Dir(dbPath)) // Use 1MB budget — should load far fewer than 5000 packets store := loadStore(t, dbPath, 1) @@ -47,15 +40,14 @@ func TestBoundedLoad_LimitedMemory(t *testing.T) { if loaded >= 5000 { t.Errorf("expected bounded load to limit packets, got %d/5000", loaded) } - if loaded < 1000 { - t.Errorf("expected at least 1000 packets (minimum), got %d", loaded) + if loaded == 0 || store.trackedBytes > 1048576 { + t.Errorf("expected a nonempty cache within 1 MiB, got %d packets / %d bytes", loaded, store.trackedBytes) } t.Logf("Loaded %d/5000 packets with 1MB budget", loaded) } func TestBoundedLoad_NewestFirst(t *testing.T) { dbPath := createTestDB(t, 5000) - defer os.RemoveAll(filepath.Dir(dbPath)) store := loadStore(t, dbPath, 1) defer store.db.conn.Close() @@ -85,7 +77,6 @@ func TestBoundedLoad_NewestFirst(t *testing.T) { func TestBoundedLoad_OldestLoadedSet(t *testing.T) { dbPath := createTestDB(t, 5000) - defer os.RemoveAll(filepath.Dir(dbPath)) store := loadStore(t, dbPath, 1) defer store.db.conn.Close() @@ -93,15 +84,18 @@ func TestBoundedLoad_OldestLoadedSet(t *testing.T) { if store.oldestLoaded == "" { t.Fatal("oldestLoaded should be set after bounded load") } - if len(store.packets) > 0 && store.oldestLoaded != store.packets[0].FirstSeen { - t.Errorf("oldestLoaded (%s) should match first packet (%s)", store.oldestLoaded, store.packets[0].FirstSeen) + if len(store.packets) > 0 && store.oldestLoaded > store.packets[0].FirstSeen { + t.Errorf("coverage floor (%s) should include first packet (%s)", store.oldestLoaded, store.packets[0].FirstSeen) + } + oldestOmitted := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC).Add(time.Duration(5000-len(store.packets)) * time.Minute).Format(time.RFC3339) + if store.oldestLoaded <= oldestOmitted { + t.Errorf("coverage floor %s claims omitted packet %s", store.oldestLoaded, oldestOmitted) } t.Logf("oldestLoaded = %s", store.oldestLoaded) } func TestBoundedLoad_UnlimitedWithZero(t *testing.T) { dbPath := createTestDB(t, 200) - defer os.RemoveAll(filepath.Dir(dbPath)) store := loadStore(t, dbPath, 0) defer store.db.conn.Close() @@ -113,7 +107,6 @@ func TestBoundedLoad_UnlimitedWithZero(t *testing.T) { func TestBoundedLoad_AscendingOrder(t *testing.T) { dbPath := createTestDB(t, 3000) - defer os.RemoveAll(filepath.Dir(dbPath)) store := loadStore(t, dbPath, 1) defer store.db.conn.Close() @@ -130,7 +123,7 @@ func TestBoundedLoad_AscendingOrder(t *testing.T) { // loadStoreWithRetention creates a PacketStore with retentionHours set. func loadStoreWithRetention(t *testing.T, dbPath string, retentionHours float64) *PacketStore { t.Helper() - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } @@ -146,10 +139,9 @@ func loadStoreWithRetention(t *testing.T, dbPath string, retentionHours float64) // the last hour and numOld packets with timestamps 48 hours ago. func createTestDBWithAgedPackets(t *testing.T, numRecent, numOld int) string { t.Helper() - dir := t.TempDir() - dbPath := filepath.Join(dir, "test.db") + dbPath := postgresTestDSN(t) - conn, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL") + conn, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } @@ -160,10 +152,10 @@ func createTestDBWithAgedPackets(t *testing.T, numRecent, numOld int) string { t.Fatalf("setup: %v\nSQL: %s", err, s) } } - execOrFail(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT)`) - execOrFail(`CREATE TABLE observations (id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, direction TEXT, snr REAL, rssi REAL, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT)`) - execOrFail(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT)`) - execOrFail(`CREATE TABLE nodes (pubkey TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, frequency REAL)`) + execOrFail(testNativeSQL(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY AUTOINCREMENT, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT)`, `CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT)`)) + execOrFail(testNativeSQL(`CREATE TABLE observations (id INTEGER PRIMARY KEY AUTOINCREMENT, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp BIGINT, raw_hex TEXT)`, `CREATE TABLE observations (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp BIGINT, raw_hex TEXT)`)) + execOrFail(testNativeSQL(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY AUTOINCREMENT, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, `CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`)) + execOrFail(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, first_seen TEXT, frequency DOUBLE PRECISION)`) execOrFail(`CREATE TABLE schema_version (version INTEGER)`) execOrFail(`INSERT INTO schema_version (version) VALUES (1)`) execOrFail(`CREATE INDEX idx_tx_first_seen ON transmissions(first_seen)`) @@ -171,31 +163,32 @@ func createTestDBWithAgedPackets(t *testing.T, numRecent, numOld int) string { now := time.Now().UTC() id := 1 // Single transaction for all inserts — see createTestDBAt for the rationale - // (modernc.org/sqlite auto-commit per Exec fsyncs per row). numOld+numRecent + // (auto-commit per Exec fsyncs per row). numOld+numRecent // is small here today, but wrapping keeps the fixture robust if callers scale // it up, and is consistent with the other builders. - if _, err := conn.Exec("BEGIN"); err != nil { + fixtureTx, err := conn.Begin() + if err != nil { t.Fatalf("test DB BEGIN: %v", err) } // Insert old packets (48 hours ago) for i := 0; i < numOld; i++ { oldT := now.Add(-48 * time.Hour).Add(time.Duration(i) * time.Second) ts := oldT.Format(time.RFC3339) - conn.Exec("INSERT INTO transmissions VALUES (?,?,?,?,0,4,1,?)", id, "aa", fmt.Sprintf("old%d", i), ts, `{}`) + fixtureTx.Exec("INSERT INTO transmissions VALUES ($1,$2,$3,$4,0,4,1,$5)", id, "aa", fmt.Sprintf("old%d", i), ts, `{}`) // observations.timestamp is INTEGER (unix seconds) in production schema // — keep the fixture consistent so the RFC3339 subquery matches. - conn.Exec("INSERT INTO observations VALUES (?,?,?,?,?,?,?,?,?,?,?)", id, id, "obs1", "Obs1", "RX", -10.0, -80.0, 5, `[]`, oldT.Unix(), "") + fixtureTx.Exec("INSERT INTO observations VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11)", id, id, "obs1", "Obs1", "RX", -10.0, -80.0, 5, `[]`, oldT.Unix(), "") id++ } // Insert recent packets (within last hour) for i := 0; i < numRecent; i++ { newT := now.Add(-30 * time.Minute).Add(time.Duration(i) * time.Second) ts := newT.Format(time.RFC3339) - conn.Exec("INSERT INTO transmissions VALUES (?,?,?,?,0,4,1,?)", id, "bb", fmt.Sprintf("new%d", i), ts, `{}`) - conn.Exec("INSERT INTO observations VALUES (?,?,?,?,?,?,?,?,?,?,?)", id, id, "obs1", "Obs1", "RX", -10.0, -80.0, 5, `[]`, newT.Unix(), "") + fixtureTx.Exec("INSERT INTO transmissions VALUES ($1,$2,$3,$4,0,4,1,$5)", id, "bb", fmt.Sprintf("new%d", i), ts, `{}`) + fixtureTx.Exec("INSERT INTO observations VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11)", id, id, "obs1", "Obs1", "RX", -10.0, -80.0, 5, `[]`, newT.Unix(), "") id++ } - if _, err := conn.Exec("COMMIT"); err != nil { + if err := fixtureTx.Commit(); err != nil { t.Fatalf("test DB COMMIT: %v", err) } return dbPath @@ -203,7 +196,6 @@ func createTestDBWithAgedPackets(t *testing.T, numRecent, numOld int) string { func TestRetentionLoad_OnlyLoadsRecentPackets(t *testing.T) { dbPath := createTestDBWithAgedPackets(t, 50, 100) - defer os.RemoveAll(filepath.Dir(dbPath)) // retention = 2 hours — should load only the 50 recent packets, not the 100 old ones store := loadStoreWithRetention(t, dbPath, 2) @@ -216,7 +208,6 @@ func TestRetentionLoad_OnlyLoadsRecentPackets(t *testing.T) { func TestRetentionLoad_ZeroRetentionLoadsAll(t *testing.T) { dbPath := createTestDBWithAgedPackets(t, 50, 100) - defer os.RemoveAll(filepath.Dir(dbPath)) // retention = 0 (unlimited) — should load all 150 packets store := loadStoreWithRetention(t, dbPath, 0) @@ -242,13 +233,12 @@ func TestEstimateStoreTxBytesTypical(t *testing.T) { } func BenchmarkLoad_Bounded(b *testing.B) { - dir := b.TempDir() - dbPath := filepath.Join(dir, "bench.db") + dbPath := postgresTestDSN(b) createTestDBAt(b, dbPath, 5000) b.ResetTimer() for i := 0; i < b.N; i++ { - db, _ := OpenDB(dbPath) + db, _ := openFixtureReader(b, dbPath) cfg := &PacketStoreConfig{MaxMemoryMB: 1} store := NewPacketStore(db, cfg) store.Load() @@ -257,13 +247,12 @@ func BenchmarkLoad_Bounded(b *testing.B) { } func BenchmarkLoad_Unlimited(b *testing.B) { - dir := b.TempDir() - dbPath := filepath.Join(dir, "bench.db") + dbPath := postgresTestDSN(b) createTestDBAt(b, dbPath, 5000) b.ResetTimer() for i := 0; i < b.N; i++ { - db, _ := OpenDB(dbPath) + db, _ := openFixtureReader(b, dbPath) cfg := &PacketStoreConfig{MaxMemoryMB: 0} store := NewPacketStore(db, cfg) store.Load() @@ -274,13 +263,12 @@ func BenchmarkLoad_Unlimited(b *testing.B) { // BenchmarkLoad_30K_Bounded benchmarks bounded Load() with 30K transmissions // and realistic observation counts (1–5 per transmission). func BenchmarkLoad_30K_Bounded(b *testing.B) { - dir := b.TempDir() - dbPath := filepath.Join(dir, "bench30k.db") + dbPath := postgresTestDSN(b) createTestDBWithObs(b, dbPath, 30000) b.ResetTimer() for i := 0; i < b.N; i++ { - db, _ := OpenDB(dbPath) + db, _ := openFixtureReader(b, dbPath) cfg := &PacketStoreConfig{MaxMemoryMB: 50} store := NewPacketStore(db, cfg) store.Load() @@ -291,13 +279,12 @@ func BenchmarkLoad_30K_Bounded(b *testing.B) { // BenchmarkLoad_30K_Unlimited benchmarks unlimited Load() with 30K transmissions // and realistic observation counts (1–5 per transmission). func BenchmarkLoad_30K_Unlimited(b *testing.B) { - dir := b.TempDir() - dbPath := filepath.Join(dir, "bench30k.db") + dbPath := postgresTestDSN(b) createTestDBWithObs(b, dbPath, 30000) b.ResetTimer() for i := 0; i < b.N; i++ { - db, _ := OpenDB(dbPath) + db, _ := openFixtureReader(b, dbPath) cfg := &PacketStoreConfig{MaxMemoryMB: 0} store := NewPacketStore(db, cfg) store.Load() @@ -308,7 +295,7 @@ func BenchmarkLoad_30K_Unlimited(b *testing.B) { // createTestDBAt is like createTestDB but writes to a specific path. func createTestDBAt(tb testing.TB, dbPath string, numTx int) { tb.Helper() - conn, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL") + conn, err := openFixtureSQL(dbPath) if err != nil { tb.Fatal(err) } @@ -319,67 +306,52 @@ func createTestDBAt(tb testing.TB, dbPath string, numTx int) { tb.Fatalf("test DB setup exec failed: %v\nSQL: %s", err, sql) } } - execOrFail(`CREATE TABLE IF NOT EXISTS transmissions ( - id INTEGER PRIMARY KEY, + execOrFail(testNativeSQL(`CREATE TABLE IF NOT EXISTS transmissions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT - )`) - execOrFail(`CREATE TABLE IF NOT EXISTS observations ( - id INTEGER PRIMARY KEY, + )`, `CREATE TABLE IF NOT EXISTS transmissions ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + raw_hex TEXT, hash TEXT, first_seen TEXT, + route_type INTEGER, payload_type INTEGER, + payload_version INTEGER, decoded_json TEXT + )`)) + execOrFail(testNativeSQL(`CREATE TABLE IF NOT EXISTS observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, - direction TEXT, snr REAL, rssi REAL, score INTEGER, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT - )`) - execOrFail(`CREATE TABLE IF NOT EXISTS observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT)`) + )`, `CREATE TABLE IF NOT EXISTS observations ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + transmission_id INTEGER, observer_id TEXT, observer_name TEXT, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, + path_json TEXT, timestamp TEXT, raw_hex TEXT + )`)) + execOrFail(testNativeSQL(`CREATE TABLE IF NOT EXISTS observers (rowid INTEGER PRIMARY KEY AUTOINCREMENT, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, `CREATE TABLE IF NOT EXISTS observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`)) execOrFail(`CREATE TABLE IF NOT EXISTS nodes ( - pubkey TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, - last_seen TEXT, first_seen TEXT, frequency REAL + public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, + last_seen TEXT, first_seen TEXT, frequency DOUBLE PRECISION )`) execOrFail(`CREATE TABLE IF NOT EXISTS schema_version (version INTEGER)`) execOrFail(`INSERT INTO schema_version (version) VALUES (1)`) execOrFail(`CREATE INDEX IF NOT EXISTS idx_tx_first_seen ON transmissions(first_seen)`) - txStmt, err := conn.Prepare("INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) VALUES (?, ?, ?, ?, ?, ?, ?, ?)") - if err != nil { - tb.Fatalf("test DB prepare transmissions insert: %v", err) - } - obsStmt, err := conn.Prepare("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") - if err != nil { - tb.Fatalf("test DB prepare observations insert: %v", err) - } - defer txStmt.Close() - defer obsStmt.Close() - base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) - // Wrap the inserts in a single transaction. Without this, modernc.org/sqlite - // (pure-Go driver) auto-commits every Exec → one fsync per row → ~2N fsyncs - // for N transmissions (tx + obs). At numTx=5000 that is ~10k fsyncs and the - // fixture blows past the test timeout (the #1741 hang). A single - // BEGIN/COMMIT makes the whole build one commit, finishing in well under a - // second regardless of numTx. - if _, err := conn.Exec("BEGIN"); err != nil { - tb.Fatalf("test DB BEGIN: %v", err) - } - for i := 1; i <= numTx; i++ { - ts := base.Add(time.Duration(i) * time.Minute).Format(time.RFC3339) - hash := fmt.Sprintf("h%04d", i) - if _, err := txStmt.Exec(i, "aabb", hash, ts, 0, 4, 1, fmt.Sprintf(`{"pubKey":"pk%04d"}`, i)); err != nil { - tb.Fatalf("test DB insert transmission %d: %v", i, err) - } - if _, err := obsStmt.Exec(i, i, "obs1", "Obs1", "RX", -10.0, -80.0, 5, `["aa","bb"]`, ts); err != nil { - tb.Fatalf("test DB insert observation %d: %v", i, err) - } - } - if _, err := conn.Exec("COMMIT"); err != nil { - tb.Fatalf("test DB COMMIT: %v", err) - } + copyTestRows(tb, conn, "transmissions", []string{"id", "raw_hex", "hash", "first_seen", "route_type", "payload_type", "payload_version", "decoded_json"}, numTx, func(n int) []any { + i := n + 1 + return []any{i, "aabb", fmt.Sprintf("h%04d", i), base.Add(time.Duration(i) * time.Minute).Format(time.RFC3339), 0, 4, 1, fmt.Sprintf(`{"pubKey":"pk%04d"}`, i)} + }) + copyTestRows(tb, conn, "observations", []string{"id", "transmission_id", "observer_id", "observer_name", "direction", "snr", "rssi", "score", "path_json", "timestamp"}, numTx, func(n int) []any { + i := n + 1 + return []any{i, i, "obs1", "Obs1", "RX", -10.0, -80.0, 5, `["aa","bb"]`, base.Add(time.Duration(i) * time.Minute).Format(time.RFC3339)} + }) } // createTestDBWithObs creates a test DB with realistic observation counts (1–5 per tx). func createTestDBWithObs(tb testing.TB, dbPath string, numTx int) { tb.Helper() - conn, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL") + conn, err := openFixtureSQL(dbPath) if err != nil { tb.Fatal(err) } @@ -390,61 +362,48 @@ func createTestDBWithObs(tb testing.TB, dbPath string, numTx int) { tb.Fatalf("test DB setup exec failed: %v\nSQL: %s", err, sqlStr) } } - execOrFail(`CREATE TABLE IF NOT EXISTS transmissions ( - id INTEGER PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, + execOrFail(testNativeSQL(`CREATE TABLE IF NOT EXISTS transmissions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT - )`) - execOrFail(`CREATE TABLE IF NOT EXISTS observations ( - id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, - direction TEXT, snr REAL, rssi REAL, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT - )`) - execOrFail(`CREATE TABLE IF NOT EXISTS observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT)`) + )`, `CREATE TABLE IF NOT EXISTS transmissions ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, + route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT + )`)) + execOrFail(testNativeSQL(`CREATE TABLE IF NOT EXISTS observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT + )`, `CREATE TABLE IF NOT EXISTS observations ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT + )`)) + execOrFail(testNativeSQL(`CREATE TABLE IF NOT EXISTS observers (rowid INTEGER PRIMARY KEY AUTOINCREMENT, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, `CREATE TABLE IF NOT EXISTS observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`)) execOrFail(`CREATE TABLE IF NOT EXISTS nodes ( - pubkey TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, - last_seen TEXT, first_seen TEXT, frequency REAL + public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, + last_seen TEXT, first_seen TEXT, frequency DOUBLE PRECISION )`) execOrFail(`CREATE TABLE IF NOT EXISTS schema_version (version INTEGER)`) execOrFail(`INSERT INTO schema_version (version) VALUES (1)`) execOrFail(`CREATE INDEX IF NOT EXISTS idx_tx_first_seen ON transmissions(first_seen)`) - txStmt, err := conn.Prepare("INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) VALUES (?, ?, ?, ?, ?, ?, ?, ?)") - if err != nil { - tb.Fatalf("test DB prepare transmissions: %v", err) - } - obsStmt, err := conn.Prepare("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") - if err != nil { - tb.Fatalf("test DB prepare observations: %v", err) - } - defer txStmt.Close() - defer obsStmt.Close() - - observers := []string{"obs1", "obs2", "obs3", "obs4", "obs5"} - obsNames := []string{"Alpha", "Bravo", "Charlie", "Delta", "Echo"} - obsID := 1 base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) - // Single transaction for all inserts — see createTestDBAt for the rationale - // (modernc.org/sqlite auto-commit per Exec would fsync per row; at numTx=30000 - // the benchmarks would otherwise stall for minutes). One BEGIN/COMMIT. - if _, err := conn.Exec("BEGIN"); err != nil { - tb.Fatalf("test DB BEGIN: %v", err) - } + copyTestRows(tb, conn, "transmissions", []string{"id", "raw_hex", "hash", "first_seen", "route_type", "payload_type", "payload_version", "decoded_json"}, numTx, func(n int) []any { + i := n + 1 + return []any{i, "aabb", fmt.Sprintf("h%06d", i), base.Add(time.Duration(i) * time.Minute).Format(time.RFC3339), 0, 4, 1, fmt.Sprintf(`{"pubKey":"pk%06d"}`, i)} + }) + observers := []string{"obs1", "obs2", "obs3", "obs4", "obs5"} + names := []string{"Alpha", "Bravo", "Charlie", "Delta", "Echo"} + totalObs := 0 for i := 1; i <= numTx; i++ { - ts := base.Add(time.Duration(i) * time.Minute).Format(time.RFC3339) - hash := fmt.Sprintf("h%06d", i) - if _, err := txStmt.Exec(i, "aabb", hash, ts, 0, 4, 1, fmt.Sprintf(`{"pubKey":"pk%06d"}`, i)); err != nil { - tb.Fatalf("test DB insert transmission %d: %v", i, err) - } - nObs := (i % 5) + 1 // 1–5 observations per transmission - for j := 0; j < nObs; j++ { - snr := -5.0 + float64(j)*2.5 - rssi := -90.0 + float64(j)*5.0 - if _, err := obsStmt.Exec(obsID, i, observers[j], obsNames[j], "RX", snr, rssi, 5-j, `["aa","bb"]`, ts); err != nil { - tb.Fatalf("test DB insert observation %d: %v", obsID, err) - } - obsID++ + totalObs += i%5 + 1 + } + txID, j := 1, 0 + copyTestRows(tb, conn, "observations", []string{"id", "transmission_id", "observer_id", "observer_name", "direction", "snr", "rssi", "score", "path_json", "timestamp"}, totalObs, func(n int) []any { + row := []any{n + 1, txID, observers[j], names[j], "RX", -5.0 + float64(j)*2.5, -90.0 + float64(j)*5.0, 5 - j, `["aa","bb"]`, base.Add(time.Duration(txID) * time.Minute).Format(time.RFC3339)} + j++ + if j > txID%5 { + txID++ + j = 0 } - } - if _, err := conn.Exec("COMMIT"); err != nil { - tb.Fatalf("test DB COMMIT: %v", err) - } + return row + }) } diff --git a/cmd/server/bridge_handle_nodes_test.go b/cmd/server/bridge_handle_nodes_test.go index b88b7b1b6..b08376f45 100644 --- a/cmd/server/bridge_handle_nodes_test.go +++ b/cmd/server/bridge_handle_nodes_test.go @@ -34,7 +34,7 @@ func TestBridgeScore_HandleNodesSurface(t *testing.T) { for _, pk := range pks { if _, err := db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, ?, 'repeater', 37.5, -122.0, ?, ?, 10)`, + VALUES ($1, $2, 'repeater', 37.5, -122.0, $3, $4, 10)`, pk, "node-"+pk[:4], recent, recent); err != nil { t.Fatal(err) } diff --git a/cmd/server/cache_refresh.go b/cmd/server/cache_refresh.go new file mode 100644 index 000000000..f8d410ddb --- /dev/null +++ b/cmd/server/cache_refresh.go @@ -0,0 +1,50 @@ +package main + +import "sync" + +// cacheRefresher runs at most one background refresh per key. The node, +// region-observer and area-node caches serve their stale value while it runs, +// so callers holding PacketStore.mu never wait on SQL (#2146). The zero value +// is ready to use. +type cacheRefresher struct { + mu sync.Mutex + inFlight map[string]bool +} + +// start runs fn in a new goroutine unless a refresh for key is already +// running. +func (r *cacheRefresher) start(key string, fn func()) { + r.mu.Lock() + if r.inFlight[key] { + r.mu.Unlock() + return + } + if r.inFlight == nil { + r.inFlight = make(map[string]bool) + } + r.inFlight[key] = true + r.mu.Unlock() + + go func() { + defer func() { + r.mu.Lock() + delete(r.inFlight, key) + r.mu.Unlock() + }() + fn() + }() +} + +func (r *cacheRefresher) running(key string) bool { + r.mu.Lock() + defer r.mu.Unlock() + return r.inFlight[key] +} + +// beforeCacheLoad calls the test hook, if any, right before a cache runs its +// SQL. +func (s *PacketStore) beforeCacheLoad(kind string) { + if s.cacheLoadHook != nil { + s.cacheLoadHook(kind) + } +} diff --git a/cmd/server/cache_swr_test.go b/cmd/server/cache_swr_test.go new file mode 100644 index 000000000..828098f9d --- /dev/null +++ b/cmd/server/cache_swr_test.go @@ -0,0 +1,240 @@ +package main + +import ( + "sync" + "sync/atomic" + "testing" + "time" +) + +// #2146: the node, region-observer and area-node caches are read by callers +// that hold s.mu. A cache past its TTL must be served as-is while a single +// background refresh runs the SQL, so no caller holding s.mu waits on the +// 4-connection read pool. + +// blockingLoadHook returns a cacheLoadHook that counts calls per kind and +// blocks every call for kind until release is closed. +func blockingLoadHook(kind string) (hook func(string), calls *int64, entered chan struct{}, release chan struct{}) { + calls = new(int64) + entered = make(chan struct{}, 64) + release = make(chan struct{}) + hook = func(k string) { + if k != kind { + return + } + atomic.AddInt64(calls, 1) + entered <- struct{}{} + <-release + } + return hook, calls, entered, release +} + +// returnsWithin runs fn and fails the test if it does not return within d. +func returnsWithin(t *testing.T, d time.Duration, what string, fn func()) { + t.Helper() + done := make(chan struct{}) + go func() { fn(); close(done) }() + select { + case <-done: + case <-time.After(d): + t.Fatalf("%s blocked for more than %v", what, d) + } +} + +func waitUntil(t *testing.T, what string, cond func() bool) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for !cond() { + if time.Now().After(deadline) { + t.Fatalf("timed out waiting for %s", what) + } + time.Sleep(5 * time.Millisecond) + } +} + +func TestNodeCacheServesStaleWhileRefreshing(t *testing.T) { + db := setupTestDB(t) + defer db.Close() + seedTestData(t, db) + s := &PacketStore{db: db} + + stale := []nodeInfo{{PublicKey: "stale0001", Name: "Stale"}} + s.nodeCache, s.nodePM = stale, buildPrefixMap(stale) + s.nodeCacheTime = time.Now().Add(-time.Hour) + + hook, calls, entered, release := blockingLoadHook("nodes") + s.cacheLoadHook = hook + + var got []nodeInfo + returnsWithin(t, time.Second, "stale getCachedNodesAndPM", func() { + got, _ = s.getCachedNodesAndPM() + }) + if len(got) != 1 || got[0].PublicKey != "stale0001" { + t.Fatalf("want the stale cache served, got %v", got) + } + <-entered // the background refresh reached the SQL + + // Further stale callers while the refresh is in flight: served from the + // stale cache, and they do not start a second refresh. + var wg sync.WaitGroup + for i := 0; i < 10; i++ { + wg.Add(1) + go func() { defer wg.Done(); s.getCachedNodesAndPM() }() + } + returnsWithin(t, time.Second, "concurrent stale callers", wg.Wait) + if n := atomic.LoadInt64(calls); n != 1 { + t.Fatalf("want exactly 1 background refresh, got %d", n) + } + + close(release) + waitUntil(t, "refreshed node cache", func() bool { + nodes, _ := s.getCachedNodesAndPM() + return len(nodes) > 0 && nodes[0].PublicKey != "stale0001" + }) +} + +func TestNodeCacheInvalidatedBuildsInline(t *testing.T) { + db := setupTestDB(t) + defer db.Close() + seedTestData(t, db) + s := &PacketStore{db: db} + + stale := []nodeInfo{{PublicKey: "stale0001", Name: "Stale"}} + s.nodeCache, s.nodePM = stale, buildPrefixMap(stale) + s.nodeCacheTime = time.Now() + s.InvalidateNodeCache() + + var calls int64 + s.cacheLoadHook = func(k string) { + if k == "nodes" { + atomic.AddInt64(&calls, 1) + } + } + nodes, _ := s.getCachedNodesAndPM() + if atomic.LoadInt64(&calls) != 1 { + t.Fatalf("an invalidated cache must rebuild inline, hook calls = %d", calls) + } + for _, n := range nodes { + if n.PublicKey == "stale0001" { + t.Fatal("an invalidated cache must not serve the old node list") + } + } + if len(nodes) == 0 { + t.Fatal("want the seeded nodes after an inline rebuild") + } +} + +// A refresh that started before InvalidateNodeCache read the DB before the +// change the invalidation announces, so it must not mark the cache fresh. +func TestNodeCacheRefreshDoesNotOverrideInvalidation(t *testing.T) { + db := setupTestDB(t) + defer db.Close() + seedTestData(t, db) + s := &PacketStore{db: db} + + stale := []nodeInfo{{PublicKey: "stale0001", Name: "Stale"}} + s.nodeCache, s.nodePM = stale, buildPrefixMap(stale) + s.nodeCacheTime = time.Now().Add(-time.Hour) + + hook, _, entered, release := blockingLoadHook("nodes") + s.cacheLoadHook = hook + s.getCachedNodesAndPM() + <-entered + + s.InvalidateNodeCache() + close(release) + waitUntil(t, "background refresh to finish", func() bool { return !s.cacheRefresh.running("nodes") }) + + s.cacheMu.Lock() + builtAt := s.nodeCacheTime + s.cacheMu.Unlock() + if !builtAt.IsZero() { + t.Fatal("a refresh that began before InvalidateNodeCache marked the cache fresh") + } +} + +func TestRegionObsServesStaleWhileRefreshing(t *testing.T) { + db := setupTestDB(t) + defer db.Close() + seedTestData(t, db) + s := &PacketStore{db: db} + + s.regionObsCache = map[string]map[string]bool{"SJC": {"stale-obs": true}} + s.regionObsCacheTime = time.Now().Add(-time.Hour) + + hook, calls, entered, release := blockingLoadHook("regionObs") + s.cacheLoadHook = hook + + var got map[string]bool + returnsWithin(t, time.Second, "stale resolveRegionObservers", func() { + got = s.resolveRegionObservers("SJC") + }) + if !got["stale-obs"] { + t.Fatalf("want the stale region set served, got %v", got) + } + <-entered + returnsWithin(t, time.Second, "second stale caller", func() { s.resolveRegionObservers("SJC") }) + if n := atomic.LoadInt64(calls); n != 1 { + t.Fatalf("want exactly 1 background refresh, got %d", n) + } + + close(release) + waitUntil(t, "refreshed region cache", func() bool { + m := s.resolveRegionObservers("SJC") + return len(m) > 0 && !m["stale-obs"] + }) +} + +// A cold region still queries inline, but it must not hold regionObsMu while +// it does: other regions are served from the cache meanwhile. +func TestRegionObsColdMissDoesNotBlockCachedRegion(t *testing.T) { + db := setupTestDB(t) + defer db.Close() + seedTestData(t, db) + s := &PacketStore{db: db} + + s.regionObsCache = map[string]map[string]bool{"SJC": {"obs-sjc": true}} + s.regionObsCacheTime = time.Now() + + hook, _, entered, release := blockingLoadHook("regionObs") + s.cacheLoadHook = hook + go s.resolveRegionObservers("LAX") + <-entered + + returnsWithin(t, time.Second, "cached region during a cold miss", func() { + if !s.resolveRegionObservers("SJC")["obs-sjc"] { + t.Error("want the cached SJC set") + } + }) + close(release) +} + +func TestAreaNodesServesStaleWhileRefreshing(t *testing.T) { + db := setupTestDBv2(t) + mustExecDB(t, db, `INSERT INTO nodes (public_key, lat, lon) VALUES ('pk-fresh', 50.85, 4.35)`) + cfg := &Config{Areas: map[string]AreaEntry{ + "BEL": {Label: "Belgium", Polygon: [][2]float64{{50.0, 2.5}, {51.5, 2.5}, {51.5, 6.4}, {50.0, 6.4}}}, + }} + s := newTestStoreWithDB(t, db, cfg) + s.areaNodeCache["BEL"] = map[string]bool{"pk-stale": true} + s.areaNodeCacheTimes["BEL"] = time.Now().Add(-time.Hour) + + hook, calls, entered, release := blockingLoadHook("areaNodes") + s.cacheLoadHook = hook + + var got map[string]bool + returnsWithin(t, time.Second, "stale resolveAreaNodes", func() { + got = s.resolveAreaNodes("BEL") + }) + if !got["pk-stale"] { + t.Fatalf("want the stale area set served, got %v", got) + } + <-entered + returnsWithin(t, time.Second, "second stale caller", func() { s.resolveAreaNodes("BEL") }) + if n := atomic.LoadInt64(calls); n != 1 { + t.Fatalf("want exactly 1 background refresh, got %d", n) + } + + close(release) + waitUntil(t, "refreshed area cache", func() bool { return s.resolveAreaNodes("BEL")["pk-fresh"] }) +} diff --git a/cmd/server/canadaverse_theme_test.go b/cmd/server/canadaverse_theme_test.go new file mode 100644 index 000000000..8792817e6 --- /dev/null +++ b/cmd/server/canadaverse_theme_test.go @@ -0,0 +1,62 @@ +package main + +import ( + "encoding/json" + "net/http/httptest" + "os" + "strings" + "testing" +) + +// The theme handler needs no database: branding must not affect data access. +func TestCanadaverseThemeDefaultsAndOverrides(t *testing.T) { + t.Chdir(t.TempDir()) + srv := &Server{cfg: &Config{}} + type themeResult struct { + Branding map[string]string `json:"branding"` + Theme map[string]string `json:"theme"` + ThemeDark map[string]string `json:"themeDark"` + Home struct { + HeroTitle string `json:"heroTitle"` + Steps []struct { + Description string `json:"description"` + } `json:"steps"` + } `json:"home"` + } + read := func() themeResult { + t.Helper() + w := httptest.NewRecorder() + srv.handleConfigTheme(w, httptest.NewRequest("GET", "/api/config/theme", nil)) + var got themeResult + if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil { + t.Fatal(err) + } + return got + } + got := read() + if got.Branding["siteName"] != "Canadaverse CoreScope" || got.Home.HeroTitle != "Canadaverse CoreScope" { + t.Fatalf("unexpected default identity: branding=%v homeTitle=%v", got.Branding, got.Home.HeroTitle) + } + if got.Theme["background"] != "#f3f8f5" || got.ThemeDark["background"] != "#020706" || got.ThemeDark["accent"] != "#18b7ff" { + t.Fatalf("unexpected light/dark palette: light=%v dark=%v", got.Theme["background"], got.ThemeDark) + } + if len(got.Home.Steps) < 2 || !strings.Contains(got.Home.Steps[1].Description, "local mesh community") || strings.Contains(got.Home.Steps[1].Description, "910.525") { + t.Fatal("onboarding must defer to the operator's local radio preset") + } + // Operator config still overrides defaults; theme.json still wins over config. + srv.cfg.Branding = map[string]interface{}{"siteName": "Operator mesh", "logoUrl": "/operator.svg"} + srv.cfg.Theme = map[string]interface{}{"accent": "#112233"} + srv.cfg.ThemeDark = map[string]interface{}{"background": "#123456"} + srv.cfg.Home = map[string]interface{}{"heroTitle": "Operator dashboard"} + got = read() + if got.Branding["siteName"] != "Operator mesh" || got.Branding["logoUrl"] != "/operator.svg" || got.Theme["accent"] != "#112233" || got.ThemeDark["background"] != "#123456" || got.Home.HeroTitle != "Operator dashboard" { + t.Fatal("operator settings must override brand defaults") + } + if err := os.WriteFile("theme.json", []byte(`{"branding":{"siteName":"Theme mesh"},"theme":{"accent":"#223344"}}`), 0600); err != nil { + t.Fatal(err) + } + got = read() + if got.Branding["siteName"] != "Theme mesh" || got.Theme["accent"] != "#223344" || got.ThemeDark["background"] != "#123456" { + t.Fatal("theme.json must keep its existing precedence") + } +} diff --git a/cmd/server/channel_message_hash_size_test.go b/cmd/server/channel_message_hash_size_test.go new file mode 100644 index 000000000..c4226915a --- /dev/null +++ b/cmd/server/channel_message_hash_size_test.go @@ -0,0 +1,132 @@ +package main + +import ( + "encoding/json" + "testing" + "time" +) + +// The Channels view shows the path hash size (1-3 bytes) each message was +// sent with, as path_hash_size (not hash_size: nodes already use that key for +// the size a node is observed to use). /api/channels/{hash}/messages is served +// by the DB query when a DB is attached and by the in-memory store otherwise, +// and the DB query has a v3 and a pre-v3 shape, so all three must carry it. +// 0 = the packet does not encode one (see packetpath.HashSize). + +var chHashSizeWant = map[string]float64{ + "dddddddddddddd01": 1, // flood, 1-byte, 2 hops + "dddddddddddddd02": 2, // flood, 2-byte, heard direct (0 hops) + "dddddddddddddd03": 3, // transport flood, 3-byte, 1 hop + "dddddddddddddd04": 0, // direct zero-hop: no size encoded +} + +func setupChannelHashSizeDB(t *testing.T) *DB { + t.Helper() + return seedChannelHashSize(t, setupTestDB(t)) +} + +// seedChannelHashSize writes one channel message per chHashSizeWant entry, each +// heard by one observation, in the observation shape of db's schema. +func seedChannelHashSize(t *testing.T, db *DB) *DB { + t.Helper() + if _, err := db.conn.Exec(`INSERT INTO observers (id, name, iata) VALUES ('obs1', 'Observer One', 'BRU')`); err != nil { + t.Fatalf("insert observer: %v", err) + } + now := time.Now().UTC() + rows := []struct { + hash, rawHex string + routeType int + }{ + {"dddddddddddddd01", "1502AABBDEADBEEF", 1}, + {"dddddddddddddd02", "1540DEADBEEF", 1}, + {"dddddddddddddd03", "141122334481AABBCCDEADBEEF", 0}, + {"dddddddddddddd04", "1600DEADBEEF", 2}, + } + for i, r := range rows { + ts := now.Add(time.Duration(i-len(rows)) * time.Minute) + var txID int64 + err := db.conn.QueryRow(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash) + VALUES ($1, $2, $3, $4, 5, '{"type":"CHAN","channel":"#hashsize","text":"Alice: msg"}', '#hashsize') RETURNING id`, + r.rawHex, r.hash, ts.Format(time.RFC3339), r.routeType).Scan(&txID) + if err != nil { + t.Fatalf("insert tx %s: %v", r.hash, err) + } + + obsSQL := `INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) + VALUES ($1, 1, 9.5, -90, '[]', $2)` + if !db.isV3 { + obsSQL = `INSERT INTO observations (transmission_id, observer_id, observer_name, snr, rssi, path_json, timestamp) + VALUES ($1, 'obs1', 'Observer One', 9.5, -90, '[]', $2)` + } + if _, err := db.conn.Exec(obsSQL, txID, ts.Unix()); err != nil { + t.Fatalf("insert obs %s: %v", r.hash, err) + } + } + return db +} + +func assertChannelMessageHashSizes(t *testing.T, messages []map[string]interface{}) { + t.Helper() + if len(messages) != len(chHashSizeWant) { + t.Fatalf("expected %d messages, got %d", len(chHashSizeWant), len(messages)) + } + for _, m := range messages { + // Compare on what the browser receives, not on the Go value type. + b, err := json.Marshal(m) + if err != nil { + t.Fatalf("marshal: %v", err) + } + var decoded map[string]interface{} + if err := json.Unmarshal(b, &decoded); err != nil { + t.Fatalf("unmarshal: %v", err) + } + h, _ := decoded["packetHash"].(string) + want, known := chHashSizeWant[h] + if !known { + t.Errorf("unexpected packetHash %q", h) + continue + } + got, present := decoded["path_hash_size"] + if !present { + t.Errorf("%s: path_hash_size key missing", h) + continue + } + if got != want { + t.Errorf("%s: path_hash_size = %#v, want %v", h, got, want) + } + } +} + +func TestDBGetChannelMessagesCarriesHashSize(t *testing.T) { + db := setupChannelHashSizeDB(t) + defer db.Close() + messages, _, err := db.GetChannelMessages("#hashsize", 100, 0) + if err != nil { + t.Fatal(err) + } + assertChannelMessageHashSizes(t, messages) +} + +func TestDBGetChannelMessagesCarriesHashSizeV2Schema(t *testing.T) { + db := seedChannelHashSize(t, setupTestDBv2(t)) + defer db.Close() + if db.isV3 { + t.Fatal("precondition: setupTestDBv2 must give the pre-v3 schema") + } + messages, _, err := db.GetChannelMessages("#hashsize", 100, 0) + if err != nil { + t.Fatal(err) + } + assertChannelMessageHashSizes(t, messages) +} + +func TestStoreGetChannelMessagesCarriesHashSize(t *testing.T) { + db := setupChannelHashSizeDB(t) + defer db.Close() + store := NewPacketStore(db, nil) + if err := store.Load(); err != nil { + t.Fatalf("store.Load: %v", err) + } + messages, _ := store.GetChannelMessages("#hashsize", 100, 0) + assertChannelMessageHashSizes(t, messages) +} diff --git a/cmd/server/channel_message_scope_name_test.go b/cmd/server/channel_message_scope_name_test.go new file mode 100644 index 000000000..ab834ee5e --- /dev/null +++ b/cmd/server/channel_message_scope_name_test.go @@ -0,0 +1,262 @@ +package main + +import ( + "encoding/json" + "net/http/httptest" + "testing" + "time" + + "github.com/gorilla/mux" +) + +// #1851: the Channels view shows the region scope a message was sent under. +// transmissions.scope_name encodes three states (see StoreTx.ScopeName): +// SQL NULL = not transport-scoped, "" = transport-scoped but no configured +// region key matched, "#name" = matched region. /api/channels/{hash}/messages +// is served by the DB query when a DB is attached and by the in-memory store +// otherwise, and live messages arrive over the WebSocket broadcast, so all +// three paths must carry the same three states. + +const ( + chScopeHashUnscoped = "cccccccccccccc01" + chScopeHashUnknown = "cccccccccccccc02" + chScopeHashMatched = "cccccccccccccc03" +) + +var chScopeWant = map[string]interface{}{ + chScopeHashUnscoped: nil, + chScopeHashUnknown: "", + chScopeHashMatched: "#belgium", +} + +// setupChannelScopeDB seeds one channel with three messages, one per scope +// state, each heard by one observation. +func setupChannelScopeDB(t *testing.T) *DB { + t.Helper() + db := setupTestDB(t) + if _, err := db.conn.Exec(`ALTER TABLE transmissions ADD COLUMN scope_name TEXT DEFAULT NULL`); err != nil { + t.Fatalf("add scope_name column: %v", err) + } + db.hasScopeName = true + if _, err := db.conn.Exec(`INSERT INTO observers (id, name, iata) VALUES ('obs1', 'Observer One', 'BRU')`); err != nil { + t.Fatalf("insert observer: %v", err) + } + now := time.Now().UTC() + rows := []struct { + hash string + routeType int + scope interface{} + text string + }{ + {chScopeHashUnscoped, 1, nil, "Alice: plain flood"}, + {chScopeHashUnknown, 0, "", "Bob: unknown region"}, + {chScopeHashMatched, 0, "#belgium", "Carol: matched region"}, + } + for i, r := range rows { + ts := now.Add(time.Duration(i-3) * time.Minute) + var txID int64 + err := db.conn.QueryRow(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash, scope_name) + VALUES ('AABB', $1, $2, $3, 5, $4, '#scopetest', $5) RETURNING id`, + r.hash, ts.Format(time.RFC3339), r.routeType, + `{"type":"CHAN","channel":"#scopetest","text":"`+r.text+`"}`, r.scope).Scan(&txID) + if err != nil { + t.Fatalf("insert tx %s: %v", r.hash, err) + } + + if _, err := db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) + VALUES ($1, 1, 9.5, -90, '[]', $2)`, txID, ts.Unix()); err != nil { + t.Fatalf("insert obs %s: %v", r.hash, err) + } + } + return db +} + +func assertChannelMessageScopes(t *testing.T, messages []map[string]interface{}) { + t.Helper() + if len(messages) != len(chScopeWant) { + t.Fatalf("expected %d messages, got %d", len(chScopeWant), len(messages)) + } + for _, m := range messages { + h, _ := m["packetHash"].(string) + want, known := chScopeWant[h] + if !known { + t.Errorf("unexpected packetHash %q", h) + continue + } + got, present := m["scope_name"] + if !present { + t.Errorf("%s: scope_name key missing", h) + continue + } + if got != want { + t.Errorf("%s: scope_name = %#v, want %#v", h, got, want) + } + } +} + +func TestDBGetChannelMessagesCarriesScopeName(t *testing.T) { + db := setupChannelScopeDB(t) + defer db.Close() + messages, _, err := db.GetChannelMessages("#scopetest", 100, 0) + if err != nil { + t.Fatal(err) + } + assertChannelMessageScopes(t, messages) +} + +func TestStoreGetChannelMessagesCarriesScopeName(t *testing.T) { + db := setupChannelScopeDB(t) + defer db.Close() + store := NewPacketStore(db, nil) + if err := store.Load(); err != nil { + t.Fatalf("store.Load: %v", err) + } + messages, _ := store.GetChannelMessages("#scopetest", 100, 0) + for i := range messages { + // The store keeps the value as a *string; normalise through JSON so + // the comparison is on what the browser receives. + b, err := json.Marshal(messages[i]) + if err != nil { + t.Fatalf("marshal: %v", err) + } + var decoded map[string]interface{} + if err := json.Unmarshal(b, &decoded); err != nil { + t.Fatalf("unmarshal: %v", err) + } + messages[i] = decoded + } + assertChannelMessageScopes(t, messages) +} + +// A database the ingestor has not migrated yet has no scope_name column. The +// DB query must not select it (the query would fail) and every message reports +// null, the same as a non-transport-scoped message. +func TestDBGetChannelMessagesWithoutScopeNameColumn(t *testing.T) { + db := setupTestDB(t) + defer db.Close() + seedTestData(t, db) + messages, _, err := db.GetChannelMessages("#test", 100, 0) + if err != nil { + t.Fatalf("GetChannelMessages on a schema without scope_name: %v", err) + } + if len(messages) == 0 { + t.Fatal("expected messages for #test") + } + for _, m := range messages { + if got := m["scope_name"]; got != nil { + t.Errorf("scope_name = %#v, want nil without the column", got) + } + } +} + +func TestChannelMessagesEndpointExposesScopeName(t *testing.T) { + db := setupChannelScopeDB(t) + defer db.Close() + for _, withDB := range []bool{true, false} { + name := "db" + if !withDB { + name = "store" + } + t.Run(name, func(t *testing.T) { + store := NewPacketStore(db, nil) + if err := store.Load(); err != nil { + t.Fatalf("store.Load: %v", err) + } + srv := NewServer(db, &Config{Port: 3000}, NewHub()) + srv.store = store + if !withDB { + srv.db = nil + } + router := mux.NewRouter() + srv.RegisterRoutes(router) + + req := httptest.NewRequest("GET", "/api/channels/%23scopetest/messages?limit=50", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != 200 { + t.Fatalf("expected 200, got %d (body: %s)", w.Code, w.Body.String()) + } + var body struct { + Messages []map[string]interface{} `json:"messages"` + } + if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil { + t.Fatalf("unmarshal: %v", err) + } + assertChannelMessageScopes(t, body.Messages) + }) + } +} + +// Live messages in the Channels view are built from the WebSocket broadcast, +// not from the REST response, so both broadcast builders must carry scope_name +// or a new message shows no region until the next REST refresh replaces it. +func TestBroadcastMapsCarryScopeName(t *testing.T) { + db := setupChannelScopeDB(t) + defer db.Close() + store := NewPacketStore(db, nil) + if err := store.Load(); err != nil { + t.Fatalf("store.Load: %v", err) + } + + assertBroadcast := func(t *testing.T, maps []map[string]interface{}, hash string, want interface{}) { + t.Helper() + found := false + for _, bm := range maps { + if bm["hash"] != hash { + continue + } + found = true + b, err := json.Marshal(bm) + if err != nil { + t.Fatalf("marshal: %v", err) + } + var decoded map[string]interface{} + if err := json.Unmarshal(b, &decoded); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if got, ok := decoded["scope_name"]; !ok || got != want { + t.Errorf("top-level scope_name = %#v (present=%v), want %#v", got, ok, want) + } + pkt, _ := decoded["packet"].(map[string]interface{}) + if got, ok := pkt["scope_name"]; !ok || got != want { + t.Errorf("packet.scope_name = %#v (present=%v), want %#v", got, ok, want) + } + } + if !found { + t.Fatalf("no broadcast map for %s", hash) + } + } + + t.Run("IngestNewFromDB", func(t *testing.T) { + txMax := store.MaxTransmissionID() + now := time.Now().UTC() + var txID int64 + err := db.conn.QueryRow(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash, scope_name) + VALUES ('AABB', 'cccccccccccccc04', $1, 0, 5, '{"type":"CHAN","channel":"#scopetest","text":"Dave: live"}', '#scopetest', '#belgium') RETURNING id`, + now.Format(time.RFC3339)).Scan(&txID) + if err != nil { + t.Fatalf("insert tx: %v", err) + } + + if _, err := db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) + VALUES ($1, 1, 7.0, -95, '[]', $2)`, txID, now.Unix()); err != nil { + t.Fatalf("insert obs: %v", err) + } + maps, _ := store.IngestNewFromDB(txMax, 100) + assertBroadcast(t, maps, "cccccccccccccc04", "#belgium") + }) + + t.Run("IngestNewObservations", func(t *testing.T) { + obsMax := db.GetMaxObservationID() + var txID int + if err := db.conn.QueryRow(`SELECT id FROM transmissions WHERE hash = $1`, chScopeHashUnknown).Scan(&txID); err != nil { + t.Fatalf("lookup tx: %v", err) + } + if _, err := db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) + VALUES ($1, 1, 3.0, -110, '["aa"]', $2)`, txID, time.Now().Unix()); err != nil { + t.Fatalf("insert obs: %v", err) + } + maps := store.IngestNewObservations(obsMax, 100) + assertBroadcast(t, maps, chScopeHashUnknown, "") + }) +} diff --git a/cmd/server/channels_message_order_1366_test.go b/cmd/server/channels_message_order_1366_test.go index 0fde9bfbb..5f307443b 100644 --- a/cmd/server/channels_message_order_1366_test.go +++ b/cmd/server/channels_message_order_1366_test.go @@ -34,20 +34,20 @@ func TestChannelMessages_TimestampUsesLatestSeen(t *testing.T) { _ = laterEpoch db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obsA', 'ObsA', 'SJC', ?, '2026-01-01T00:00:00Z', 10)`, firstSeen) + VALUES ('obsA', 'ObsA', 'SJC', $1, '2026-01-01T00:00:00Z', 10)`, firstSeen) db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obsB', 'ObsB', 'LAX', ?, '2026-01-01T00:00:00Z', 10)`, firstSeen) + VALUES ('obsB', 'ObsB', 'LAX', $1, '2026-01-01T00:00:00Z', 10)`, firstSeen) // One transmission with two observations: T0 (7h ago) and T1 (5m ago). db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash) - VALUES ('AA01', 'hash_repeated_msg', ?, 1, 5, + VALUES ('AA01', 'hash_repeated_msg', $1, 1, 5, '{"type":"CHAN","channel":"#test","text":"Heartbeat: ping","sender":"Heartbeat","sender_timestamp":`+ strconv.FormatInt(firstSeenEpoch, 10)+`}', '#test')`, firstSeen) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 10.0, -90, '["aa"]', ?)`, firstSeenEpoch) + VALUES (1, 1, 10.0, -90, '["aa"]', $1)`, firstSeenEpoch) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 2, 11.0, -88, '["bb"]', ?)`, laterEpoch) + VALUES (1, 2, 11.0, -88, '["bb"]', $1)`, laterEpoch) store := NewPacketStore(db, nil) store.Load() @@ -102,14 +102,14 @@ func TestChannelMessages_TimestampNotSenderTimestamp(t *testing.T) { badSenderTs := int64(946684800) // 2000-01-01 UTC db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obsX', 'ObsX', 'SJC', ?, '2026-01-01T00:00:00Z', 1)`, firstSeen) + VALUES ('obsX', 'ObsX', 'SJC', $1, '2026-01-01T00:00:00Z', 1)`, firstSeen) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash) - VALUES ('BB01', 'hash_bad_clock', ?, 1, 5, + VALUES ('BB01', 'hash_bad_clock', $1, 1, 5, '{"type":"CHAN","channel":"#bad","text":"Alice: ping","sender":"Alice","sender_timestamp":`+ strconv.FormatInt(badSenderTs, 10)+`}', '#bad')`, firstSeen) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 10.0, -90, '["aa"]', ?)`, firstSeenEpoch) + VALUES (1, 1, 10.0, -90, '["aa"]', $1)`, firstSeenEpoch) store := NewPacketStore(db, nil) store.Load() @@ -143,13 +143,13 @@ func TestChannelMessages_TimestampIsUTCZ(t *testing.T) { ep := now.Add(-30 * time.Minute).Unix() db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obsZ', 'ObsZ', 'SJC', ?, '2026-01-01T00:00:00Z', 1)`, fs) + VALUES ('obsZ', 'ObsZ', 'SJC', $1, '2026-01-01T00:00:00Z', 1)`, fs) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash) - VALUES ('ZZ01', 'hash_zone_check', ?, 1, 5, + VALUES ('ZZ01', 'hash_zone_check', $1, 1, 5, '{"type":"CHAN","channel":"#zone","text":"Carol: ping","sender":"Carol"}', '#zone')`, fs) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 11.0, -89, '["zz"]', ?)`, ep) + VALUES (1, 1, 11.0, -89, '["zz"]', $1)`, ep) store := NewPacketStore(db, nil) store.Load() @@ -197,30 +197,30 @@ func TestChannelMessages_OrderedByLatestSeen_InMemory(t *testing.T) { tNewestStr := tNewest.Format(time.RFC3339) db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obsO', 'ObsO', 'SJC', ?, '2026-01-01T00:00:00Z', 10)`, tOldStr) + VALUES ('obsO', 'ObsO', 'SJC', $1, '2026-01-01T00:00:00Z', 10)`, tOldStr) // tx-A: FirstSeen 24h ago, LatestSeen NOW (T-1m). Old insertion order. db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash) - VALUES ('AAAA', 'order_hash_a', ?, 1, 5, + VALUES ('AAAA', 'order_hash_a', $1, 1, 5, '{"type":"CHAN","channel":"#ord","text":"Alpha: hb","sender":"Alpha"}', '#ord')`, tOldStr) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 10.0, -90, '["aa"]', ?)`, tOld.Unix()) + VALUES (1, 1, 10.0, -90, '["aa"]', $1)`, tOld.Unix()) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 11.0, -88, '["aa"]', ?)`, tFresh.Unix()) + VALUES (1, 1, 11.0, -88, '["aa"]', $1)`, tFresh.Unix()) // tx-B: FirstSeen 1h ago, LatestSeen 1h ago. OLDEST. db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash) - VALUES ('BBBB', 'order_hash_b', ?, 1, 5, + VALUES ('BBBB', 'order_hash_b', $1, 1, 5, '{"type":"CHAN","channel":"#ord","text":"Bravo: msg","sender":"Bravo"}', '#ord')`, tMidStr) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (2, 1, 9.0, -91, '["bb"]', ?)`, tMid.Unix()) + VALUES (2, 1, 9.0, -91, '["bb"]', $1)`, tMid.Unix()) // tx-C: FirstSeen 30m ago, LatestSeen 30m ago. Middle. db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash) - VALUES ('CCCC', 'order_hash_c', ?, 1, 5, + VALUES ('CCCC', 'order_hash_c', $1, 1, 5, '{"type":"CHAN","channel":"#ord","text":"Charlie: msg","sender":"Charlie"}', '#ord')`, tNewestStr) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (3, 1, 9.0, -91, '["cc"]', ?)`, tNewest.Unix()) + VALUES (3, 1, 9.0, -91, '["cc"]', $1)`, tNewest.Unix()) store := NewPacketStore(db, nil) store.Load() @@ -276,34 +276,34 @@ func TestChannelMessages_OrderedByLatestSeen_DB(t *testing.T) { tNewestStr := tNewest.Format(time.RFC3339) db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obsD', 'ObsD', 'SJC', ?, '2026-01-01T00:00:00Z', 10)`, tOldStr) + VALUES ('obsD', 'ObsD', 'SJC', $1, '2026-01-01T00:00:00Z', 10)`, tOldStr) // tx-A: FirstSeen 24h ago, observations at T-24h and T-1m (LatestSeen // = T-1m, the FRESHEST). Despite the freshest LatestSeen, a // FirstSeen-DESC selection would push it OFF a small page. db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash) - VALUES ('AADB', 'order_db_hash_a', ?, 1, 5, + VALUES ('AADB', 'order_db_hash_a', $1, 1, 5, '{"type":"CHAN","channel":"#ordb","text":"Alpha: hb","sender":"Alpha"}', '#ordb')`, tOldStr) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 10.0, -90, '["aa"]', ?)`, tOld.Unix()) + VALUES (1, 1, 10.0, -90, '["aa"]', $1)`, tOld.Unix()) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 11.0, -88, '["aa"]', ?)`, tFresh.Unix()) + VALUES (1, 1, 11.0, -88, '["aa"]', $1)`, tFresh.Unix()) // tx-B: FirstSeen 1h ago, LatestSeen 1h ago. OLDEST LatestSeen. db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash) - VALUES ('BBDB', 'order_db_hash_b', ?, 1, 5, + VALUES ('BBDB', 'order_db_hash_b', $1, 1, 5, '{"type":"CHAN","channel":"#ordb","text":"Bravo: msg","sender":"Bravo"}', '#ordb')`, tMidStr) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (2, 1, 9.0, -91, '["bb"]', ?)`, tMid.Unix()) + VALUES (2, 1, 9.0, -91, '["bb"]', $1)`, tMid.Unix()) // tx-C: FirstSeen 30m ago, LatestSeen 30m ago. Middle LatestSeen. // With FirstSeen-DESC selection + limit=2, page = [tx-C, tx-B] and // tx-A is EXCLUDED — that's the selection bug fix #2 gates. db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash) - VALUES ('CCDB', 'order_db_hash_c', ?, 1, 5, + VALUES ('CCDB', 'order_db_hash_c', $1, 1, 5, '{"type":"CHAN","channel":"#ordb","text":"Charlie: msg","sender":"Charlie"}', '#ordb')`, tNewestStr) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (3, 1, 9.0, -91, '["cc"]', ?)`, tNewest.Unix()) + VALUES (3, 1, 9.0, -91, '["cc"]', $1)`, tNewest.Unix()) msgs, total, err := db.GetChannelMessages("#ordb", 2, 0) if err != nil { diff --git a/cmd/server/channels_no_unknown_bucket_1373_test.go b/cmd/server/channels_no_unknown_bucket_1373_test.go index 3f709f399..d917e4e4b 100644 --- a/cmd/server/channels_no_unknown_bucket_1373_test.go +++ b/cmd/server/channels_no_unknown_bucket_1373_test.go @@ -67,7 +67,7 @@ func TestGetChannels_DB_NoUnknownBucket_1373(t *testing.T) { for i := 0; i < 5; i++ { _, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash) - VALUES (?, ?, '2026-05-25T12:00:00Z', 1, 5, + VALUES ($1, $2, '2026-05-25T12:00:00Z', 1, 5, '{"type":"CHAN","channel":"","text":"","sender":""}', NULL)`, "AA", sqlHashFor(i)) if err != nil { @@ -79,7 +79,7 @@ func TestGetChannels_DB_NoUnknownBucket_1373(t *testing.T) { for i := 0; i < 3; i++ { _, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash) - VALUES (?, ?, '2026-05-25T12:00:00Z', 1, 5, + VALUES ($1, $2, '2026-05-25T12:00:00Z', 1, 5, '{"type":"CHAN","channel":"#real","text":"Alice: hi","sender":"Alice"}', '#real')`, "BB", sqlHashFor(100+i)) if err != nil { diff --git a/cmd/server/chunked_load.go b/cmd/server/chunked_load.go index 258061fd0..485e579a1 100644 --- a/cmd/server/chunked_load.go +++ b/cmd/server/chunked_load.go @@ -3,9 +3,10 @@ package main // Chunked startup load + early HTTP readiness for issue #1009. // // Design: -// * LoadChunked paginates transmissions in id-ordered chunks of +// * LoadChunked paginates transmissions in bounded chunks of // `chunkSize` (default 10000 via Config.DBLoadChunkSize). After the // first chunk is merged into the store, FirstChunkReady is closed. +// Memory-capped stores prioritize newest rows; unlimited stores use IDs. // main.go binds the HTTP listener on that signal and serves // partial data while remaining chunks stream in the background. // * RunStartupLoad is the orchestrator: it runs LoadChunked @@ -22,15 +23,13 @@ package main // logging / tests. // // Concurrency: each chunk acquires s.mu.Lock() ONLY while merging the -// chunk's rows into store-shared maps. SQLite reads run lock-free so +// chunk's rows into store-shared maps. Database reads run without the store lock so // HTTP handlers (which take s.mu.RLock) stay responsive. import ( - "database/sql" "fmt" "log" "net/http" - "sort" "strings" "sync" "sync/atomic" @@ -96,9 +95,35 @@ func (s *PacketStore) OnChunkLoaded(fn func(rowsThisChunk, totalRows int)) { func (s *PacketStore) chunkedLoadInit() { s.chunkInitOnce.Do(func() { s.firstChunkReady = make(chan struct{}) + s.startupLoadDone = make(chan struct{}) }) } +// StartupLoadDone returns a channel closed once RunStartupLoad has +// returned: LoadChunked AND the background fill loader are finished, +// whether they succeeded or not. Nothing more is loaded from the database +// after it closes. LoadComplete() is not a substitute: it flips at the +// end of the hot window, before the background fill starts. +func (s *PacketStore) StartupLoadDone() <-chan struct{} { + s.chunkedLoadInit() + return s.startupLoadDone +} + +func (s *PacketStore) signalStartupLoadDone() { + s.chunkedLoadInit() + if !s.startupLoadSignaled.CompareAndSwap(false, true) { + return + } + // The analytics recomputes this signal triggers read these TTL + // caches. Drop what was computed from the partial store so they see + // the loaded data now rather than after the TTL. + s.hashSizeInfoMu.Lock() + s.hashSizeInfoCache = nil + s.hashSizeInfoMu.Unlock() + s.clockSkew.Invalidate() + close(s.startupLoadDone) +} + func (s *PacketStore) signalFirstChunk() { if s.firstChunkSignaled.CompareAndSwap(false, true) { close(s.firstChunkReady) @@ -153,6 +178,7 @@ func (s *PacketStore) fireChunkCallbacks(rowsThisChunk, totalRows int) { // LoadChunked. // // Steady-state contracts post-return: +// - StartupLoadDone() is closed, on every path below. // - LoadChunked error: backgroundLoadFailed=true, backgroundLoadDone // is also set true (terminal observable state — see dij #1). // backgroundLoadErr non-empty. Returns the error. @@ -160,8 +186,8 @@ func (s *PacketStore) fireChunkCallbacks(rowsThisChunk, totalRows int) { // (no background work was needed). Coverage is computed against // row count so /api/perf does not report ratio=0.0 (dij #2). // - hotStartupHours > 0 success: terminal state is whatever -// loadBackgroundChunks set (done=true on full coverage, -// failed=true on partial / chunk errors — see #1690). +// loadBackgroundChunks set (done=true on full coverage or the configured +// memory limit; failed=true on unexpected partial coverage/chunk errors). // // Issue #1809 root cause: previously main.go spawned loadBackgroundChunks // at FirstChunkReady while LoadChunked was still merging the remainder @@ -172,6 +198,7 @@ func (s *PacketStore) fireChunkCallbacks(rowsThisChunk, totalRows int) { // parallelism while ensuring oldestLoaded has a valid floor when the // bg loader starts. func (s *PacketStore) RunStartupLoad(chunkSize int) error { + defer s.signalStartupLoadDone() // Clear any stale error from a previous invocation (single-call // invariant — see godoc above). Production never re-enters but // test fixtures may construct fresh stores that share no state; @@ -237,8 +264,8 @@ func (s *PacketStore) RunStartupLoad(chunkSize int) error { return nil } -// LoadChunked streams transmissions + observations from SQLite into -// the in-memory store in id-ordered chunks of `chunkSize` rows. Pass +// LoadChunked streams transmissions + observations from PostgreSQL into +// the in-memory store in bounded chunks of `chunkSize` rows. Pass // 0 to use the default (10000). // // After the first chunk is merged, FirstChunkReady is closed and the @@ -299,7 +326,7 @@ func (s *PacketStore) LoadChunked(chunkSize int) error { // the prior `t2.first_seen >= cutoff` query loaded only hashes // first-inserted within the window (0.3% of DB on prod). // - // Test/legacy DBs without the column (PRAGMA-detected as + // Test/legacy DBs without the column (schema-detected as // hasLastSeen=false) fall back to the legacy first_seen axis to // keep existing fixtures green. Production goes through // dbschema.AssertReady which fail-fasts when the column is @@ -325,143 +352,47 @@ func (s *PacketStore) LoadChunked(chunkSize int) error { totalInDB = -1 } - // Memory cap honoured by clamping the maximum cursor walk. - var maxPackets int64 + // Polling starts after the first chunk. Capture the complete database's + // high-water marks before loading a subset, so older omitted rows are not + // mistaken for new ingest and replayed into the cache immediately. if s.maxMemoryMB > 0 { - avgBytes := int64(1000) - if sample := estimateStoreTxBytesTypical(10); sample > avgBytes { - avgBytes = sample - } - maxPackets = (int64(s.maxMemoryMB) * 1048576) / avgBytes - if maxPackets < 1000 { - maxPackets = 1000 + var maxTxID, maxObsID int + if err := s.db.conn.QueryRow(`SELECT + (SELECT COALESCE(MAX(id), 0) FROM transmissions), + (SELECT COALESCE(MAX(id), 0) FROM observations)`).Scan(&maxTxID, &maxObsID); err != nil { + return fmt.Errorf("load watermarks: %w", err) } + s.mu.Lock() + s.maxTxID, s.maxObsID = maxTxID, maxObsID + s.mu.Unlock() + } + var from time.Time + if hotCutoffStr != "" { + from, _ = time.Parse(time.RFC3339, hotCutoffStr) } - chunkIdx := 0 - totalLoaded := 0 - // Start the id cursor BELOW the minimum possible row id so the - // first chunk's `t2.id > cursorID` predicate includes id=0. The - // e2e fixture seed for issue #1486 inserts the grouped-packet row - // with id=0 (so it sorts LAST in the default packets view via - // `ORDER BY id DESC` / oldest first_seen). Seeding the cursor at - // 0 silently excluded that row, leaving the page with no - // tr[data-hash] and timing out the playwright wait. Legacy Load() - // had no id cursor and loaded id=0 unconditionally — we restore - // that semantic by starting one below SQLite's minimum rowid (-1). - var cursorID int64 = -1 - - // Relay-hop fallback inputs, fetched ONCE before the chunk-query loop. - // getCachedNodesAndPM issues its own DB query, so calling it while a - // chunk cursor is open would deadlock on a single-connection SQLite - // pool. resolved_path is never persisted post-#1287, so scanAndMergeChunk - // re-resolves relay hops from path_json using these snapshots. - // PR #1643 R1 munger #1: cold load uses unique_prefix-only gate, so - // the neighbor graph is no longer consulted here (affinity-tier - // resolution against ≤168h-old observations would silently mis-attribute). - s.mu.RLock() - _, relayPM := s.getCachedNodesAndPM() - s.mu.RUnlock() - var coldLoadAmbiguousHopsSkipped int - - for { - conds := append([]string{}, loadConditions...) - conds = append(conds, fmt.Sprintf("t2.id > %d", cursorID)) - whereClause := "WHERE " + strings.Join(conds, " AND ") - - rpCol := "" - if s.db.hasResolvedPath { - rpCol = ", o.resolved_path" - } - obsRawHexCol := "" - if s.db.hasObsRawHex { - obsRawHexCol = ", o.raw_hex" - } - // #1751: scope_name is on the transmission row, so appending it as the - // last selected column is safe regardless of the observation fan-out. - scopeNameCol := "" - if s.db.hasScopeName { - scopeNameCol = ", t.scope_name" - } - - var chunkSQL string - if s.db.isV3 { - chunkSQL = `SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type, - t.payload_type, t.payload_version, t.decoded_json, - o.id, obs.id, obs.name, COALESCE(obs.iata, ''), o.direction, - o.snr, o.rssi, o.score, o.path_json, strftime('%Y-%m-%dT%H:%M:%fZ', o.timestamp, 'unixepoch')` + obsRawHexCol + rpCol + scopeNameCol + ` - FROM (SELECT * FROM transmissions t2 ` + whereClause + ` ORDER BY t2.id ASC LIMIT ` + fmt.Sprintf("%d", chunkSize) + `) AS t - LEFT JOIN observations o ON o.transmission_id = t.id - LEFT JOIN observers obs ON obs.rowid = o.observer_idx - ORDER BY t.id ASC, o.timestamp DESC` - } else { - chunkSQL = `SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type, - t.payload_type, t.payload_version, t.decoded_json, - o.id, o.observer_id, o.observer_name, COALESCE(obs.iata, ''), o.direction, - o.snr, o.rssi, o.score, o.path_json, o.timestamp` + obsRawHexCol + rpCol + scopeNameCol + ` - FROM (SELECT * FROM transmissions t2 ` + whereClause + ` ORDER BY t2.id ASC LIMIT ` + fmt.Sprintf("%d", chunkSize) + `) AS t - LEFT JOIN observations o ON o.transmission_id = t.id - LEFT JOIN observers obs ON obs.id = o.observer_id - ORDER BY t.id ASC, o.timestamp DESC` - } - - rows, err := s.db.conn.Query(chunkSQL) - if err != nil { - return fmt.Errorf("chunk %d: query: %w", chunkIdx, err) - } - - chunkTxCount, lastID, err := s.scanAndMergeChunk(rows, relayPM, &coldLoadAmbiguousHopsSkipped) - rows.Close() - if err != nil { - return fmt.Errorf("chunk %d: scan: %w", chunkIdx, err) - } - - if chunkTxCount == 0 { - break - } - - cursorID = lastID - totalLoaded += chunkTxCount + totalLoaded, err := s.loadPacketRange(from, time.Time{}, chunkSize, func(count, total int) { chunkIdx++ - s.loadProgressRows.Store(int64(totalLoaded)) + s.loadProgressRows.Store(int64(total)) s.signalFirstChunk() - s.fireChunkCallbacks(chunkTxCount, totalLoaded) - - if maxPackets > 0 && int64(totalLoaded) >= maxPackets { - break - } - if chunkTxCount < chunkSize { - break - } + s.fireChunkCallbacks(count, total) + }) + if err != nil { + return fmt.Errorf("chunk %d: %w", chunkIdx, err) } - // Post-load: pick best observation, build indexes — same shape as - // legacy Load(). + // Each chunk publishes complete packets and indexes atomically in + // first_seen order. Build the derived analytics indexes once at the end. s.mu.Lock() - for _, tx := range s.packets { - pickBestObservation(tx) - s.indexByNode(tx) - } - // Restore the "s.packets sorted oldest-first by FirstSeen" invariant - // that legacy Load() got for free from "ORDER BY t.first_seen ASC". - // LoadChunked walks chunks in id-ASC order so the slice ends up - // id-ordered, which only equals first_seen-ordered when ids and - // timestamps are correlated. After tools/freshen-fixture.sh (or any - // real-world out-of-order ingest) they're not, leaving - // s.packets[0].FirstSeen pointing at the newest row — which then - // poisons oldestLoaded below and routes legitimate in-memory queries - // to the SQL fallback. GetTimestamps (store.go) and QueryPackets - // both rely on this invariant. See PR #1596 / mobile e2e regression. - sort.SliceStable(s.packets, func(i, j int) bool { - return s.packets[i].FirstSeen < s.packets[j].FirstSeen - }) s.buildSubpathIndex() s.buildPathHopIndex() s.buildDistanceIndex() - if s.hotStartupHours > 0 { - s.oldestLoaded = hotCutoffStr - } else if len(s.packets) > 0 { - s.oldestLoaded = s.packets[0].FirstSeen + if !s.startupMemoryLimited { + if s.hotStartupHours > 0 { + s.oldestLoaded = hotCutoffStr + } else if len(s.packets) > 0 { + s.oldestLoaded = s.packets[0].FirstSeen + } } s.loaded = true s.mu.Unlock() @@ -477,10 +408,6 @@ func (s *PacketStore) LoadChunked(chunkSize int) error { elapsed := time.Since(t0) log.Printf("[store] LoadChunked: %d transmissions (%d observations) across %d chunk(s) in %v (chunkSize=%d, DB total=%d)", totalLoaded, s.totalObs, chunkIdx, elapsed, chunkSize, totalInDB) - if coldLoadAmbiguousHopsSkipped > 0 { - log.Printf("[store] LoadChunked: skipped %d ambiguous-prefix relay hops (unique_prefix gate, PR #1643 R1)", - coldLoadAmbiguousHopsSkipped) - } s.loadMultibyteCapFromDB() // Mark complete on the success path only — see the function-level // defer above for why this is NOT in a deferred call. Probes that @@ -490,160 +417,6 @@ func (s *PacketStore) LoadChunked(chunkSize int) error { return nil } -// scanAndMergeChunk consumes one chunk's rows under s.mu.Lock and -// returns the number of distinct transmissions seen + the max -// transmission id (cursor for the next chunk). -func (s *PacketStore) scanAndMergeChunk(rows *sql.Rows, relayPM *prefixMap, coldLoadAmbiguousHopsSkipped *int) (int, int64, error) { - s.mu.Lock() - defer s.mu.Unlock() - - hopsSeen := make(map[string]bool) - seenTxIDs := make(map[int]bool) - var maxID int64 - - for rows.Next() { - var txID int - var rawHex, hash, firstSeen, decodedJSON sql.NullString - var routeType, payloadType, payloadVersion sql.NullInt64 - var obsID sql.NullInt64 - var observerID, observerName, observerIATA, direction, pathJSON, obsTimestamp sql.NullString - var snr, rssi sql.NullFloat64 - var score sql.NullInt64 - var obsRawHex sql.NullString - var resolvedPathStr sql.NullString - var scopeName sql.NullString - - scanArgs := []interface{}{&txID, &rawHex, &hash, &firstSeen, &routeType, &payloadType, - &payloadVersion, &decodedJSON, - &obsID, &observerID, &observerName, &observerIATA, &direction, - &snr, &rssi, &score, &pathJSON, &obsTimestamp} - if s.db.hasObsRawHex { - scanArgs = append(scanArgs, &obsRawHex) - } - if s.db.hasResolvedPath { - scanArgs = append(scanArgs, &resolvedPathStr) - } - if s.db.hasScopeName { - scanArgs = append(scanArgs, &scopeName) - } - if err := rows.Scan(scanArgs...); err != nil { - log.Printf("[store] LoadChunked scan error: %v", err) - continue - } - - if int64(txID) > maxID { - maxID = int64(txID) - } - seenTxIDs[txID] = true - - hashStr := nullStrVal(hash) - tx := s.byHash[hashStr] - if tx == nil { - tx = &StoreTx{ - ID: txID, - RawHex: nullStrVal(rawHex), - Hash: hashStr, - FirstSeen: nullStrVal(firstSeen), - LatestSeen: nullStrVal(firstSeen), - RouteType: nullIntPtr(routeType), - PayloadType: nullIntPtr(payloadType), - DecodedJSON: nullStrVal(decodedJSON), - ScopeName: nullStrPtr(scopeName), - obsKeys: make(map[string]bool), - observerSet: make(map[string]bool), - } - s.byHash[hashStr] = tx - s.packets = append(s.packets, tx) - s.byTxID[txID] = tx - if txID > s.maxTxID { - s.maxTxID = txID - } - s.indexByNode(tx) - if tx.PayloadType != nil { - pt := *tx.PayloadType - s.byPayloadType[pt] = append(s.byPayloadType[pt], tx) - } - s.trackAdvertPubkey(tx) - s.trackedBytes += estimateStoreTxBytes(tx) - } - - if obsID.Valid { - oid := int(obsID.Int64) - obsIDStr := nullStrVal(observerID) - obsPJ := nullStrVal(pathJSON) - - dk := obsIDStr + "|" + obsPJ - if tx.obsKeys[dk] { - continue - } - - obs := &StoreObs{ - ID: oid, - TransmissionID: txID, - ObserverID: obsIDStr, - ObserverName: nullStrVal(observerName), - ObserverIATA: nullStrVal(observerIATA), - Direction: nullStrVal(direction), - SNR: nullFloatPtr(snr), - RSSI: nullFloatPtr(rssi), - Score: nullIntPtr(score), - PathJSON: obsPJ, - // obs.RawHex deliberately NOT stored: it duplicates the parent - // tx.RawHex (same content hash ⇒ same frame) and enrichObs falls - // back to tx.RawHex when obs.RawHex == "". obsRawHex is still - // scanned to keep scanArgs aligned with the o.raw_hex column. - Timestamp: normalizeTimestamp(nullStrVal(obsTimestamp)), - } - - rpStr := nullStrVal(resolvedPathStr) - if rpStr != "" { - rp := unmarshalResolvedPath(rpStr) - pks := extractResolvedPubkeys(rp) - s.indexResolvedPathHops(tx, pks, hopsSeen) - } else if relayPM != nil && obsPJ != "" && obsPJ != "[]" { - // resolved_path is NULL on live (since #1287 relay data is - // persisted as neighbor_edges, not per-observation). Re-resolve - // relay-hop attribution from path_json so relay nodes keep their - // analytics history across a restart instead of rebuilding only - // from post-restart live traffic. relayPM is passed in from - // LoadChunked (fetched before any chunk cursor opened). - // byNode ONLY — see the Load() counterpart for why the - // resolved_path/path-hop indexes must NOT be populated here. - // PR #1643 R1 munger #1: unique_prefix-only gate. - rp := resolvePathForObsColdLoad(obsPJ, obsIDStr, tx, relayPM, coldLoadAmbiguousHopsSkipped) - for _, pk := range extractResolvedPubkeys(rp) { - s.addToByNode(tx, pk) - } - } - - tx.Observations = append(tx.Observations, obs) - tx.obsKeys[dk] = true - if obs.ObserverID != "" && !tx.observerSet[obs.ObserverID] { - tx.observerSet[obs.ObserverID] = true - tx.UniqueObserverCount++ - } - tx.ObservationCount++ - if obs.Timestamp > tx.LatestSeen { - tx.LatestSeen = obs.Timestamp - } - - s.byObsID[oid] = obs - if oid > s.maxObsID { - s.maxObsID = oid - } - if obsIDStr != "" { - s.byObserver[obsIDStr] = append(s.byObserver[obsIDStr], obs) - } - s.totalObs++ - s.trackedBytes += estimateStoreObsBytes(obs) - } - } - if err := rows.Err(); err != nil { - return len(seenTxIDs), maxID, err - } - return len(seenTxIDs), maxID, nil -} - // loadStatusMiddleware sets X-CoreScope-Load-Status on every response. // While LoadChunked is in flight the header reports // "loading; progress="; after completion it reports "ready". diff --git a/cmd/server/chunked_load_id_zero_test.go b/cmd/server/chunked_load_id_zero_test.go index e6cf74bbe..47b7c6ec2 100644 --- a/cmd/server/chunked_load_id_zero_test.go +++ b/cmd/server/chunked_load_id_zero_test.go @@ -20,38 +20,46 @@ package main // row into s.byHash. import ( - "database/sql" "fmt" - "path/filepath" "testing" "time" ) func createTestDBWithIDZero(tb testing.TB, dbPath string, extraTx int) { tb.Helper() - conn, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL") + conn, err := openFixtureSQL(dbPath) if err != nil { tb.Fatal(err) } defer conn.Close() stmts := []string{ - `CREATE TABLE IF NOT EXISTS transmissions ( - id INTEGER PRIMARY KEY, + testNativeSQL(`CREATE TABLE IF NOT EXISTS transmissions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT - )`, - `CREATE TABLE IF NOT EXISTS observations ( - id INTEGER PRIMARY KEY, + )`, `CREATE TABLE IF NOT EXISTS transmissions ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + raw_hex TEXT, hash TEXT, first_seen TEXT, + route_type INTEGER, payload_type INTEGER, + payload_version INTEGER, decoded_json TEXT + )`), + testNativeSQL(`CREATE TABLE IF NOT EXISTS observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, - direction TEXT, snr REAL, rssi REAL, score INTEGER, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT - )`, - `CREATE TABLE IF NOT EXISTS observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT)`, + )`, `CREATE TABLE IF NOT EXISTS observations ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + transmission_id INTEGER, observer_id TEXT, observer_name TEXT, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, + path_json TEXT, timestamp TEXT, raw_hex TEXT + )`), + testNativeSQL(`CREATE TABLE IF NOT EXISTS observers (rowid INTEGER PRIMARY KEY AUTOINCREMENT, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, `CREATE TABLE IF NOT EXISTS observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`), `CREATE TABLE IF NOT EXISTS nodes ( - pubkey TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, - last_seen TEXT, first_seen TEXT, frequency REAL + public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, + last_seen TEXT, first_seen TEXT, frequency DOUBLE PRECISION )`, `CREATE TABLE IF NOT EXISTS schema_version (version INTEGER)`, `INSERT INTO schema_version (version) VALUES (1)`, @@ -63,8 +71,8 @@ func createTestDBWithIDZero(tb testing.TB, dbPath string, extraTx int) { } } - txStmt, _ := conn.Prepare("INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) VALUES (?, ?, ?, ?, ?, ?, ?, ?)") - obsStmt, _ := conn.Prepare("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") + txStmt, _ := conn.Prepare("INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)") + obsStmt, _ := conn.Prepare("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)") defer txStmt.Close() defer obsStmt.Close() @@ -89,11 +97,10 @@ func createTestDBWithIDZero(tb testing.TB, dbPath string, extraTx int) { // rows are silently dropped. This breaks the #1486 e2e fixture seed // which uses id=0 to sort the grouped row last in the default view. func TestLoadChunked_IncludesIDZero(t *testing.T) { - dir := t.TempDir() - dbPath := filepath.Join(dir, "idzero.db") + dbPath := postgresTestDSN(t) createTestDBWithIDZero(t, dbPath, 10) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatalf("OpenDB: %v", err) } diff --git a/cmd/server/chunked_load_oldest_test.go b/cmd/server/chunked_load_oldest_test.go index 8b9a5f826..f45a08a1d 100644 --- a/cmd/server/chunked_load_oldest_test.go +++ b/cmd/server/chunked_load_oldest_test.go @@ -19,9 +19,7 @@ package main // transmissions, not the FirstSeen of the first row in s.packets. import ( - "database/sql" "fmt" - "path/filepath" "testing" "time" ) @@ -31,29 +29,39 @@ import ( // This mirrors the freshen-fixture-shifted e2e DB exactly. func createTestDBReverseTime(tb testing.TB, dbPath string, numTx int) { tb.Helper() - conn, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL") + conn, err := openFixtureSQL(dbPath) if err != nil { tb.Fatal(err) } defer conn.Close() stmts := []string{ - `CREATE TABLE IF NOT EXISTS transmissions ( - id INTEGER PRIMARY KEY, + testNativeSQL(`CREATE TABLE IF NOT EXISTS transmissions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT - )`, - `CREATE TABLE IF NOT EXISTS observations ( - id INTEGER PRIMARY KEY, + )`, `CREATE TABLE IF NOT EXISTS transmissions ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + raw_hex TEXT, hash TEXT, first_seen TEXT, + route_type INTEGER, payload_type INTEGER, + payload_version INTEGER, decoded_json TEXT + )`), + testNativeSQL(`CREATE TABLE IF NOT EXISTS observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, - direction TEXT, snr REAL, rssi REAL, score INTEGER, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT - )`, - `CREATE TABLE IF NOT EXISTS observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT)`, + )`, `CREATE TABLE IF NOT EXISTS observations ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + transmission_id INTEGER, observer_id TEXT, observer_name TEXT, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, + path_json TEXT, timestamp TEXT, raw_hex TEXT + )`), + testNativeSQL(`CREATE TABLE IF NOT EXISTS observers (rowid INTEGER PRIMARY KEY AUTOINCREMENT, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, `CREATE TABLE IF NOT EXISTS observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`), `CREATE TABLE IF NOT EXISTS nodes ( - pubkey TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, - last_seen TEXT, first_seen TEXT, frequency REAL + public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, + last_seen TEXT, first_seen TEXT, frequency DOUBLE PRECISION )`, `CREATE TABLE IF NOT EXISTS schema_version (version INTEGER)`, `INSERT INTO schema_version (version) VALUES (1)`, @@ -65,8 +73,8 @@ func createTestDBReverseTime(tb testing.TB, dbPath string, numTx int) { } } - txStmt, _ := conn.Prepare("INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) VALUES (?, ?, ?, ?, ?, ?, ?, ?)") - obsStmt, _ := conn.Prepare("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") + txStmt, _ := conn.Prepare("INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)") + obsStmt, _ := conn.Prepare("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)") defer txStmt.Close() defer obsStmt.Close() @@ -83,11 +91,10 @@ func createTestDBReverseTime(tb testing.TB, dbPath string, numTx int) { func openReverseTimeStore(t *testing.T, numTx int) *PacketStore { t.Helper() - dir := t.TempDir() - dbPath := filepath.Join(dir, "rev.db") + dbPath := postgresTestDSN(t) createTestDBReverseTime(t, dbPath, numTx) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatalf("OpenDB: %v", err) } diff --git a/cmd/server/chunked_load_test.go b/cmd/server/chunked_load_test.go index c603acd3f..bd33c2aa4 100644 --- a/cmd/server/chunked_load_test.go +++ b/cmd/server/chunked_load_test.go @@ -19,20 +19,16 @@ package main import ( "net/http" "net/http/httptest" - "os" - "path/filepath" "testing" "time" ) func openChunkedTestStore(t *testing.T, numTx int) *PacketStore { t.Helper() - dir := t.TempDir() - dbPath := filepath.Join(dir, "chunked.db") + dbPath := postgresTestDSN(t) createTestDBAt(t, dbPath, numTx) - t.Cleanup(func() { os.RemoveAll(dir) }) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatalf("OpenDB: %v", err) } diff --git a/cmd/server/clock_skew.go b/cmd/server/clock_skew.go index 51bdf9cdd..b545d4002 100644 --- a/cmd/server/clock_skew.go +++ b/cmd/server/clock_skew.go @@ -220,6 +220,14 @@ func NewClockSkewEngine() *ClockSkewEngine { } } +// Invalidate makes the next Recompute run even if the last one is +// younger than computeInterval. +func (e *ClockSkewEngine) Invalidate() { + e.mu.Lock() + e.lastComputed = time.Time{} + e.mu.Unlock() +} + // Recompute recalculates all clock skew data from the packet store. // Called periodically or on demand. Holds store RLock externally. // Uses read-copy-update: heavy computation runs outside the write lock, diff --git a/cmd/server/collision_details_test.go b/cmd/server/collision_details_test.go index c59492e27..cbf6e931a 100644 --- a/cmd/server/collision_details_test.go +++ b/cmd/server/collision_details_test.go @@ -21,14 +21,14 @@ func TestCollisionDetailsIncludeNodePairs(t *testing.T) { // Add advert transmissions with hash_size=3 path bytes (0x80 = bits 10 → size 3) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('0180aabb11ccdd', 'col_hash_01', ?, 1, 4, '{"pubKey":"aabb11ccdd001122","name":"Node Alpha","type":"ADVERT"}')`, recent) + VALUES ('0180aabb11ccdd', 'col_hash_01', $1, 1, 4, '{"pubKey":"aabb11ccdd001122","name":"Node Alpha","type":"ADVERT"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 10.0, -91, '["aabb11"]', ?)`, recentEpoch) + VALUES (1, 1, 10.0, -91, '["aabb11"]', $1)`, recentEpoch) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('0180aabb11eeff', 'col_hash_02', ?, 1, 4, '{"pubKey":"aabb11eeff334455","name":"Node Beta","type":"ADVERT"}')`, recent) + VALUES ('0180aabb11eeff', 'col_hash_02', $1, 1, 4, '{"pubKey":"aabb11eeff334455","name":"Node Beta","type":"ADVERT"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (2, 1, 9.0, -93, '["aabb11"]', ?)`, recentEpoch) + VALUES (2, 1, 9.0, -93, '["aabb11"]', $1)`, recentEpoch) store := NewPacketStore(db, nil) store.Load() @@ -102,9 +102,9 @@ func TestCollisionDetailsEmptyWhenNoCollisions(t *testing.T) { db.conn.Exec(`INSERT INTO nodes (public_key, name, role) VALUES ('aabb11ccdd001122', 'Solo Node', 'repeater')`) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('0180aabb11ccdd', 'solo_hash_01', ?, 1, 4, '{"pubKey":"aabb11ccdd001122","name":"Solo Node","type":"ADVERT"}')`, recent) + VALUES ('0180aabb11ccdd', 'solo_hash_01', $1, 1, 4, '{"pubKey":"aabb11ccdd001122","name":"Solo Node","type":"ADVERT"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 10.0, -91, '["aabb11"]', ?)`, recentEpoch) + VALUES (1, 1, 10.0, -91, '["aabb11"]', $1)`, recentEpoch) store := NewPacketStore(db, nil) store.Load() diff --git a/cmd/server/config.go b/cmd/server/config.go index 28605c2bc..8f08e1964 100644 --- a/cmd/server/config.go +++ b/cmd/server/config.go @@ -37,10 +37,12 @@ type ListLimitsConfig struct { // Config mirrors the Node.js config.json structure (read-only fields). type Config struct { - Port int `json:"port"` - APIKey string `json:"apiKey"` - DBPath string `json:"dbPath"` - ListLimits *ListLimitsConfig `json:"listLimits"` + Port int `json:"port"` + APIKey string `json:"apiKey"` + DBPath string `json:"dbPath,omitempty"` // legacy SQLite path; an installed selection is authoritative + DatabaseURL string `json:"databaseURL"` + StateDir string `json:"stateDir"` + ListLimits *ListLimitsConfig `json:"listLimits"` // NodeBlacklist is a list of public keys to exclude from all API responses. // Blacklisted nodes are hidden from node lists, search, detail, map, and stats. @@ -147,6 +149,10 @@ type Config struct { // so browsers enforce same-origin policy. Set to ["*"] to allow all origins. CORSAllowedOrigins []string `json:"corsAllowedOrigins,omitempty"` + // WebSocket carries the /ws transport limits from #1794. Omitted entirely + // means: rate limit at its default, connection cap off, no deny list. + WebSocket *WebSocketConfig `json:"webSocket,omitempty"` + DebugAffinity bool `json:"debugAffinity,omitempty"` // MapDarkTileProvider selects the default dark-mode basemap provider for @@ -169,6 +175,25 @@ type Config struct { // ⇒ off; see ClientRxCoverageEnabled. ClientRxCoverage *ClientRxCoverageConfig `json:"clientRxCoverage,omitempty"` + // ClientRfSamples gates the opt-in RF environment sample stream (companion + // radio noise-floor/counters paired with GPS, table client_rf_samples). + // This is the same flag cmd/ingestor checks before recording samples; + // mirrored here so the server can gate reading them back. Absent/nil ⇒ + // off; see ClientRfSamplesEnabled. + ClientRfSamples *ClientRfSamplesConfig `json:"clientRfSamples,omitempty"` + + // UserManagement gates optional accounts + // (docs/specs/2026-10-06-user-management-design.md). Absent/nil means off; + // see UserManagementEnabled. + UserManagement *UserManagementConfig `json:"userManagement,omitempty"` + + // HashChannels and ChannelKeys are the ingestor's channel configuration, + // read here for the names only: a channel proposal for a configured name + // is refused (see configuredChannelNames). ChannelKeys drops the key + // values while unmarshalling, so the server never holds key material. + HashChannels []string `json:"hashChannels,omitempty"` + ChannelKeys channelKeyNames `json:"channelKeys,omitempty"` + ResolvedPath *ResolvedPathConfig `json:"resolvedPath,omitempty"` NeighborGraph *NeighborGraphConfig `json:"neighborGraph,omitempty"` @@ -269,6 +294,20 @@ func (c *Config) ClientRxCoverageEnabled() bool { return c != nil && c.ClientRxCoverage != nil && c.ClientRxCoverage.Enabled } +// ClientRfSamplesConfig gates the opt-in RF environment sample stream. +// Mirrors cmd/ingestor's ClientRfSamplesConfig (same JSON section, read by +// both binaries from the same config.json). +type ClientRfSamplesConfig struct { + Enabled bool `json:"enabled"` +} + +// ClientRfSamplesEnabled reports whether the opt-in RF environment sample +// stream is on. Nil config or absent/nil section ⇒ off (the safe default), +// mirroring ClientRxCoverageEnabled. +func (c *Config) ClientRfSamplesEnabled() bool { + return c != nil && c.ClientRfSamples != nil && c.ClientRfSamples.Enabled +} + // WSCompressionEnabled returns true when WebSocket permessage-deflate is explicitly enabled. func (c *Config) WSCompressionEnabled() bool { return c.Compression != nil && c.Compression.Websocket @@ -332,7 +371,7 @@ type RetentionConfig struct { MetricsDays int `json:"metricsDays"` } -// DBConfig is the shared SQLite vacuum/maintenance config (#919, #921). +// DBConfig shares startup-load settings and accepts obsolete SQLite config keys. type DBConfig = dbconfig.DBConfig // IncrementalVacuumPages returns the configured pages per vacuum or 1024 default. @@ -456,10 +495,13 @@ func LoadConfig(baseDirs ...string) (*Config, error) { for _, p := range paths { data, err := os.ReadFile(p) if err != nil { - continue + if os.IsNotExist(err) { + continue + } + return nil, fmt.Errorf("cannot read %s; refusing configuration fallback", filepath.Base(p)) } if err := json.Unmarshal(data, cfg); err != nil { - continue + return nil, fmt.Errorf("%s is malformed or has invalid field types; refusing configuration fallback", filepath.Base(p)) } cfg.NormalizeTimestampConfig() cfg.migrateDeprecatedConfig() @@ -474,6 +516,81 @@ func LoadConfig(baseDirs ...string) (*Config, error) { return cfg, nil // defaults } +// WebSocketConfig holds the /ws transport limits (#1794). These sit behind +// the CheckOrigin allowlist (#1793) and catch non-browser clients, which can +// omit or forge Origin. +type WebSocketConfig struct { + // MaxConnsPerIP caps concurrent /ws connections from one client address. + // DEFAULT 0, meaning OFF, and that is a deliberate departure from the + // value floated on #1794. + // + // A cap of 5 is safe only when one address means one household. It does + // not on mobile: carrier-grade NAT puts thousands of unrelated subscribers + // behind a single public IPv4, so a low cap would refuse real visitors on + // phones while barely inconveniencing a scraper that can rent more + // addresses. Operators who know their audience can set it; we must not + // pick it for them. + MaxConnsPerIP int `json:"maxConnsPerIP,omitempty"` + + // UpgradesPerMinPerIP caps handshakes per minute from one client address. + // DEFAULT 30, on. Unlike the connection cap this is safe under CGNAT: a + // real client upgrades a handful of times per minute even while + // reconnecting, so 30 leaves ordinary traffic untouched while flattening + // the reconnect loop that makes scrapers expensive. + UpgradesPerMinPerIP *int `json:"upgradesPerMinPerIP,omitempty"` + + // TrustedProxies lists the addresses or CIDRs of reverse proxies whose + // X-Forwarded-For may be believed. REQUIRED for the limits to do anything + // behind nginx/Caddy/Traefik/ingress: without it every visitor shares the + // proxy's address, so the limits are skipped and a warning is logged. + // Never list an address you do not control; anyone reaching the server + // from a trusted address can name any client IP they like. + TrustedProxies []string `json:"trustedProxies,omitempty"` + + // Deny blocks addresses outright at the upgrade, before the handshake. + // Accepts both bare addresses ("1.2.3.4") and CIDRs ("1.2.3.0/24"). + // Applies even when clients cannot be told apart, because it is an + // explicit instruction rather than an inference. + Deny []string `json:"deny,omitempty"` +} + +// WSMaxConnsPerIP returns the configured concurrent-connection cap, 0 = off. +func (c *Config) WSMaxConnsPerIP() int { + if c.WebSocket == nil || c.WebSocket.MaxConnsPerIP < 0 { + return 0 + } + return c.WebSocket.MaxConnsPerIP +} + +// WSUpgradesPerMinPerIP returns the upgrade-rate cap, defaulting to 30 when +// unset. A pointer distinguishes "not configured" (use the default) from an +// explicit 0, which an operator uses to turn the limit off. +func (c *Config) WSUpgradesPerMinPerIP() int { + if c.WebSocket == nil || c.WebSocket.UpgradesPerMinPerIP == nil { + return 30 + } + if v := *c.WebSocket.UpgradesPerMinPerIP; v > 0 { + return v + } + return 0 +} + +// WSTrustedProxies returns the configured reverse-proxy addresses, if any. +func (c *Config) WSTrustedProxies() []string { + if c.WebSocket == nil { + return nil + } + return c.WebSocket.TrustedProxies +} + +// WSDeny returns the configured deny entries, if any. +func (c *Config) WSDeny() []string { + if c.WebSocket == nil { + return nil + } + return c.WebSocket.Deny +} + func (c *Config) applyListLimitsDefaults() { if c.ListLimits == nil { c.ListLimits = &ListLimitsConfig{} @@ -623,14 +740,27 @@ func (h HealthThresholds) ToClientMs() map[string]int { } } -func (c *Config) ResolveDBPath(baseDir string) string { - if c.DBPath != "" { - return c.DBPath +// ResolveDatabaseURL refuses legacy paths rather than interpreting them as DSNs. +func (c *Config) ResolveDatabaseURL() (string, error) { + if c.DBPath != "" || os.Getenv("DB_PATH") != "" { + return "", fmt.Errorf("dbPath and DB_PATH are no longer supported; import the SQLite database and configure databaseURL or CORESCOPE_DATABASE_URL") } - if v := os.Getenv("DB_PATH"); v != "" { + v := envOrValue(os.Getenv, "CORESCOPE_READER_DATABASE_URL", envOrValue(os.Getenv, "CORESCOPE_DATABASE_URL", c.DatabaseURL)) + if v == "" { + return "", fmt.Errorf("configure databaseURL or CORESCOPE_DATABASE_URL for the PostgreSQL telemetry reader") + } + return v, nil +} + +func (c *Config) ResolveStateDir(baseDir string) string { + v := envOrValue(os.Getenv, "CORESCOPE_STATE_DIR", c.StateDir) + if v == "" { + v = "data" + } + if filepath.IsAbs(v) { return v } - return filepath.Join(baseDir, "data", "meshcore.db") + return filepath.Join(baseDir, v) } func (c *Config) NormalizeTimestampConfig() { @@ -891,9 +1021,15 @@ func SaveGeoFilter(configDir string, gf *GeoFilterConfig) error { } out = append(out, '\n') - // Atomic write: temp file + rename. + // Atomic write: temp file + rename. Keep the original file's mode: + // config.json holds the API key and broker passwords, and an operator + // who made it 0600 should not find it 0644 after a geo-filter save. + mode := os.FileMode(0644) + if fi, err := os.Stat(configPath); err == nil { + mode = fi.Mode().Perm() + } tmp := configPath + ".tmp" - if err := os.WriteFile(tmp, out, 0644); err != nil { + if err := os.WriteFile(tmp, out, mode); err != nil { return fmt.Errorf("write config: %w", err) } if err := os.Rename(tmp, configPath); err != nil { @@ -996,3 +1132,54 @@ func (c *Config) AnalyticsRecomputeIntervals() AnalyticsRecomputeIntervals { out.NodesClockSkew = get("nodesClockSkew") return out } + +// channelKeyNames is config.json channelKeys ({name: hexKey}) reduced to the +// names; the key values are discarded during unmarshalling. A value that is +// not an object is ignored with a log line instead of failing: LoadConfig +// falls back to pure defaults on any parse error, and this field must not +// change config loading for deployments without channel proposals. +type channelKeyNames []string + +func (n *channelKeyNames) UnmarshalJSON(b []byte) error { + var m map[string]json.RawMessage + if err := json.Unmarshal(b, &m); err != nil { + log.Printf("[config] channelKeys is not an object; ignoring it for channel-proposal checks") + *n = nil + return nil + } + names := make(channelKeyNames, 0, len(m)) + for name := range m { + names = append(names, name) + } + *n = names + return nil +} + +// configuredChannelNames is the set of channel names the ingestor already +// decrypts from config.json, normalised as cmd/ingestor's loadChannelKeys +// does: hashChannels trimmed and prefixed with '#', channelKeys names as +// written except the well-known casing fix (public becomes Public). +// Case-sensitive, like the ingestor's key map. +func (c *Config) configuredChannelNames() map[string]bool { + names := map[string]bool{} + if c == nil { + return names + } + for _, raw := range c.HashChannels { + name := strings.TrimSpace(raw) + if name == "" { + continue + } + if !strings.HasPrefix(name, "#") { + name = "#" + name + } + names[name] = true + } + for _, name := range c.ChannelKeys { + if strings.EqualFold(name, "public") { + name = "Public" + } + names[name] = true + } + return names +} diff --git a/cmd/server/config_mode_test.go b/cmd/server/config_mode_test.go new file mode 100644 index 000000000..175b85a38 --- /dev/null +++ b/cmd/server/config_mode_test.go @@ -0,0 +1,47 @@ +package main + +import ( + "os" + "path/filepath" + "testing" +) + +// SaveGeoFilter rewrites config.json through a temp file. The file holds the +// API key and broker passwords, so the rewrite must keep whatever mode the +// operator set rather than resetting it to 0644. +func TestSaveGeoFilterPreservesFileMode(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "config.json") + if err := os.WriteFile(path, []byte(`{"apiKey":"secret-key-that-is-long-enough"}`+"\n"), 0600); err != nil { + t.Fatal(err) + } + before, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + want := before.Mode().Perm() + gf := &GeoFilterConfig{Polygon: [][2]float64{{0, 0}, {0, 1}, {1, 1}, {1, 0}}} + if err := SaveGeoFilter(dir, gf); err != nil { + t.Fatalf("SaveGeoFilter: %v", err) + } + fi, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if got := fi.Mode().Perm(); got != want { + t.Fatalf("config.json mode after save = %o, want %o", got, want) + } + // Default stays 0644 for a file that was 0644. + if err := os.Chmod(path, 0644); err != nil { + t.Fatal(err) + } + before, _ = os.Stat(path) + want = before.Mode().Perm() + if err := SaveGeoFilter(dir, nil); err != nil { + t.Fatalf("SaveGeoFilter(nil): %v", err) + } + fi, _ = os.Stat(path) + if got := fi.Mode().Perm(); got != want { + t.Fatalf("config.json mode after save = %o, want %o", got, want) + } +} diff --git a/cmd/server/config_test.go b/cmd/server/config_test.go index b78968ab6..b241e445d 100644 --- a/cmd/server/config_test.go +++ b/cmd/server/config_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "os" "path/filepath" + "strings" "testing" ) @@ -103,15 +104,47 @@ func TestLoadConfigNoFiles(t *testing.T) { func TestLoadConfigInvalidJSON(t *testing.T) { dir := t.TempDir() - os.WriteFile(filepath.Join(dir, "config.json"), []byte("{invalid"), 0644) - - cfg, err := LoadConfig(dir) - if err != nil { + if err := os.WriteFile(filepath.Join(dir, "config.json"), []byte("{invalid"), 0600); err != nil { t.Fatal(err) } - // Should return defaults when JSON is invalid - if cfg.Port != 3000 { - t.Errorf("expected default port 3000, got %d", cfg.Port) + if cfg, err := LoadConfig(dir); err == nil || cfg != nil { + t.Fatal("malformed primary config fell back to defaults") + } +} + +func TestLoadConfigNeverUsesFallbackAfterExistingConfigError(t *testing.T) { + for _, kind := range []string{"invalid JSON", "invalid field type", "unreadable file"} { + t.Run(kind, func(t *testing.T) { + dir := t.TempDir() + if err := os.Mkdir(filepath.Join(dir, "data"), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "data", "config.json"), []byte(`{"stateDir":"unintended-installation","dbPath":"other.db"}`), 0600); err != nil { + t.Fatal(err) + } + primary := filepath.Join(dir, "config.json") + switch kind { + case "invalid JSON": + if err := os.WriteFile(primary, []byte(`{"stateDir":`), 0600); err != nil { + t.Fatal(err) + } + case "invalid field type": + if err := os.WriteFile(primary, []byte(`{"databaseURL":"postgresql://user:private-password@host/database","port":"invalid"}`), 0600); err != nil { + t.Fatal(err) + } + case "unreadable file": + if err := os.Mkdir(primary, 0700); err != nil { + t.Fatal(err) + } + } + cfg, err := LoadConfig(dir) + if err == nil || cfg != nil { + t.Fatal("existing config error selected fallback storage") + } + if strings.Contains(err.Error(), "private-password") { + t.Fatal("config error exposed credentials") + } + }) } } @@ -320,33 +353,45 @@ func TestGetHealthMs(t *testing.T) { } } -func TestResolveDBPath(t *testing.T) { - t.Run("DBPath set", func(t *testing.T) { - cfg := &Config{DBPath: "/explicit/path.db"} - got := cfg.ResolveDBPath("/base") - if got != "/explicit/path.db" { - t.Errorf("expected /explicit/path.db, got %s", got) - } - }) - - t.Run("env var", func(t *testing.T) { - cfg := &Config{} - t.Setenv("DB_PATH", "/env/path.db") - got := cfg.ResolveDBPath("/base") - if got != "/env/path.db" { - t.Errorf("expected /env/path.db, got %s", got) - } - }) +func TestResolveDatabaseURL(t *testing.T) { + t.Setenv("CORESCOPE_READER_DATABASE_URL", "") + t.Setenv("DB_PATH", "") + t.Setenv("CORESCOPE_DATABASE_URL", "") + cfg := &Config{DatabaseURL: "postgresql://reader@localhost/telemetry"} + if got, err := cfg.ResolveDatabaseURL(); err != nil || got != cfg.DatabaseURL { + t.Fatalf("config URL: %v", err) + } + t.Setenv("CORESCOPE_DATABASE_URL", "postgresql://reader@localhost/envdb") + if got, err := cfg.ResolveDatabaseURL(); err != nil || got != "postgresql://reader@localhost/envdb" { + t.Fatalf("environment URL: %v", err) + } + t.Setenv("CORESCOPE_READER_DATABASE_URL", "postgresql://reader@localhost/role") + if got, err := cfg.ResolveDatabaseURL(); err != nil || got != "postgresql://reader@localhost/role" { + t.Fatalf("reader URL precedence: %v", err) + } + t.Setenv("CORESCOPE_READER_DATABASE_URL", "") + cfg.DBPath = "private.db" + if _, err := cfg.ResolveDatabaseURL(); err == nil || strings.Contains(err.Error(), cfg.DBPath) { + t.Fatalf("legacy path must be rejected without echoing input: %v", err) + } + cfg.DBPath = "" + t.Setenv("CORESCOPE_DATABASE_URL", "") + cfg.DatabaseURL = "" + if _, err := cfg.ResolveDatabaseURL(); err == nil { + t.Fatal("missing URL accepted") + } +} - t.Run("default", func(t *testing.T) { - cfg := &Config{} - t.Setenv("DB_PATH", "") - got := cfg.ResolveDBPath("/base") - expected := filepath.Join("/base", "data", "meshcore.db") - if got != expected { - t.Errorf("expected %s, got %s", expected, got) - } - }) +func TestResolveStateDir(t *testing.T) { + t.Setenv("CORESCOPE_STATE_DIR", "") + cfg := &Config{} + if got := cfg.ResolveStateDir("base"); got != filepath.Join("base", "data") { + t.Fatal(got) + } + t.Setenv("CORESCOPE_STATE_DIR", "queues") + if got := cfg.ResolveStateDir("base"); got != filepath.Join("base", "queues") { + t.Fatal(got) + } } func TestPropagationBufferMs(t *testing.T) { diff --git a/cmd/server/coverage_test.go b/cmd/server/coverage_test.go index de4ba76d6..83a3a2d8b 100644 --- a/cmd/server/coverage_test.go +++ b/cmd/server/coverage_test.go @@ -15,60 +15,85 @@ import ( "time" "github.com/gorilla/mux" - _ "modernc.org/sqlite" ) // --- helpers --- func setupTestDBv2(t *testing.T) *DB { t.Helper() - conn, err := sql.Open("sqlite", ":memory:") + conn, err := openPostgresTestDB(t) if err != nil { t.Fatal(err) } // Force single connection so all goroutines share the same in-memory DB conn.SetMaxOpenConns(1) - schema := ` + schema := testNativeSQL(` CREATE TABLE nodes ( public_key TEXT PRIMARY KEY, name TEXT, role TEXT, - lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, advert_count INTEGER DEFAULT 0, - battery_mv INTEGER, temperature_c REAL, foreign_advert INTEGER DEFAULT 0 + lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, first_seen TEXT, advert_count INTEGER DEFAULT 0, + battery_mv INTEGER, temperature_c DOUBLE PRECISION, foreign_advert INTEGER DEFAULT 0 ); - CREATE TABLE observers ( - id TEXT PRIMARY KEY, name TEXT, iata TEXT, last_seen TEXT, first_seen TEXT, + CREATE TABLE observers (id TEXT PRIMARY KEY, name TEXT, iata TEXT, last_seen TEXT, first_seen TEXT, packet_count INTEGER DEFAULT 0, model TEXT, firmware TEXT, - client_version TEXT, radio TEXT, battery_mv INTEGER, uptime_secs INTEGER, noise_floor REAL, + client_version TEXT, radio TEXT, battery_mv INTEGER, uptime_secs INTEGER, noise_floor DOUBLE PRECISION, inactive INTEGER DEFAULT 0 ); CREATE TABLE transmissions ( id INTEGER PRIMARY KEY AUTOINCREMENT, raw_hex TEXT NOT NULL, hash TEXT NOT NULL UNIQUE, first_seen TEXT NOT NULL, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, - decoded_json TEXT, channel_hash TEXT DEFAULT NULL, from_pubkey TEXT DEFAULT NULL, created_at TEXT DEFAULT (datetime('now')) + decoded_json TEXT, channel_hash TEXT DEFAULT NULL, from_pubkey TEXT DEFAULT NULL, created_at TEXT DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ','now')) ); CREATE TABLE observations ( id INTEGER PRIMARY KEY AUTOINCREMENT, transmission_id INTEGER NOT NULL REFERENCES transmissions(id), observer_id TEXT, observer_name TEXT, direction TEXT, - snr REAL, rssi REAL, score INTEGER, path_json TEXT, timestamp INTEGER NOT NULL, raw_hex TEXT + snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp INTEGER NOT NULL, raw_hex TEXT + ); + CREATE TRIGGER test_from_pubkey_advert AFTER INSERT ON transmissions + WHEN NEW.from_pubkey IS NULL AND NEW.payload_type=4 AND + CASE WHEN json_valid(NEW.decoded_json) THEN json_type(NEW.decoded_json)='object' ELSE 0 END + BEGIN UPDATE transmissions SET from_pubkey=NULLIF(json_extract(NEW.decoded_json,'$.pubKey'),'') WHERE id=NEW.id; END; + CREATE INDEX IF NOT EXISTS idx_transmissions_from_pubkey ON transmissions(from_pubkey); + `, ` + CREATE TABLE nodes ( + public_key TEXT PRIMARY KEY, name TEXT, role TEXT, + lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, first_seen TEXT, advert_count INTEGER DEFAULT 0, + battery_mv INTEGER, temperature_c DOUBLE PRECISION, foreign_advert INTEGER DEFAULT 0 + ); + CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY UNIQUE, + id TEXT PRIMARY KEY, name TEXT, iata TEXT, last_seen TEXT, first_seen TEXT, + packet_count INTEGER DEFAULT 0, model TEXT, firmware TEXT, + client_version TEXT, radio TEXT, battery_mv INTEGER, uptime_secs INTEGER, noise_floor DOUBLE PRECISION, + inactive INTEGER DEFAULT 0 + ); + CREATE TABLE transmissions ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, raw_hex TEXT NOT NULL, + hash TEXT NOT NULL UNIQUE, first_seen TEXT NOT NULL, + route_type INTEGER, payload_type INTEGER, payload_version INTEGER, + decoded_json TEXT, channel_hash TEXT DEFAULT NULL, from_pubkey TEXT DEFAULT NULL, created_at TEXT DEFAULT (to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"')) ); - CREATE TRIGGER IF NOT EXISTS test_from_pubkey_advert - AFTER INSERT ON transmissions - FOR EACH ROW - WHEN NEW.from_pubkey IS NULL AND NEW.payload_type = 4 AND NEW.decoded_json IS NOT NULL - AND json_extract(NEW.decoded_json, '$.pubKey') IS NOT NULL - AND json_extract(NEW.decoded_json, '$.pubKey') <> '' + CREATE TABLE observations ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + transmission_id INTEGER NOT NULL REFERENCES transmissions(id), + observer_id TEXT, observer_name TEXT, direction TEXT, + snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp INTEGER NOT NULL, raw_hex TEXT + ); + CREATE FUNCTION test_from_pubkey_advert() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN - UPDATE transmissions - SET from_pubkey = json_extract(NEW.decoded_json, '$.pubKey') - WHERE id = NEW.id; - END; + IF NEW.from_pubkey IS NULL AND NEW.payload_type = 4 AND NEW.decoded_json IS JSON OBJECT THEN + NEW.from_pubkey := NULLIF(NEW.decoded_json::json ->> 'pubKey', ''); + END IF; + RETURN NEW; + END $$; + CREATE TRIGGER test_from_pubkey_advert BEFORE INSERT ON transmissions + FOR EACH ROW EXECUTE FUNCTION test_from_pubkey_advert(); CREATE INDEX IF NOT EXISTS idx_transmissions_from_pubkey ON transmissions(from_pubkey); - ` + `) if _, err := conn.Exec(schema); err != nil { t.Fatal(err) } - return &DB{conn: conn, isV3: false} + return &DB{backend: testBackendValue(), conn: conn, isV3: false} } func seedV2Data(t *testing.T, db *DB) { @@ -78,13 +103,13 @@ func seedV2Data(t *testing.T, db *DB) { epoch := now.Add(-1 * time.Hour).Unix() db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obs1', 'Obs One', 'SJC', ?, '2026-01-01T00:00:00Z', 100)`, recent) + VALUES ('obs1', 'Obs One', 'SJC', $1, '2026-01-01T00:00:00Z', 100)`, recent) db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES ('aabbccdd11223344', 'TestRepeater', 'repeater', 37.5, -122.0, ?, '2026-01-01T00:00:00Z', 50)`, recent) + VALUES ('aabbccdd11223344', 'TestRepeater', 'repeater', 37.5, -122.0, $1, '2026-01-01T00:00:00Z', 50)`, recent) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('AABB', 'abc123def4567890', ?, 1, 4, '{"pubKey":"aabbccdd11223344","name":"TestRepeater","type":"ADVERT"}')`, recent) + VALUES ('AABB', 'abc123def4567890', $1, 1, 4, '{"pubKey":"aabbccdd11223344","name":"TestRepeater","type":"ADVERT"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_id, observer_name, snr, rssi, path_json, timestamp) - VALUES (1, 'obs1', 'Obs One', 12.5, -90, '["aa","bb"]', ?)`, epoch) + VALUES (1, 'obs1', 'Obs One', 12.5, -90, '["aa","bb"]', $1)`, epoch) } func setupNoStoreServer(t *testing.T) (*Server, *mux.Router) { @@ -339,13 +364,13 @@ func TestIngestNewFromDB(t *testing.T) { // Insert a new transmission in DB now := time.Now().UTC().Format(time.RFC3339) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('EEFF', 'newhash123456abcd', ?, 1, 4, '{"pubKey":"aabbccdd11223344","type":"ADVERT"}')`, now) + VALUES ('EEFF', 'newhash123456abcd', $1, 1, 4, '{"pubKey":"aabbccdd11223344","type":"ADVERT"}')`, now) newTxID := 0 db.conn.QueryRow("SELECT MAX(id) FROM transmissions").Scan(&newTxID) // Add observation for the new transmission db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (?, 1, 10.0, -92, '["cc"]', ?)`, newTxID, time.Now().Unix()) + VALUES ($1, 1, 10.0, -92, '["cc"]', $2)`, newTxID, time.Now().Unix()) // Ingest broadcastMaps, newMax := store.IngestNewFromDB(initialMax, 100) @@ -409,11 +434,11 @@ func TestIngestNewFromDBv2(t *testing.T) { now := time.Now().UTC().Format(time.RFC3339) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('EEFF', 'v2newhash12345678', ?, 1, 4, '{"pubKey":"aabbccdd11223344","type":"ADVERT"}')`, now) + VALUES ('EEFF', 'v2newhash12345678', $1, 1, 4, '{"pubKey":"aabbccdd11223344","type":"ADVERT"}')`, now) newTxID := 0 db.conn.QueryRow("SELECT MAX(id) FROM transmissions").Scan(&newTxID) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_id, observer_name, snr, rssi, path_json, timestamp) - VALUES (?, 'obs1', 'Obs One', 10.0, -92, '["cc"]', ?)`, newTxID, time.Now().Unix()) + VALUES ($1, 'obs1', 'Obs One', 10.0, -92, '["cc"]', $2)`, newTxID, time.Now().Unix()) broadcastMaps, newMax := store.IngestNewFromDB(initialMax, 100) if newMax <= initialMax { @@ -729,15 +754,15 @@ func TestGetChannelMessagesDedupe(t *testing.T) { // Insert a duplicate channel message with the same hash as existing db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('DDEE', 'dupchannelhash1234', ?, 1, 5, '{"type":"CHAN","channel":"#test","text":"Hello: World","sender":"TestUser"}')`, recent) + VALUES ('DDEE', 'dupchannelhash1234', $1, 1, 5, '{"type":"CHAN","channel":"#test","text":"Hello: World","sender":"TestUser"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (3, 1, 11.0, -91, '["aa"]', ?)`, epoch) + VALUES (3, 1, 11.0, -91, '["aa"]', $1)`, epoch) // Insert another dupe same hash as above (should dedup) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('DDFF', 'dupchannelhash5678', ?, 1, 5, '{"type":"CHAN","channel":"#test","text":"Hello: World","sender":"TestUser"}')`, recent) + VALUES ('DDFF', 'dupchannelhash5678', $1, 1, 5, '{"type":"CHAN","channel":"#test","text":"Hello: World","sender":"TestUser"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (4, 2, 9.0, -93, '[]', ?)`, epoch) + VALUES (4, 2, 9.0, -93, '[]', $1)`, epoch) store := NewPacketStore(db, nil) store.Load() @@ -1130,9 +1155,9 @@ func TestPollerStartWithStore(t *testing.T) { // Insert new data while poller running now := time.Now().UTC().Format(time.RFC3339) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type) - VALUES ('FFEE', 'pollerhash12345678', ?, 1, 4)`, now) + VALUES ('FFEE', 'pollerhash12345678', $1, 1, 4)`, now) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES ((SELECT MAX(id) FROM transmissions), 1, 10.0, -92, '[]', ?)`, time.Now().Unix()) + VALUES ((SELECT MAX(id) FROM transmissions), 1, 10.0, -92, '[]', $1)`, time.Now().Unix()) time.Sleep(200 * time.Millisecond) poller.Stop() @@ -1374,7 +1399,7 @@ func TestBuildTransmissionWhereRFC3339(t *testing.T) { // PR #1187 r2: RFC3339 since/until MUST use observations.timestamp // subquery so re-observed packets (older first_seen but recent // observation) are still included. Anything else breaks semantics. - if !strings.Contains(where[0], "observations") || !strings.Contains(where[0], "timestamp >= ?") { + if !strings.Contains(where[0], "observations") || !strings.Contains(where[0], "timestamp >= "+db.parameter(1)+"") { t.Errorf("expected observations.timestamp subquery for RFC3339 since, got %q", where[0]) } }) @@ -1388,7 +1413,7 @@ func TestBuildTransmissionWhereRFC3339(t *testing.T) { if len(args) != 1 { t.Errorf("expected 1 arg, got %d", len(args)) } - if !strings.Contains(where[0], "observations") || !strings.Contains(where[0], "timestamp <= ?") { + if !strings.Contains(where[0], "observations") || !strings.Contains(where[0], "timestamp <= "+db.parameter(1)+"") { t.Errorf("expected observations.timestamp subquery for RFC3339 until, got %q", where[0]) } }) @@ -1874,7 +1899,7 @@ func TestEnrichObs(t *testing.T) { t.Skip("no observations loaded") } - enriched := store.enrichObs(obs) + enriched := store.enrichObsWithTx(obs, store.byTxID[obs.TransmissionID]) if enriched["observer_id"] == nil { t.Error("expected observer_id") } @@ -2188,21 +2213,21 @@ func setupRichTestDB(t *testing.T) *DB { // route_type 1 = FLOOD, path byte at position 1 (hex index 2..3) // header: 0x01 (route_type=1), path byte: 0x40 (hashSize bits=01 → size 2) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('0140aabbccdd', 'hash_with_path_01', ?, 1, 4, '{"pubKey":"aabbccdd11223344","name":"TestRepeater","type":"ADVERT"}')`, recent) + VALUES ('0140aabbccdd', 'hash_with_path_01', $1, 1, 4, '{"pubKey":"aabbccdd11223344","name":"TestRepeater","type":"ADVERT"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (3, 1, 10.0, -91, '["aabb","ccdd"]', ?)`, recentEpoch) + VALUES (3, 1, 10.0, -91, '["aabb","ccdd"]', $1)`, recentEpoch) // Another advert with 3-byte hash size: header 0x01, path byte 0x80 (bits=10 → size 3) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('0180eeff0011', 'hash_with_path_02', ?, 1, 4, '{"pubKey":"eeff00112233aabb","name":"TestCompanion","type":"ADVERT"}')`, yesterday) + VALUES ('0180eeff0011', 'hash_with_path_02', $1, 1, 4, '{"pubKey":"eeff00112233aabb","name":"TestCompanion","type":"ADVERT"}')`, yesterday) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (4, 2, 8.5, -94, '["eeff","0011","2233"]', ?)`, yesterdayEpoch) + VALUES (4, 2, 8.5, -94, '["eeff","0011","2233"]', $1)`, yesterdayEpoch) // Another channel message with different sender for analytics db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('CC01', 'chan_msg_hash_001', ?, 1, 5, '{"type":"CHAN","channel":"#test","text":"User2: Another msg","sender":"User2","channelHash":"abc123"}')`, recent) + VALUES ('CC01', 'chan_msg_hash_001', $1, 1, 5, '{"type":"CHAN","channel":"#test","text":"User2: Another msg","sender":"User2","channelHash":"abc123"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (5, 1, 14.0, -88, '["aa"]', ?)`, recentEpoch) + VALUES (5, 1, 14.0, -88, '["aa"]', $1)`, recentEpoch) return db } @@ -2556,20 +2581,20 @@ func TestStoreGetAnalyticsChannelsNumericHash(t *testing.T) { // Insert GRP_TXT packets with numeric channelHash (matches decoder.js output) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('DD01', 'grp_num_hash_1', ?, 1, 5, '{"type":"GRP_TXT","channelHash":97,"channelHashHex":"61","decryptionStatus":"no_key"}')`, recent) + VALUES ('DD01', 'grp_num_hash_1', $1, 1, 5, '{"type":"GRP_TXT","channelHash":97,"channelHashHex":"61","decryptionStatus":"no_key"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (4, 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (4, 1, 10.0, -90, '[]', $1)`, recentEpoch) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('DD02', 'grp_num_hash_2', ?, 1, 5, '{"type":"GRP_TXT","channelHash":42,"channelHashHex":"2A","decryptionStatus":"no_key"}')`, recent) + VALUES ('DD02', 'grp_num_hash_2', $1, 1, 5, '{"type":"GRP_TXT","channelHash":42,"channelHashHex":"2A","decryptionStatus":"no_key"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (5, 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (5, 1, 10.0, -90, '[]', $1)`, recentEpoch) // Also a decrypted CHAN with numeric channelHash — use hash 198 which is the real hash for #general db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('DD03', 'chan_num_hash_3', ?, 1, 5, '{"type":"CHAN","channel":"general","channelHash":198,"channelHashHex":"C6","text":"hello","sender":"Alice"}')`, recent) + VALUES ('DD03', 'chan_num_hash_3', $1, 1, 5, '{"type":"CHAN","channel":"general","channelHash":198,"channelHashHex":"C6","text":"hello","sender":"Alice"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (6, 1, 12.0, -88, '[]', ?)`, recentEpoch) + VALUES (6, 1, 12.0, -88, '[]', $1)`, recentEpoch) store := NewPacketStore(db, nil) store.Load() @@ -2705,7 +2730,12 @@ func TestHandleAnalyticsSubpathsWithStore(t *testing.T) { hub := NewHub() srv := NewServer(db, cfg, hub) store := NewPacketStore(db, nil) - store.Load() + if err := store.Load(); err != nil { + t.Fatalf("store.Load failed: %v", err) + } + if !store.WaitIndexesReady(5 * time.Second) { + t.Fatal("indexes not ready after 5s") + } srv.store = store router := mux.NewRouter() srv.RegisterRoutes(router) @@ -2725,7 +2755,12 @@ func TestHandleAnalyticsSubpathDetailWithStore(t *testing.T) { hub := NewHub() srv := NewServer(db, cfg, hub) store := NewPacketStore(db, nil) - store.Load() + if err := store.Load(); err != nil { + t.Fatalf("store.Load failed: %v", err) + } + if !store.WaitIndexesReady(5 * time.Second) { + t.Fatal("indexes not ready after 5s") + } srv.store = store router := mux.NewRouter() srv.RegisterRoutes(router) @@ -3210,64 +3245,64 @@ func TestGetNodeHashSizeInfoEdgeCases(t *testing.T) { // Observers db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obs1', 'Obs', 'SJC', ?, '2026-01-01T00:00:00Z', 10)`, recent) + VALUES ('obs1', 'Obs', 'SJC', $1, '2026-01-01T00:00:00Z', 10)`, recent) // Adverts with various edge cases // 1. Valid advert with pubKey db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('0140aabbccdd', 'hs_valid_1', ?, 1, 4, '{"pubKey":"aabbccdd11223344","name":"NodeA","type":"ADVERT"}')`, recent) + VALUES ('0140aabbccdd', 'hs_valid_1', $1, 1, 4, '{"pubKey":"aabbccdd11223344","name":"NodeA","type":"ADVERT"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (1, 1, 10.0, -90, '[]', $1)`, recentEpoch) // 2. Short raw_hex (< 4 chars) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('01', 'hs_short_hex', ?, 1, 4, '{"pubKey":"eeff00112233aabb","name":"NodeB","type":"ADVERT"}')`, recent) + VALUES ('01', 'hs_short_hex', $1, 1, 4, '{"pubKey":"eeff00112233aabb","name":"NodeB","type":"ADVERT"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (2, 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (2, 1, 10.0, -90, '[]', $1)`, recentEpoch) // 3. Invalid hex in path byte position db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('01GGHHII', 'hs_bad_hex', ?, 1, 4, '{"pubKey":"1122334455667788","name":"NodeC","type":"ADVERT"}')`, recent) + VALUES ('01GGHHII', 'hs_bad_hex', $1, 1, 4, '{"pubKey":"1122334455667788","name":"NodeC","type":"ADVERT"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (3, 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (3, 1, 10.0, -90, '[]', $1)`, recentEpoch) // 4. Invalid JSON db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('0140aabb', 'hs_bad_json', ?, 1, 4, 'not-json')`, recent) + VALUES ('0140aabb', 'hs_bad_json', $1, 1, 4, 'not-json')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (4, 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (4, 1, 10.0, -90, '[]', $1)`, recentEpoch) // 5. JSON with public_key field instead of pubKey db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('0180eeff', 'hs_alt_key', ?, 1, 4, '{"public_key":"aabbccdd11223344","name":"NodeA","type":"ADVERT"}')`, recent) + VALUES ('0180eeff', 'hs_alt_key', $1, 1, 4, '{"public_key":"aabbccdd11223344","name":"NodeA","type":"ADVERT"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (5, 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (5, 1, 10.0, -90, '[]', $1)`, recentEpoch) // 6. JSON with no pubKey at all db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('01C0ffee', 'hs_no_pk', ?, 1, 4, '{"name":"NodeZ","type":"ADVERT"}')`, recent) + VALUES ('01C0ffee', 'hs_no_pk', $1, 1, 4, '{"name":"NodeZ","type":"ADVERT"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (6, 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (6, 1, 10.0, -90, '[]', $1)`, recentEpoch) // 7. Empty decoded_json db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('0140bbcc', 'hs_empty_json', ?, 1, 4, '')`, recent) + VALUES ('0140bbcc', 'hs_empty_json', $1, 1, 4, '')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (7, 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (7, 1, 10.0, -90, '[]', $1)`, recentEpoch) // 8-10. Multiple adverts for same node with different hash sizes (flip-flop test) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('0140dd01', 'hs_flip_1', ?, 1, 4, '{"pubKey":"ffff000011112222","name":"Flipper","type":"ADVERT"}')`, recent) + VALUES ('0140dd01', 'hs_flip_1', $1, 1, 4, '{"pubKey":"ffff000011112222","name":"Flipper","type":"ADVERT"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (8, 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (8, 1, 10.0, -90, '[]', $1)`, recentEpoch) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('0180dd02', 'hs_flip_2', ?, 1, 4, '{"pubKey":"ffff000011112222","name":"Flipper","type":"ADVERT"}')`, recent) + VALUES ('0180dd02', 'hs_flip_2', $1, 1, 4, '{"pubKey":"ffff000011112222","name":"Flipper","type":"ADVERT"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (9, 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (9, 1, 10.0, -90, '[]', $1)`, recentEpoch) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('0140dd03', 'hs_flip_3', ?, 1, 4, '{"pubKey":"ffff000011112222","name":"Flipper","type":"ADVERT"}')`, recent) + VALUES ('0140dd03', 'hs_flip_3', $1, 1, 4, '{"pubKey":"ffff000011112222","name":"Flipper","type":"ADVERT"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (10, 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (10, 1, 10.0, -90, '[]', $1)`, recentEpoch) store := NewPacketStore(db, nil) store.Load() @@ -3311,7 +3346,7 @@ func TestHashSizeTransportRoutePathByteOffset(t *testing.T) { recentEpoch := now.Add(-1 * time.Hour).Unix() db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obs1', 'Obs', 'SJC', ?, '2026-01-01T00:00:00Z', 10)`, recent) + VALUES ('obs1', 'Obs', 'SJC', $1, '2026-01-01T00:00:00Z', 10)`, recent) // Route type 0 (TRANSPORT_FLOOD): header=0x04 (payload_type=1, route_type=0) // 4 transport bytes + path byte at offset 5. @@ -3319,25 +3354,25 @@ func TestHashSizeTransportRoutePathByteOffset(t *testing.T) { // If bug is present, code reads byte 1 (0xAA) → hash_size bits = 10 → size 3 (coincidence) // Use path byte 0x40 (hash_size=2) and transport byte 0x01 at offset 1 (hash_size=1 if misread) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('100102030440aabb', 'tf_offset', ?, 0, 4, '{"pubKey":"aaaa000000000001","name":"TF-Node","type":"ADVERT"}')`, recent) + VALUES ('100102030440aabb', 'tf_offset', $1, 0, 4, '{"pubKey":"aaaa000000000001","name":"TF-Node","type":"ADVERT"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (1, 1, 10.0, -90, '[]', $1)`, recentEpoch) // Route type 3 (TRANSPORT_DIRECT): header=0x13 (payload_type=4, route_type=3) // 4 transport bytes + path byte at offset 5. // Path byte 0xC1 → hash_size bits = 11 → size 4, hop_count = 1 (not zero-hop) // Byte 1 = 0x05 → hash_size bits = 00 → size 1 if misread db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('1305060708C1bbcc', 'td_offset', ?, 3, 4, '{"pubKey":"aaaa000000000002","name":"TD-Node","type":"ADVERT"}')`, recent) + VALUES ('1305060708C1bbcc', 'td_offset', $1, 3, 4, '{"pubKey":"aaaa000000000002","name":"TD-Node","type":"ADVERT"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (2, 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (2, 1, 10.0, -90, '[]', $1)`, recentEpoch) // Route type 1 (FLOOD): header=0x11 (payload_type=4, route_type=1) // Path byte at offset 1. Path byte 0x80 → hash_size bits = 10 → size 3 db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('1180aabbccdd', 'flood_offset', ?, 1, 4, '{"pubKey":"aaaa000000000003","name":"Flood-Node","type":"ADVERT"}')`, recent) + VALUES ('1180aabbccdd', 'flood_offset', $1, 1, 4, '{"pubKey":"aaaa000000000003","name":"Flood-Node","type":"ADVERT"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (3, 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (3, 1, 10.0, -90, '[]', $1)`, recentEpoch) store := NewPacketStore(db, nil) store.Load() @@ -3376,7 +3411,7 @@ func TestHashSizeTransportDirectZeroHopSkipped(t *testing.T) { recentEpoch := now.Add(-1 * time.Hour).Unix() db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obs1', 'Obs', 'SJC', ?, '2026-01-01T00:00:00Z', 10)`, recent) + VALUES ('obs1', 'Obs', 'SJC', $1, '2026-01-01T00:00:00Z', 10)`, recent) // RouteDirect (2) zero-hop with an all-zero path byte: the sender wiped the // size bits, so there is nothing to read. Should be skipped. @@ -3384,23 +3419,23 @@ func TestHashSizeTransportDirectZeroHopSkipped(t *testing.T) { // a deliberate declaration — see TestZeroHopDirectAdvertDeclaredSizeIsEvidence // — so the skip is keyed on the byte's content, not on the route type.) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('1200aabbccdd', 'direct_zh', ?, 2, 4, '{"pubKey":"bbbb000000000001","name":"Direct-ZH","type":"ADVERT"}')`, recent) + VALUES ('1200aabbccdd', 'direct_zh', $1, 2, 4, '{"pubKey":"bbbb000000000001","name":"Direct-ZH","type":"ADVERT"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (1, 1, 10.0, -90, '[]', $1)`, recentEpoch) // RouteTransportDirect (3) zero-hop: 4 transport bytes + all-zero path byte // Should ALSO be skipped (this was the missing case in #747) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('130102030400aabb', 'tdirect_zh', ?, 3, 4, '{"pubKey":"bbbb000000000002","name":"TDirect-ZH","type":"ADVERT"}')`, recent) + VALUES ('130102030400aabb', 'tdirect_zh', $1, 3, 4, '{"pubKey":"bbbb000000000002","name":"TDirect-ZH","type":"ADVERT"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (2, 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (2, 1, 10.0, -90, '[]', $1)`, recentEpoch) // RouteDirect (2) non-zero-hop: path byte 0x41 → hop_count=1 // Should NOT be skipped db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('1241aabbccdd', 'direct_1h', ?, 2, 4, '{"pubKey":"bbbb000000000003","name":"Direct-1H","type":"ADVERT"}')`, recent) + VALUES ('1241aabbccdd', 'direct_1h', $1, 2, 4, '{"pubKey":"bbbb000000000003","name":"Direct-1H","type":"ADVERT"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (3, 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (3, 1, 10.0, -90, '[]', $1)`, recentEpoch) store := NewPacketStore(db, nil) store.Load() @@ -3436,10 +3471,10 @@ func TestAnalyticsHashSizesZeroHopSkip(t *testing.T) { recentEpoch := now.Add(-1 * time.Hour).Unix() db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obs1', 'Obs', 'SJC', ?, '2026-01-01T00:00:00Z', 10)`, recent) + VALUES ('obs1', 'Obs', 'SJC', $1, '2026-01-01T00:00:00Z', 10)`, recent) pk := "cccc000000000001" - db.conn.Exec(`INSERT INTO nodes (public_key, name, role) VALUES (?, 'ZH-Analytics', 'repeater')`, pk) + db.conn.Exec(`INSERT INTO nodes (public_key, name, role) VALUES ($1, 'ZH-Analytics', 'repeater')`, pk) decoded := `{"pubKey":"` + pk + `","name":"ZH-Analytics","type":"ADVERT"}` @@ -3447,16 +3482,16 @@ func TestAnalyticsHashSizesZeroHopSkip(t *testing.T) { // header 0x11 = route_type 1 (flood), payload_type 4 // pathByte 0x41 = hashSize bits 01 → size 2, hop_count 1 db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('1141aabbccdd', 'az_flood', ?, 1, 4, ?)`, recent, decoded) + VALUES ('1141aabbccdd', 'az_flood', $1, 1, 4, $2)`, recent, decoded) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 10.0, -90, '["aabb"]', ?)`, recentEpoch) + VALUES (1, 1, 10.0, -90, '["aabb"]', $1)`, recentEpoch) // Second: a direct zero-hop advert with pathByte=0x00 → would give hashSize=1 // header 0x12 = route_type 2 (direct), payload_type 4 db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('1200aabbccdd', 'az_direct', ?, 2, 4, ?)`, recent, decoded) + VALUES ('1200aabbccdd', 'az_direct', $1, 2, 4, $2)`, recent, decoded) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (2, 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (2, 1, 10.0, -90, '[]', $1)`, recentEpoch) store := NewPacketStore(db, nil) store.Load() @@ -3590,32 +3625,32 @@ func TestGetChannelMessagesDedupeRepeats(t *testing.T) { recentEpoch := now.Add(-1 * time.Hour).Unix() db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obs1', 'Obs1', 'SJC', ?, '2026-01-01T00:00:00Z', 10)`, recent) + VALUES ('obs1', 'Obs1', 'SJC', $1, '2026-01-01T00:00:00Z', 10)`, recent) db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obs2', 'Obs2', 'LAX', ?, '2026-01-01T00:00:00Z', 10)`, recent) + VALUES ('obs2', 'Obs2', 'LAX', $1, '2026-01-01T00:00:00Z', 10)`, recent) // Insert two copies of same CHAN message (same hash, different observers) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('CC01', 'dedup_chan_1', ?, 1, 5, '{"type":"CHAN","channel":"#general","text":"Alice: hello","sender":"Alice"}')`, recent) + VALUES ('CC01', 'dedup_chan_1', $1, 1, 5, '{"type":"CHAN","channel":"#general","text":"Alice: hello","sender":"Alice"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 12.0, -88, '["aa"]', ?)`, recentEpoch) + VALUES (1, 1, 12.0, -88, '["aa"]', $1)`, recentEpoch) // Same sender + hash → different observation (simulates dedup) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('CC02', 'dedup_chan_1', ?, 1, 5, '{"type":"CHAN","channel":"#general","text":"Alice: hello","sender":"Alice"}')`, recent) + VALUES ('CC02', 'dedup_chan_1', $1, 1, 5, '{"type":"CHAN","channel":"#general","text":"Alice: hello","sender":"Alice"}')`, recent) // Note: won't load due to UNIQUE constraint on hash → tests the code path with single tx having multiple obs // Second different message db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('CC03', 'dedup_chan_2', ?, 1, 5, '{"type":"CHAN","channel":"#general","text":"Bob: world","sender":"Bob"}')`, recent) + VALUES ('CC03', 'dedup_chan_2', $1, 1, 5, '{"type":"CHAN","channel":"#general","text":"Bob: world","sender":"Bob"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (2, 2, 10.0, -90, '["bb"]', ?)`, recentEpoch) + VALUES (2, 2, 10.0, -90, '["bb"]', $1)`, recentEpoch) // GRP_TXT (not CHAN) — should be skipped by GetChannelMessages db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('DD01', 'grp_msg_hash_1', ?, 1, 5, '{"type":"GRP_TXT","channelHash":"42","text":"encrypted"}')`, recent) + VALUES ('DD01', 'grp_msg_hash_1', $1, 1, 5, '{"type":"GRP_TXT","channelHash":"42","text":"encrypted"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (3, 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (3, 1, 10.0, -90, '[]', $1)`, recentEpoch) store := NewPacketStore(db, nil) store.Load() @@ -3937,16 +3972,16 @@ func TestIngestNewFromDBDuplicateObs(t *testing.T) { // Insert new transmission with same hash as existing (should merge into existing tx) now := time.Now().UTC().Format(time.RFC3339) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('AABB', 'dedup_test_hash_01', ?, 1, 4, '{"pubKey":"aabbccdd11223344","type":"ADVERT"}')`, now) + VALUES ('AABB', 'dedup_test_hash_01', $1, 1, 4, '{"pubKey":"aabbccdd11223344","type":"ADVERT"}')`, now) newTxID := 0 db.conn.QueryRow("SELECT MAX(id) FROM transmissions").Scan(&newTxID) // Add observation db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (?, 1, 11.0, -89, '["dd"]', ?)`, newTxID, time.Now().Unix()) + VALUES ($1, 1, 11.0, -89, '["dd"]', $2)`, newTxID, time.Now().Unix()) // Add duplicate observation (same observer_id + path_json) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (?, 1, 11.0, -89, '["dd"]', ?)`, newTxID, time.Now().Unix()) + VALUES ($1, 1, 11.0, -89, '["dd"]', $2)`, newTxID, time.Now().Unix()) _, newMax := store.IngestNewFromDB(initialMax, 100) if newMax <= initialMax { @@ -3979,7 +4014,7 @@ func TestIngestNewObservations(t *testing.T) { // Simulate a new observation arriving for the existing transmission AFTER // the poller has already advanced past its transmission ID db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 2, 5.0, -100, '["aa","bb","cc"]', ?)`, time.Now().Unix()) + VALUES (1, 2, 5.0, -100, '["aa","bb","cc"]', $1)`, time.Now().Unix()) // Verify IngestNewFromDB does NOT pick up the new observation (tx id hasn't changed) txMax := store.MaxTransmissionID() @@ -4017,7 +4052,7 @@ func TestIngestNewObservations(t *testing.T) { t.Run("dedup by observer+path", func(t *testing.T) { // Insert duplicate observation (same observer + path as existing) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 12.5, -90, '["aa","bb"]', ?)`, time.Now().Unix()) + VALUES (1, 1, 12.5, -90, '["aa","bb"]', $1)`, time.Now().Unix()) prevCount := initialTx.ObservationCount maps := store.IngestNewObservations(db.GetMaxObservationID()-1, 500) if initialTx.ObservationCount != prevCount { @@ -4051,7 +4086,7 @@ func TestIngestNewObservationsV2(t *testing.T) { // Add new observation for existing transmission db.conn.Exec(`INSERT INTO observations (transmission_id, observer_id, observer_name, snr, rssi, path_json, timestamp) - VALUES (1, 'obs2', 'Obs Two', 6.0, -98, '["dd","ee"]', ?)`, time.Now().Unix()) + VALUES (1, 'obs2', 'Obs Two', 6.0, -98, '["dd","ee"]', $1)`, time.Now().Unix()) newMaps := store.IngestNewObservations(maxObsID, 500) if len(newMaps) != 1 { @@ -4103,7 +4138,7 @@ func TestHandleNodeAnalyticsNameless(t *testing.T) { // Insert a node without a name db.conn.Exec(`INSERT INTO nodes (public_key, role, lat, lon, last_seen, first_seen, advert_count) - VALUES ('nameless_node_pk_1', 'repeater', 37.5, -122.0, ?, '2026-01-01', 1)`, + VALUES ('nameless_node_pk_1', 'repeater', 37.5, -122.0, $1, '2026-01-01', 1)`, time.Now().UTC().Format(time.RFC3339)) cfg := &Config{Port: 3000} @@ -4227,8 +4262,8 @@ func TestGetDBSizeStatsMemory(t *testing.T) { seedTestData(t, db) stats := db.GetDBSizeStats() - if stats["dbSizeMB"] != float64(0) { - t.Errorf("expected 0 for in-memory, got %v", stats["dbSizeMB"]) + if stats["dbSizeMB"].(float64) < 0 || (testBackendValue() == "postgres" && stats["dbSizeMB"].(float64) == 0) { + t.Errorf("expected positive PostgreSQL database size, got %v", stats["dbSizeMB"]) } } @@ -4251,11 +4286,11 @@ func TestGetChannelMessagesAfterIngest(t *testing.T) { now := time.Now().UTC() nowStr := now.Format(time.RFC3339) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('FF01', 'newchannelmsg19800', ?, 1, 5, '{"type":"CHAN","channel":"#test","text":"NewUser: brand new message","sender":"NewUser"}')`, nowStr) + VALUES ('FF01', 'newchannelmsg19800', $1, 1, 5, '{"type":"CHAN","channel":"#test","text":"NewUser: brand new message","sender":"NewUser"}')`, nowStr) newTxID := 0 db.conn.QueryRow("SELECT MAX(id) FROM transmissions").Scan(&newTxID) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (?, 1, 12.0, -88, '[]', ?)`, newTxID, now.Unix()) + VALUES ($1, 1, 12.0, -88, '[]', $2)`, newTxID, now.Unix()) // Ingest the new data _, newMax := store.IngestNewFromDB(initialMax, 100) @@ -4549,8 +4584,8 @@ func TestBuildTransmissionWhereMultiObserver(t *testing.T) { t.Fatalf("expected 1 WHERE clause, got %d", len(where)) } clause := where[0] - if !strings.Contains(clause, "IN (?,?)") { - t.Errorf("expected IN (?,?) in clause, got: %s", clause) + if !strings.Contains(clause, "IN ("+db.parameter(1)+","+db.parameter(2)+")") { + t.Errorf("expected IN ($1,$2) in clause, got: %s", clause) } if len(args) != 2 { t.Fatalf("expected 2 args, got %d", len(args)) @@ -4577,8 +4612,8 @@ func TestBuildTransmissionWhereMultiObserver(t *testing.T) { if len(where) != 1 { t.Fatalf("expected 1 WHERE clause, got %d", len(where)) } - if !strings.Contains(where[0], "IN (?)") { - t.Errorf("expected IN (?) for single observer, got: %s", where[0]) + if !strings.Contains(where[0], "IN ("+db.parameter(1)+")") { + t.Errorf("expected IN ($1) for single observer, got: %s", where[0]) } if len(args) != 1 || args[0] != "obs1" { t.Errorf("expected [obs1], got %v", args) @@ -4602,7 +4637,7 @@ func TestDistanceIncrementalUpdate(t *testing.T) { // Insert a new observation with a different path to trigger an incremental update. maxObsID := db.GetMaxObservationID() db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 2, 5.0, -100, '["xx","yy","zz"]', ?)`, time.Now().Unix()) + VALUES (1, 2, 5.0, -100, '["xx","yy","zz"]', $1)`, time.Now().Unix()) store.IngestNewObservations(maxObsID, 500) @@ -4614,7 +4649,7 @@ func TestDistanceIncrementalUpdate(t *testing.T) { // Insert another observation with yet another path. maxObsID = db.GetMaxObservationID() db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 2, 7.0, -95, '["aa","bb","cc","dd"]', ?)`, time.Now().Unix()) + VALUES (1, 2, 7.0, -95, '["aa","bb","cc","dd"]', $1)`, time.Now().Unix()) store.IngestNewObservations(maxObsID, 500) @@ -4714,11 +4749,11 @@ func TestIngestTraceBroadcastIncludesPath(t *testing.T) { traceHex := "2502AABB010000000200000000DEADBEEF" now := time.Now().UTC().Format(time.RFC3339) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES (?, 'tracehash683test', ?, 1, 9, '')`, traceHex, now) + VALUES ($1, 'tracehash683test', $2, 1, 9, '')`, traceHex, now) newTxID := 0 db.conn.QueryRow("SELECT MAX(id) FROM transmissions").Scan(&newTxID) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (?, 1, 5.0, -100, '["aa"]', ?)`, newTxID, time.Now().Unix()) + VALUES ($1, 1, 5.0, -100, '["aa"]', $2)`, newTxID, time.Now().Unix()) broadcastMaps, _ := store.IngestNewFromDB(initialMax, 100) if len(broadcastMaps) < 1 { diff --git a/cmd/server/db.go b/cmd/server/db.go index 980099459..b0829d0c8 100644 --- a/cmd/server/db.go +++ b/cmd/server/db.go @@ -7,15 +7,16 @@ import ( "fmt" "log" "math" - "os" "sort" "strings" "sync" + "sync/atomic" "time" - "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/dbconfig" "github.com/meshcore-analyzer/geofilter" - _ "modernc.org/sqlite" + "github.com/meshcore-analyzer/packetpath" + "golang.org/x/sync/singleflight" ) // routeTypeTransport covers TRANSPORT_FLOOD (0) and TRANSPORT_DIRECT (3) — @@ -33,26 +34,59 @@ const routeTypeTransportSQL = "route_type IN (0, 3)" // per MeshCore protocol (#1838). const routeTypeNonTransportSQL = "route_type IN (1, 2)" -// DB wraps a read-only connection to the MeshCore SQLite database. +// DB wraps a read-only connection to the selected telemetry database. type DB struct { - conn *sql.DB - path string // filesystem path to the database file - isV3 bool // v3 schema: observer_idx in observations (vs observer_id in v2) - hasResolvedPath bool // observations table has resolved_path column - hasObsRawHex bool // observations table has raw_hex column (#881) - hasScopeName bool // transmissions.scope_name column exists (#899) - hasDefaultScope bool // nodes.default_scope column exists (#899) - hasMultibyteSupCols bool // nodes/inactive_nodes have multibyte_sup/multibyte_evidence (#903) - hasLastSeen bool // transmissions.last_seen column exists (#1690) + backend dbconfig.Backend + advertEvidenceTable atomic.Bool + advertEvidenceReadHook func() // test-only: immediately before a bulk mask query + conn *sql.DB + path string // PostgreSQL URL; never log or use as a filesystem path + stateDir string // local queues and small sidecars + isV3 bool // v3 schema: observer_idx in observations (vs observer_id in v2) + hasResolvedPath bool // observations table has resolved_path column + hasObsRawHex bool // observations table has raw_hex column (#881) + hasScopeName bool // transmissions.scope_name column exists (#899) + hasDefaultScope bool // nodes.default_scope column exists (#899) + hasConfiguredScope bool // nodes.configured_scope column exists (#1865) + hasDeclaredRegionsTable bool // node_declared_regions table exists at startup (#1975); read via declaredRegionsTablePresent + hasMultibyteSupCols bool // nodes/inactive_nodes have multibyte_sup/multibyte_evidence (#903) + hasLastSeen bool // transmissions.last_seen column exists (#1690) + + // declaredRegionsTableLate latches true once node_declared_regions is found + // after startup. The ingestor creates that table, and the two processes + // start together, so on the first run of a build that adds it the server's + // startup probe can lose the race. See declaredRegionsTablePresent. + declaredRegionsTableLate atomic.Bool + + pgStatsMu sync.Mutex + pgStatsCache postgresDatabaseSample + pgStatsExpires time.Time + pgStatsSF singleflight.Group + pgStatsMissHook func() // test-only: after an initial cache miss + pgStatsQueryHook func() // test-only: immediately before the shared sample query // Channel list caches, keyed by region param — avoids repeated GROUP BY // scans (#762). Keyed per-region (not a single slot) so mixed-region // traffic doesn't evict and re-run the query on every request. channelsCacheMu sync.Mutex channelsCache map[string]channelsCacheEntry + // Collapses the TTL-boundary herd (same #1910 pattern as statsSF/ + // regionMembershipSF): without this, every request that arrives while + // the cache is expired reruns the region-scoped GROUP BY scan itself. + // Measured live: a single cold request ran 3-5s, 5 concurrent requests + // for the same region ran ~8s each, all serialized against each other. + channelsSF singleflight.Group + // Test-only hook fired immediately before the real GetChannels query + // executes inside channelsSF's flight (i.e. once per coalesced miss, + // not once per caller). Nil in production. See db_singleflight_test.go. + channelsQueryHook func() encChannelsCacheMu sync.Mutex encChannelsCache map[string]channelsCacheEntry + encChannelsSF singleflight.Group + // Test-only hook, same contract as channelsQueryHook but for + // GetEncryptedChannels. Nil in production. + encChannelsQueryHook func() // Channel messages cache, keyed by hash+limit+offset+region. Unlike // GetChannels, this previously had no cache at all — every page @@ -101,45 +135,14 @@ type channelMessagesCacheEntry struct { exp time.Time } -// OpenDB opens a read-only SQLite connection with WAL mode. -func OpenDB(path string) (*DB, error) { - dsn := fmt.Sprintf("file:%s?mode=ro&_journal_mode=WAL&_busy_timeout=5000", path) - conn, err := sql.Open("sqlite", dsn) - if err != nil { - return nil, err - } - conn.SetMaxOpenConns(4) - conn.SetMaxIdleConns(2) - if err := conn.Ping(); err != nil { - conn.Close() - return nil, fmt.Errorf("ping failed: %w", err) - } - d := &DB{conn: conn, path: path} - // Detect the on-disk schema on a single pinned connection and fail loudly if - // it cannot be probed. A swallowed detection failure would silently cache the - // wrong schema mode (isV3=false against a v3 DB) for the entire process - // lifetime, breaking every read path until a manual restart (#1901). Aborting - // here lets the supervisor restart us, which clears any transient cause (e.g. - // a WAL recovery racing the read-only open). - ctx := context.Background() - sc, err := conn.Conn(ctx) - if err != nil { - conn.Close() - return nil, fmt.Errorf("schema detection: acquire connection: %w", err) +// OpenDB preserves unambiguous legacy path/URL callers. Runtime main uses +// explicit Storage from the shared selection record. +func OpenDB(target string) (*DB, error) { + storage := dbconfig.Storage{Backend: dbconfig.SQLite, DBPath: target} + if strings.HasPrefix(target, "postgres://") || strings.HasPrefix(target, "postgresql://") { + storage = dbconfig.Storage{Backend: dbconfig.Postgres, ReaderDatabaseURL: target} } - derr := d.detectSchema(ctx, sc) - _ = sc.Close() - if derr != nil { - conn.Close() - return nil, fmt.Errorf("schema detection failed: %w", derr) - } - // Statements are prepared after schema detection so they can never be - // compiled against a schema mode that turned out to be wrong (#1901). - if err := d.prepareStatements(); err != nil { - conn.Close() - return nil, fmt.Errorf("prepare statements: %w", err) - } - return d, nil + return OpenStorage(storage) } // stmtQueryRow returns a QueryRow-like helper that uses the prepared statement @@ -153,7 +156,7 @@ func (db *DB) stmtQueryRow(stmt *sql.Stmt, fallbackSQL string, args ...interface } // prepareStatements creates prepared statements for frequently-called queries. -// SQLite compiles and caches the query plan once, avoiding re-parsing per request. +// PostgreSQL prepares these queries once per connection. func (db *DB) prepareStatements() error { var err error prepare := func(query string) *sql.Stmt { @@ -167,15 +170,15 @@ func (db *DB) prepareStatements() error { db.stmtCountTransmissions = prepare("SELECT COUNT(*) FROM transmissions") db.stmtCountObservations = prepare("SELECT COUNT(*) FROM observations") - db.stmtCountNodesActive = prepare("SELECT COUNT(*) FROM nodes WHERE last_seen > ?") + db.stmtCountNodesActive = prepare("SELECT COUNT(*) FROM nodes WHERE last_seen > " + db.parameter(1)) db.stmtCountNodesAll = prepare("SELECT COUNT(*) FROM nodes") db.stmtCountObservers = prepare("SELECT COUNT(*) FROM observers WHERE inactive IS NULL OR inactive = 0") - db.stmtCountObsLastHour = prepare("SELECT COUNT(*) FROM observations WHERE timestamp > ?") - db.stmtCountObsLastDay = prepare("SELECT COUNT(*) FROM observations WHERE timestamp > ?") - db.stmtNodeLookup = prepare("SELECT public_key FROM nodes WHERE public_key = ? OR name = ? LIMIT 1") - db.stmtTxByHash = prepare("SELECT id FROM transmissions WHERE hash = ?") - db.stmtCountNodesByRole = prepare("SELECT COUNT(*) FROM nodes WHERE role = ? AND last_seen > ?") - db.stmtCountNodesByRoleAll = prepare("SELECT COUNT(*) FROM nodes WHERE role = ?") + db.stmtCountObsLastHour = prepare("SELECT COUNT(*) FROM observations WHERE timestamp > " + db.parameter(1)) + db.stmtCountObsLastDay = prepare("SELECT COUNT(*) FROM observations WHERE timestamp > " + db.parameter(1)) + db.stmtNodeLookup = prepare("SELECT public_key FROM nodes WHERE public_key = " + db.parameter(1) + " OR name = " + db.parameter(2) + " LIMIT 1") + db.stmtTxByHash = prepare("SELECT id FROM transmissions WHERE hash = " + db.parameter(1)) + db.stmtCountNodesByRole = prepare("SELECT COUNT(*) FROM nodes WHERE role = " + db.parameter(1) + " AND last_seen > " + db.parameter(2)) + db.stmtCountNodesByRoleAll = prepare("SELECT COUNT(*) FROM nodes WHERE role = " + db.parameter(1)) db.stmtMaxTxID = prepare("SELECT COALESCE(MAX(id), 0) FROM transmissions") db.stmtMaxObsID = prepare("SELECT COALESCE(MAX(id), 0) FROM observations") return err @@ -196,11 +199,6 @@ func (db *DB) Close() error { s.Close() } } - // No WAL checkpoint here. The connection is opened read-only (mode=ro in - // OpenDB), so PRAGMA wal_checkpoint(TRUNCATE) always fails with "disk I/O - // error (778)" on it. Attempting it emitted a misleading storage-fault line - // on every shutdown that wasted incident investigation time (#1901); the - // ingestor (the writer) owns checkpointing. return db.conn.Close() } @@ -209,11 +207,11 @@ func (db *DB) Close() error { // It returns an error if any probe query fails. Previously these failures were // swallowed with a bare return, leaving isV3 (and the feature flags) at their // zero value for the whole process lifetime: a single transient failure of the -// first PRAGMA silently ran v2 SQL against a v3 database until the process was +// first schema probe silently ran v2 SQL against a v3 database until the process was // restarted (#1901). Detection is now all-or-nothing — on any probe error the // caller aborts startup so the supervisor can retry. func (db *DB) detectSchema(ctx context.Context, q rowQuerier) error { - obs, err := schemaColumns(ctx, q, "observations") + obs, err := schemaColumns(ctx, q, "observations", db.Backend()) if err != nil { return fmt.Errorf("probe observations: %w", err) } @@ -221,19 +219,27 @@ func (db *DB) detectSchema(ctx context.Context, q rowQuerier) error { db.hasResolvedPath = obs["resolved_path"] db.hasObsRawHex = obs["raw_hex"] - tx, err := schemaColumns(ctx, q, "transmissions") + tx, err := schemaColumns(ctx, q, "transmissions", db.Backend()) if err != nil { return fmt.Errorf("probe transmissions: %w", err) } db.hasScopeName = tx["scope_name"] db.hasLastSeen = tx["last_seen"] - nodes, err := schemaColumns(ctx, q, "nodes") + nodes, err := schemaColumns(ctx, q, "nodes", db.Backend()) if err != nil { return fmt.Errorf("probe nodes: %w", err) } db.hasDefaultScope = nodes["default_scope"] db.hasMultibyteSupCols = nodes["multibyte_sup"] + db.hasConfiguredScope = nodes["configured_scope"] + + // #1975: an optional second confirmed-scope source. Absent on a stock + // install, so schemaColumns returns nothing and the flag stays false; + // present on deployments that collect the same fact by another route. + // A missing table is not an error here. + ndr, ndrErr := schemaColumns(ctx, q, "node_declared_regions", db.Backend()) + db.hasDeclaredRegionsTable = ndrErr == nil && len(ndr) > 0 if db.isV3 { log.Printf("[db] schema mode: v3 (observer_idx)") @@ -250,24 +256,22 @@ type rowQuerier interface { QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) } -// schemaColumns returns the set of column names present on the given table via -// PRAGMA table_info. Unlike the previous inline scans, a query or scan error is -// returned rather than swallowed (#1901). The table name is a trusted literal -// supplied by the caller, not user input. -func schemaColumns(ctx context.Context, q rowQuerier, table string) (map[string]bool, error) { - rows, err := q.QueryContext(ctx, fmt.Sprintf("PRAGMA table_info(%s)", table)) +// schemaColumns reads visible columns through the selected native catalog. +func schemaColumns(ctx context.Context, q rowQuerier, table string, backend ...dbconfig.Backend) (map[string]bool, error) { + query := `SELECT column_name FROM information_schema.columns + WHERE table_schema = current_schema() AND table_name = $1` + if len(backend) > 0 && backend[0] == dbconfig.SQLite { + query = `SELECT name FROM pragma_table_info(?1)` + } + rows, err := q.QueryContext(ctx, query, table) if err != nil { return nil, err } defer rows.Close() cols := make(map[string]bool) for rows.Next() { - var cid int var name string - var ctype sql.NullString - var notnull, pk int - var dflt sql.NullString - if err := rows.Scan(&cid, &name, &ctype, ¬null, &dflt, &pk); err != nil { + if err := rows.Scan(&name); err != nil { return nil, err } cols[name] = true @@ -282,6 +286,10 @@ func (db *DB) nodeSelectCols() string { if db.hasDefaultScope { cols += ", default_scope" } + // #1865: confirmed scopes appended after default_scope; scan order must match. + if db.hasConfiguredScope { + cols += ", configured_scope, configured_scope_at" + } return cols } @@ -294,7 +302,7 @@ func (db *DB) transmissionBaseSQL() (selectCols, observerJoin string) { o.snr, o.rssi, o.path_json, o.direction` observerJoin = `LEFT JOIN observations o ON o.id = ( SELECT id FROM observations WHERE transmission_id = t.id - ORDER BY length(COALESCE(path_json,'')) DESC LIMIT 1 + ORDER BY length(COALESCE(path_json,'')) DESC NULLS LAST LIMIT 1 ) LEFT JOIN observers obs ON obs.rowid = o.observer_idx` } else { @@ -304,7 +312,7 @@ func (db *DB) transmissionBaseSQL() (selectCols, observerJoin string) { o.snr, o.rssi, o.path_json, o.direction` observerJoin = `LEFT JOIN observations o ON o.id = ( SELECT id FROM observations WHERE transmission_id = t.id - ORDER BY length(COALESCE(path_json,'')) DESC LIMIT 1 + ORDER BY length(COALESCE(path_json,'')) DESC NULLS LAST LIMIT 1 ) LEFT JOIN observers obs2 ON obs2.id = o.observer_id` } @@ -426,7 +434,7 @@ type Observation struct { Direction *string `json:"direction"` SNR *float64 `json:"snr"` RSSI *float64 `json:"rssi"` - Score *int `json:"score"` + Score *float64 `json:"score"` Hash *string `json:"hash"` RouteType *int `json:"route_type"` PayloadType *int `json:"payload_type"` @@ -448,6 +456,16 @@ type Stats struct { PacketsLast24h int `json:"packetsLast24h"` } +// CountActiveNodes counts nodes seen in the last 7 days: GetStats' totalNodes +// on its own, without GetStats' whole-table counts. +func (db *DB) CountActiveNodes() (int, error) { + // Node.js uses 7-day active nodes for totalNodes + sevenDaysAgo := time.Now().Add(-7 * 24 * time.Hour).Format(time.RFC3339) + var n int + err := db.stmtQueryRow(db.stmtCountNodesActive, "SELECT COUNT(*) FROM nodes WHERE last_seen > "+db.parameter(1), sevenDaysAgo).Scan(&n) + return n, err +} + // GetStats returns aggregate counts (matches Node.js db.getStats shape). func (db *DB) GetStats() (*Stats, error) { s := &Stats{} @@ -458,129 +476,64 @@ func (db *DB) GetStats() (*Stats, error) { s.TotalPackets = s.TotalTransmissions db.stmtQueryRow(db.stmtCountObservations, "SELECT COUNT(*) FROM observations").Scan(&s.TotalObservations) - // Node.js uses 7-day active nodes for totalNodes - sevenDaysAgo := time.Now().Add(-7 * 24 * time.Hour).Format(time.RFC3339) - db.stmtQueryRow(db.stmtCountNodesActive, "SELECT COUNT(*) FROM nodes WHERE last_seen > ?", sevenDaysAgo).Scan(&s.TotalNodes) + s.TotalNodes, _ = db.CountActiveNodes() db.stmtQueryRow(db.stmtCountNodesAll, "SELECT COUNT(*) FROM nodes").Scan(&s.TotalNodesAllTime) db.stmtQueryRow(db.stmtCountObservers, "SELECT COUNT(*) FROM observers WHERE inactive IS NULL OR inactive = 0").Scan(&s.TotalObservers) oneHourAgo := time.Now().Add(-1 * time.Hour).Unix() - db.stmtQueryRow(db.stmtCountObsLastHour, "SELECT COUNT(*) FROM observations WHERE timestamp > ?", oneHourAgo).Scan(&s.PacketsLastHour) + db.stmtQueryRow(db.stmtCountObsLastHour, "SELECT COUNT(*) FROM observations WHERE timestamp > "+db.parameter(1), oneHourAgo).Scan(&s.PacketsLastHour) oneDayAgo := time.Now().Add(-24 * time.Hour).Unix() - db.stmtQueryRow(db.stmtCountObsLastDay, "SELECT COUNT(*) FROM observations WHERE timestamp > ?", oneDayAgo).Scan(&s.PacketsLast24h) + db.stmtQueryRow(db.stmtCountObsLastDay, "SELECT COUNT(*) FROM observations WHERE timestamp > "+db.parameter(1), oneDayAgo).Scan(&s.PacketsLast24h) return s, nil } -// GetDBSizeStats returns SQLite file sizes and row counts (matching Node.js /api/perf sqlite shape). +// GetDBSizeStats retains the legacy map caller with native database metrics. func (db *DB) GetDBSizeStats() map[string]interface{} { - result := map[string]interface{}{} - - // DB file size - var dbSizeMB float64 - if db.path != "" && db.path != ":memory:" { - if info, err := os.Stat(db.path); err == nil { - dbSizeMB = math.Round(float64(info.Size())/1048576*10) / 10 - } + stats := db.GetDBSizeStatsTyped() + var size any + var rows map[string]int + if stats.DbSizeMB != nil { + size = *stats.DbSizeMB } - result["dbSizeMB"] = dbSizeMB - - // WAL file size - var walSizeMB float64 - if db.path != "" && db.path != ":memory:" { - if info, err := os.Stat(db.path + "-wal"); err == nil { - walSizeMB = math.Round(float64(info.Size())/1048576*10) / 10 - } + if stats.Rows != nil { + rows = map[string]int{"transmissions": stats.Rows.Transmissions, + "observations": stats.Rows.Observations, "nodes": stats.Rows.Nodes, + "observers": stats.Rows.Observers} } - result["walSizeMB"] = walSizeMB - - // Freelist size via PRAGMA (matches Node.js: page_size * freelist_count) - var pageSize, freelistCount int64 - db.conn.QueryRow("PRAGMA page_size").Scan(&pageSize) - db.conn.QueryRow("PRAGMA freelist_count").Scan(&freelistCount) - freelistMB := math.Round(float64(pageSize*freelistCount)/1048576*10) / 10 - result["freelistMB"] = freelistMB - - // WAL checkpoint info (matches Node.js: PRAGMA wal_checkpoint(PASSIVE)) - var walBusy, walLog, walCheckpointed int - err := db.conn.QueryRow("PRAGMA wal_checkpoint(PASSIVE)").Scan(&walBusy, &walLog, &walCheckpointed) - if err == nil { - result["walPages"] = map[string]interface{}{ - "total": walLog, - "checkpointed": walCheckpointed, - "busy": walBusy, - } - } else { - result["walPages"] = map[string]interface{}{ - "total": 0, - "checkpointed": 0, - "busy": 0, - } + result := map[string]interface{}{ + "engine": stats.Engine, "dbSizeMB": size, "rows": rows, "error": stats.Error, } - - // Row counts per table - rows := map[string]int{} - for _, table := range []string{"transmissions", "observations", "nodes", "observers"} { - var count int - db.conn.QueryRow("SELECT COUNT(*) FROM " + table).Scan(&count) - rows[table] = count + if stats.Engine == "sqlite" { + result["walSizeMB"] = stats.WalSizeMB + result["freelistMB"] = stats.FreelistMB } - result["rows"] = rows - return result } -// GetDBSizeStatsTyped returns SQLite file sizes and row counts as a typed struct. -func (db *DB) GetDBSizeStatsTyped() SqliteStats { - result := SqliteStats{} - - if db.path != "" && db.path != ":memory:" { - if info, err := os.Stat(db.path); err == nil { - result.DbSizeMB = math.Round(float64(info.Size())/1048576*10) / 10 - } +// GetDBSizeStatsTyped reports native database size and table counts. +// WAL is cluster-wide and cannot be attributed to this database by a reader. +func (db *DB) GetDBSizeStatsTyped() PostgresStats { + sample, err := db.postgresStats(context.Background()) + result := PostgresStats{ + Engine: "postgresql", SampledAt: postgresSampleStamp(sample), + SampleIntervalSeconds: int(postgresStatsTTL / time.Second), Stale: err != nil, } - - if db.path != "" && db.path != ":memory:" { - if info, err := os.Stat(db.path + "-wal"); err == nil { - result.WalSizeMB = math.Round(float64(info.Size())/1048576*10) / 10 - } + if db.Backend() == dbconfig.SQLite { + result.Engine = "sqlite" } - - var pageSize, freelistCount int64 - db.conn.QueryRow("PRAGMA page_size").Scan(&pageSize) - db.conn.QueryRow("PRAGMA freelist_count").Scan(&freelistCount) - result.FreelistMB = math.Round(float64(pageSize*freelistCount)/1048576*10) / 10 - - var walBusy, walLog, walCheckpointed int - err := db.conn.QueryRow("PRAGMA wal_checkpoint(PASSIVE)").Scan(&walBusy, &walLog, &walCheckpointed) - if err == nil { - result.WalPages = &WalPages{ - Total: walLog, - Checkpointed: walCheckpointed, - Busy: walBusy, - } - } else { - result.WalPages = &WalPages{} + if sample.SampledAt.IsZero() { + result.Error = "database diagnostics unavailable" + return result } - - rows := &SqliteRowCounts{} - for _, table := range []string{"transmissions", "observations", "nodes", "observers"} { - var count int - db.conn.QueryRow("SELECT COUNT(*) FROM " + table).Scan(&count) - switch table { - case "transmissions": - rows.Transmissions = count - case "observations": - rows.Observations = count - case "nodes": - rows.Nodes = count - case "observers": - rows.Observers = count - } + mb := math.Round(float64(sample.DatabaseBytes)/1048576*10) / 10 + result.DbSizeMB, result.Rows = &mb, &sample.Rows + if sample.SQLite != nil { + walMB := float64(sample.SQLite.WalSize) / 1048576 + freeMB := float64(sample.SQLite.FreelistCount*sample.SQLite.PageSize) / 1048576 + result.WalSizeMB, result.FreelistMB = &walMB, &freeMB } - result.Rows = rows - return result } @@ -590,7 +543,7 @@ func (db *DB) GetRoleCounts() map[string]int { counts := map[string]int{} for _, role := range []string{"repeater", "room", "companion", "sensor"} { var c int - db.stmtQueryRow(db.stmtCountNodesByRole, "SELECT COUNT(*) FROM nodes WHERE role = ? AND last_seen > ?", role, sevenDaysAgo).Scan(&c) + db.stmtQueryRow(db.stmtCountNodesByRole, "SELECT COUNT(*) FROM nodes WHERE role = "+db.parameter(1)+" AND last_seen > "+db.parameter(2), role, sevenDaysAgo).Scan(&c) counts[role+"s"] = c } return counts @@ -601,7 +554,7 @@ func (db *DB) GetAllRoleCounts() map[string]int { counts := map[string]int{} for _, role := range []string{"repeater", "room", "companion", "sensor"} { var c int - db.stmtQueryRow(db.stmtCountNodesByRoleAll, "SELECT COUNT(*) FROM nodes WHERE role = ?", role).Scan(&c) + db.stmtQueryRow(db.stmtCountNodesByRoleAll, "SELECT COUNT(*) FROM nodes WHERE role = "+db.parameter(1), role).Scan(&c) counts[role+"s"] = c } return counts @@ -635,6 +588,7 @@ type PacketResult struct { // QueryPackets returns paginated, filtered packets as transmissions (matching Node.js shape). func (db *DB) QueryPackets(q PacketQuery) (*PacketResult, error) { + q.Offset = max(0, q.Offset) if q.Limit <= 0 { q.Limit = 50 } @@ -664,8 +618,8 @@ func (db *DB) QueryPackets(q PacketQuery) (*PacketResult, error) { // is what the packets page is showing. The `since=` filter still uses // first_seen / observation timestamp, preserving "received-by-radio since X." selectCols, observerJoin := db.transmissionBaseSQL() - querySQL := fmt.Sprintf("SELECT %s FROM transmissions t %s %s ORDER BY t.id %s LIMIT ? OFFSET ?", - selectCols, observerJoin, w, q.Order) + querySQL := fmt.Sprintf("SELECT %s FROM transmissions t %s %s ORDER BY t.id %s LIMIT %s OFFSET %s", + selectCols, observerJoin, w, q.Order, db.parameter(len(args)+1), db.parameter(len(args)+2)) qArgs := make([]interface{}, len(args)) copy(qArgs, args) @@ -690,6 +644,7 @@ func (db *DB) QueryPackets(q PacketQuery) (*PacketResult, error) { // QueryGroupedPackets groups by hash (transmissions) — queries transmissions table directly for performance. func (db *DB) QueryGroupedPackets(q PacketQuery) (*PacketResult, error) { + q.Offset = max(0, q.Offset) if q.Limit <= 0 { q.Limit = 50 } @@ -708,50 +663,7 @@ func (db *DB) QueryGroupedPackets(q PacketQuery) (*PacketResult, error) { db.conn.QueryRow(fmt.Sprintf("SELECT COUNT(*) FROM transmissions t %s", w), args...).Scan(&total) } - // Build grouped query using transmissions table with correlated subqueries. - // #1189 R2: distinct_iatas is a NEW column — comma-separated DISTINCT IATA - // codes across all observers of the transmission, with empty/NULL IATAs - // excluded. Frontend needs this on the DEFAULT COLLAPSED VIEW (where - // p._children is empty), so we compute it server-side. - // - // scope_name lives on the transmission row, so appending it as the last - // selected column is safe for both query shapes. - scopeNameCol := "" - if db.hasScopeName { - scopeNameCol = ", t.scope_name" - } - var querySQL string - if db.isV3 { - querySQL = fmt.Sprintf(`SELECT t.hash, t.first_seen, t.raw_hex, t.decoded_json, t.payload_type, t.route_type, - COALESCE((SELECT COUNT(*) FROM observations oi WHERE oi.transmission_id = t.id), 0) AS count, - COALESCE((SELECT COUNT(DISTINCT oi.observer_idx) FROM observations oi WHERE oi.transmission_id = t.id), 0) AS observer_count, - COALESCE((SELECT MAX(strftime('%%Y-%%m-%%dT%%H:%%M:%%fZ', oi.timestamp, 'unixepoch')) FROM observations oi WHERE oi.transmission_id = t.id), t.first_seen) AS latest, - obs.id AS observer_id, obs.name AS observer_name, COALESCE(obs.iata, '') AS observer_iata, - o.snr, o.rssi, o.path_json, - COALESCE((SELECT GROUP_CONCAT(DISTINCT obi.iata) FROM observations oi JOIN observers obi ON obi.rowid = oi.observer_idx WHERE oi.transmission_id = t.id AND obi.iata IS NOT NULL AND obi.iata != ''), '') AS distinct_iatas`+scopeNameCol+` - FROM transmissions t - LEFT JOIN observations o ON o.id = ( - SELECT id FROM observations WHERE transmission_id = t.id - ORDER BY length(COALESCE(path_json,'')) DESC LIMIT 1 - ) - LEFT JOIN observers obs ON obs.rowid = o.observer_idx - %s ORDER BY latest DESC LIMIT ? OFFSET ?`, w) - } else { - querySQL = fmt.Sprintf(`SELECT t.hash, t.first_seen, t.raw_hex, t.decoded_json, t.payload_type, t.route_type, - COALESCE((SELECT COUNT(*) FROM observations oi WHERE oi.transmission_id = t.id), 0) AS count, - COALESCE((SELECT COUNT(DISTINCT oi.observer_id) FROM observations oi WHERE oi.transmission_id = t.id), 0) AS observer_count, - COALESCE((SELECT MAX(oi.timestamp) FROM observations oi WHERE oi.transmission_id = t.id), t.first_seen) AS latest, - o.observer_id, o.observer_name, COALESCE(obs2.iata, '') AS observer_iata, - o.snr, o.rssi, o.path_json, - COALESCE((SELECT GROUP_CONCAT(DISTINCT obi.iata) FROM observations oi JOIN observers obi ON obi.id = oi.observer_id WHERE oi.transmission_id = t.id AND obi.iata IS NOT NULL AND obi.iata != ''), '') AS distinct_iatas`+scopeNameCol+` - FROM transmissions t - LEFT JOIN observations o ON o.id = ( - SELECT id FROM observations WHERE transmission_id = t.id - ORDER BY length(COALESCE(path_json,'')) DESC LIMIT 1 - ) - LEFT JOIN observers obs2 ON obs2.id = o.observer_id - %s ORDER BY latest DESC LIMIT ? OFFSET ?`, w) - } + querySQL := db.groupedPacketSelectSQL(w, len(args)) qArgs := make([]interface{}, len(args)) copy(qArgs, args) @@ -806,7 +718,55 @@ func (db *DB) QueryGroupedPackets(q PacketQuery) (*PacketResult, error) { return &PacketResult{Packets: packets, Total: total}, nil } -// parseDistinctIatasCSV turns SQLite GROUP_CONCAT output ("SJC,SFO,OAK") into +// groupedPacketSelectSQL ranks and pages matching transmissions before loading +// their observation details. The timestamp index can answer numeric MAX with +// one backwards lookup; formatting every observation before MAX prevents that. +func (db *DB) groupedPacketSelectSQL(w string, argumentCount int) string { + observerKey := "observer_id" + observerID, observerName := "o.observer_id", "o.observer_name" + observerJoin := "obs.id = o.observer_id" + iataJoin := "obi.id = oi.observer_id" + latest := `COALESCE((SELECT CAST(MAX(oi.timestamp) AS TEXT) FROM observations oi WHERE oi.transmission_id = t.id), t.first_seen)` + if db.isV3 { + observerKey = "observer_idx" + observerID, observerName = "obs.id", "obs.name" + observerJoin = "obs.rowid = o.observer_idx" + iataJoin = "obi.rowid = oi.observer_idx" + latest = db.nativeSQL( + `COALESCE((SELECT strftime('%Y-%m-%dT%H:%M:%fZ', MAX(oi.timestamp), 'unixepoch') FROM observations oi WHERE oi.transmission_id = t.id), t.first_seen)`, + `COALESCE((SELECT to_char(to_timestamp(MAX(oi.timestamp)) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"') FROM observations oi WHERE oi.transmission_id = t.id), t.first_seen)`) + } + // Keep latest independent of the observation filters: those decide which + // transmissions match, while the returned group describes all observations. + // LIMIT inside the derived table prevents both engines from joining/sorting + // best-path observations for transmissions outside the requested page. + page := fmt.Sprintf(`SELECT t.id, %s AS latest FROM transmissions t %s + ORDER BY latest DESC NULLS LAST, t.id DESC LIMIT %s OFFSET %s`, + latest, w, db.parameter(argumentCount+1), db.parameter(argumentCount+2)) + iatas := db.nativeSQL("group_concat(DISTINCT obi.iata)", "string_agg(DISTINCT obi.iata, ',')") + scopeNameCol := "" + if db.hasScopeName { + scopeNameCol = ", t.scope_name" + } + return fmt.Sprintf(`SELECT t.hash, t.first_seen, t.raw_hex, t.decoded_json, t.payload_type, t.route_type, + COALESCE((SELECT COUNT(*) FROM observations oi WHERE oi.transmission_id = t.id), 0) AS count, + COALESCE((SELECT COUNT(DISTINCT oi.%s) FROM observations oi WHERE oi.transmission_id = t.id), 0) AS observer_count, + page.latest, %s AS observer_id, %s AS observer_name, COALESCE(obs.iata, '') AS observer_iata, + o.snr, o.rssi, o.path_json, + COALESCE((SELECT %s FROM observations oi JOIN observers obi ON %s + WHERE oi.transmission_id = t.id AND obi.iata IS NOT NULL AND obi.iata != ''), '') AS distinct_iatas%s + FROM (%s) AS page + JOIN transmissions t ON t.id = page.id + LEFT JOIN observations o ON o.id = ( + SELECT id FROM observations WHERE transmission_id = t.id + ORDER BY length(COALESCE(path_json,'')) DESC NULLS LAST LIMIT 1 + ) + LEFT JOIN observers obs ON %s + ORDER BY page.latest DESC NULLS LAST, t.id DESC`, + observerKey, observerID, observerName, iatas, iataJoin, scopeNameCol, page, observerJoin) +} + +// parseDistinctIatasCSV turns PostgreSQL string_agg output ("SJC,SFO,OAK") into // a sorted, deduped []string. Returns an empty (non-nil) slice when the input // is empty/nil so JSON serialization stays consistent (`[]` not `null`). func parseDistinctIatasCSV(v interface{}) []string { @@ -834,38 +794,38 @@ func (db *DB) buildPacketWhere(q PacketQuery) ([]string, []interface{}) { var args []interface{} if q.Type != nil { - where = append(where, "payload_type = ?") + where = append(where, fmt.Sprintf("payload_type = %s", db.parameter(len(args)+1))) args = append(args, *q.Type) } if q.Route != nil { - where = append(where, "route_type = ?") + where = append(where, fmt.Sprintf("route_type = %s", db.parameter(len(args)+1))) args = append(args, *q.Route) } if q.Observer != "" { - where = append(where, "observer_id = ?") + where = append(where, fmt.Sprintf("observer_id = %s", db.parameter(len(args)+1))) args = append(args, q.Observer) } if q.Hash != "" { - where = append(where, "hash = ?") + where = append(where, fmt.Sprintf("hash = %s", db.parameter(len(args)+1))) args = append(args, strings.ToLower(q.Hash)) } if q.Since != "" { - where = append(where, "timestamp > ?") + where = append(where, fmt.Sprintf("timestamp > %s", db.parameter(len(args)+1))) args = append(args, q.Since) } if q.Until != "" { - where = append(where, "timestamp < ?") + where = append(where, fmt.Sprintf("timestamp < %s", db.parameter(len(args)+1))) args = append(args, q.Until) } if q.Region != "" { - where = append(where, "observer_id IN (SELECT id FROM observers WHERE iata = ?)") + where = append(where, fmt.Sprintf("observer_id IN (SELECT id FROM observers WHERE iata = %s)", db.parameter(len(args)+1))) args = append(args, q.Region) } if q.Node != "" { pk := db.resolveNodePubkey(q.Node) // #1143: exact-match on the dedicated from_pubkey column instead of // LIKE-on-JSON substring (adversarial spoof + same-name false positives). - where = append(where, "from_pubkey = ?") + where = append(where, fmt.Sprintf("from_pubkey = %s", db.parameter(len(args)+1))) args = append(args, pk) } return where, args @@ -878,15 +838,15 @@ func (db *DB) buildTransmissionWhere(q PacketQuery) ([]string, []interface{}) { var args []interface{} if q.Type != nil { - where = append(where, "t.payload_type = ?") + where = append(where, fmt.Sprintf("t.payload_type = %s", db.parameter(len(args)+1))) args = append(args, *q.Type) } if q.Route != nil { - where = append(where, "t.route_type = ?") + where = append(where, fmt.Sprintf("t.route_type = %s", db.parameter(len(args)+1))) args = append(args, *q.Route) } if q.Hash != "" { - where = append(where, "t.hash = ?") + where = append(where, fmt.Sprintf("t.hash = %s", db.parameter(len(args)+1))) args = append(args, strings.ToLower(q.Hash)) } if q.Since != "" { @@ -895,37 +855,36 @@ func (db *DB) buildTransmissionWhere(q PacketQuery) ([]string, []interface{}) { // but which have observations inside the window) are still included. // Non-RFC3339 falls back to t.first_seen string compare. if ts, err := time.Parse(time.RFC3339Nano, q.Since); err == nil { - where = append(where, "t.id IN (SELECT DISTINCT transmission_id FROM observations WHERE timestamp >= ?)") + where = append(where, fmt.Sprintf("t.id IN (SELECT DISTINCT transmission_id FROM observations WHERE timestamp >= %s)", db.parameter(len(args)+1))) args = append(args, ts.Unix()) } else { - where = append(where, "t.first_seen > ?") + where = append(where, fmt.Sprintf("t.first_seen > %s", db.parameter(len(args)+1))) args = append(args, q.Since) } } if q.Until != "" { if ts, err := time.Parse(time.RFC3339Nano, q.Until); err == nil { - where = append(where, "t.id IN (SELECT DISTINCT transmission_id FROM observations WHERE timestamp <= ?)") + where = append(where, fmt.Sprintf("t.id IN (SELECT DISTINCT transmission_id FROM observations WHERE timestamp <= %s)", db.parameter(len(args)+1))) args = append(args, ts.Unix()) } else { - where = append(where, "t.first_seen < ?") + where = append(where, fmt.Sprintf("t.first_seen < %s", db.parameter(len(args)+1))) args = append(args, q.Until) } } if q.Node != "" { pk := db.resolveNodePubkey(q.Node) // #1143: exact-match on dedicated from_pubkey column. - where = append(where, "t.from_pubkey = ?") + where = append(where, fmt.Sprintf("t.from_pubkey = %s", db.parameter(len(args)+1))) args = append(args, pk) } if q.Channel != "" { // channel_hash column is indexed for payload_type = 5; filter is exact match. - where = append(where, "t.channel_hash = ?") + where = append(where, fmt.Sprintf("t.channel_hash = %s", db.parameter(len(args)+1))) args = append(args, q.Channel) } if q.Observer != "" { ids := strings.Split(q.Observer, ",") - placeholders := strings.Repeat("?,", len(ids)) - placeholders = placeholders[:len(placeholders)-1] + placeholders := db.sqlPlaceholders(len(ids), len(args)+1) if db.isV3 { where = append(where, "EXISTS (SELECT 1 FROM observations oi JOIN observers obi ON obi.rowid = oi.observer_idx WHERE oi.transmission_id = t.id AND obi.id IN ("+placeholders+"))") } else { @@ -937,9 +896,9 @@ func (db *DB) buildTransmissionWhere(q PacketQuery) ([]string, []interface{}) { } if q.Region != "" { if db.isV3 { - where = append(where, "EXISTS (SELECT 1 FROM observations oi JOIN observers obi ON obi.rowid = oi.observer_idx WHERE oi.transmission_id = t.id AND obi.iata = ?)") + where = append(where, fmt.Sprintf("EXISTS (SELECT 1 FROM observations oi JOIN observers obi ON obi.rowid = oi.observer_idx WHERE oi.transmission_id = t.id AND obi.iata = %s)", db.parameter(len(args)+1))) } else { - where = append(where, "EXISTS (SELECT 1 FROM observations oi JOIN observers obi ON obi.id = oi.observer_id WHERE oi.transmission_id = t.id AND obi.iata = ?)") + where = append(where, fmt.Sprintf("EXISTS (SELECT 1 FROM observations oi JOIN observers obi ON obi.id = oi.observer_id WHERE oi.transmission_id = t.id AND obi.iata = %s)", db.parameter(len(args)+1))) } args = append(args, q.Region) } @@ -948,7 +907,7 @@ func (db *DB) buildTransmissionWhere(q PacketQuery) ([]string, []interface{}) { func (db *DB) resolveNodePubkey(nodeIDOrName string) string { var pk string - err := db.stmtQueryRow(db.stmtNodeLookup, "SELECT public_key FROM nodes WHERE public_key = ? OR name = ? LIMIT 1", nodeIDOrName, nodeIDOrName).Scan(&pk) + err := db.stmtQueryRow(db.stmtNodeLookup, "SELECT public_key FROM nodes WHERE public_key = "+db.parameter(1)+" OR name = "+db.parameter(2)+" LIMIT 1", nodeIDOrName, nodeIDOrName).Scan(&pk) if err != nil { return nodeIDOrName } @@ -958,7 +917,7 @@ func (db *DB) resolveNodePubkey(nodeIDOrName string) string { // GetTransmissionByID fetches from transmissions table with observer data. func (db *DB) GetTransmissionByID(id int) (map[string]interface{}, error) { selectCols, observerJoin := db.transmissionBaseSQL() - querySQL := fmt.Sprintf("SELECT %s FROM transmissions t %s WHERE t.id = ?", selectCols, observerJoin) + querySQL := fmt.Sprintf("SELECT %s FROM transmissions t %s WHERE t.id = "+db.parameter(1), selectCols, observerJoin) rows, err := db.conn.Query(querySQL, id) if err != nil { @@ -974,7 +933,7 @@ func (db *DB) GetTransmissionByID(id int) (map[string]interface{}, error) { // GetPacketByHash fetches a transmission by content hash with observer data. func (db *DB) GetPacketByHash(hash string) (map[string]interface{}, error) { selectCols, observerJoin := db.transmissionBaseSQL() - querySQL := fmt.Sprintf("SELECT %s FROM transmissions t %s WHERE t.hash = ?", selectCols, observerJoin) + querySQL := fmt.Sprintf("SELECT %s FROM transmissions t %s WHERE t.hash = "+db.parameter(1), selectCols, observerJoin) rows, err := db.conn.Query(querySQL, strings.ToLower(hash)) if err != nil { @@ -992,7 +951,7 @@ func (db *DB) GetPacketByHash(hash string) (map[string]interface{}, error) { // when the in-memory PacketStore has pruned the entry but the DB still has it. func (db *DB) GetObservationsForHash(hash string) []map[string]interface{} { var txID int - err := db.stmtQueryRow(db.stmtTxByHash, "SELECT id FROM transmissions WHERE hash = ?", strings.ToLower(hash)).Scan(&txID) + err := db.stmtQueryRow(db.stmtTxByHash, "SELECT id FROM transmissions WHERE hash = "+db.parameter(1), strings.ToLower(hash)).Scan(&txID) if err != nil { return nil } @@ -1000,21 +959,92 @@ func (db *DB) GetObservationsForHash(hash string) []map[string]interface{} { return obsByTx[txID] } +// ObservationRawHexForHash returns the stored wire bytes per observation id for +// one transmission, keyed by observations.id. Empty when the schema has no +// observations.raw_hex column (#881 made it optional) or nothing is stored. +// +// Why this is read on demand instead of held in memory (#1999): the store +// deliberately does not retain obs.RawHex. #881 measured ~98MB wasted on a +// ~1.7M-observation store, because at the time the frames were believed to be +// identical per transmission ("same content hash implies same frame"). They are +// not: the firmware hashes payload and type independently of the relay path, so +// observations of one transmission legitimately carry different bytes. Keeping +// the memory saving and paying one query on the packet-detail path, which is a +// single packet a human is looking at, is the trade this makes. +// +// Two indexed lookups, one query, regardless of how many observations the +// transmission has: transmissions.hash via idx_transmissions_hash (the prepared +// stmtTxByHash), then observations.transmission_id via +// idx_observations_transmission_id. +func (db *DB) ObservationRawHexForHash(hash string) map[int]string { + if db == nil || db.conn == nil || !db.hasObsRawHex || hash == "" { + return nil + } + var txID int + if err := db.stmtQueryRow(db.stmtTxByHash, "SELECT id FROM transmissions WHERE hash = "+db.parameter(1), + strings.ToLower(hash)).Scan(&txID); err != nil { + return nil + } + rows, err := db.conn.Query( + `SELECT id, raw_hex FROM observations + WHERE transmission_id = `+db.parameter(1)+` AND raw_hex IS NOT NULL AND raw_hex <> ''`, txID) + if err != nil { + return nil + } + defer rows.Close() + out := make(map[int]string) + for rows.Next() { + var id int + var hx sql.NullString + if err := rows.Scan(&id, &hx); err != nil { + continue + } + if hx.Valid && hx.String != "" { + out[id] = hx.String + } + } + if err := rows.Err(); err != nil { + return nil + } + return out +} + +// NodeQuery selects and orders nodes for GetNodes. Region filters by the SQL +// subquery over adverts and observers; RegionPubkeys, when non-nil, is that +// filter already resolved to a set of node pubkeys (#2101, see +// PacketStore.RegionNodePubkeys) and takes its place. An empty, non-nil +// RegionPubkeys matches no node. +type NodeQuery struct { + Limit, Offset int + Role string + Search string + Before string + LastHeard string + SortBy string + Region string + RegionPubkeys []string +} + // GetNodes returns filtered, paginated node list. -func (db *DB) GetNodes(limit, offset int, role, search, before, lastHeard, sortBy, region string) ([]map[string]interface{}, int, map[string]int, error) { +func (db *DB) GetNodes(nq NodeQuery) ([]map[string]interface{}, int, map[string]int, error) { + limit, offset := nq.Limit, max(0, nq.Offset) + role, search, before, lastHeard, sortBy, region := nq.Role, nq.Search, nq.Before, nq.LastHeard, nq.SortBy, nq.Region + if nq.RegionPubkeys != nil { + region = "" + } var where []string var args []interface{} if role != "" { - where = append(where, "role = ?") + where = append(where, fmt.Sprintf("role = %s", db.parameter(len(args)+1))) args = append(args, role) } if search != "" { - where = append(where, "name LIKE ?") + where = append(where, fmt.Sprintf(db.nativeSQL("name LIKE %s", "name ILIKE %s ESCAPE ''"), db.parameter(len(args)+1))) args = append(args, "%"+search+"%") } if before != "" { - where = append(where, "first_seen <= ?") + where = append(where, fmt.Sprintf("first_seen <= %s", db.parameter(len(args)+1))) args = append(args, before) } if lastHeard != "" { @@ -1024,7 +1054,7 @@ func (db *DB) GetNodes(limit, offset int, role, search, before, lastHeard, sortB } if ms, ok := durations[lastHeard]; ok { since := time.Now().Add(-time.Duration(ms) * time.Millisecond).Format(time.RFC3339) - where = append(where, "last_seen > ?") + where = append(where, fmt.Sprintf("last_seen > %s", db.parameter(len(args)+1))) args = append(args, since) } } @@ -1035,7 +1065,7 @@ func (db *DB) GetNodes(limit, offset int, role, search, before, lastHeard, sortB placeholders := make([]string, len(codes)) regionArgs := make([]interface{}, len(codes)) for i, c := range codes { - placeholders[i] = "?" + placeholders[i] = fmt.Sprintf("%s", db.parameter(len(args)+i+1)) regionArgs[i] = c } joinCond := "obs.rowid = o.observer_idx" @@ -1057,6 +1087,21 @@ func (db *DB) GetNodes(limit, offset int, role, search, before, lastHeard, sortB args = append(args, regionArgs...) } } + if nq.RegionPubkeys != nil { + if len(nq.RegionPubkeys) == 0 { + where = append(where, "FALSE") + } else { + // One JSON parameter rather than one placeholder per key: a region + // can hold thousands of nodes, and the IN list stays a primary-key + // lookup. + keysJSON, err := json.Marshal(nq.RegionPubkeys) + if err != nil { + return nil, 0, nil, err + } + where = append(where, fmt.Sprintf(db.nativeSQL("public_key IN (SELECT value FROM json_each(%s))", "public_key IN (SELECT value FROM json_array_elements_text(%s::json))"), db.parameter(len(args)+1))) + args = append(args, string(keysJSON)) + } + } w := "" if len(where) > 0 { @@ -1064,9 +1109,9 @@ func (db *DB) GetNodes(limit, offset int, role, search, before, lastHeard, sortB } sortMap := map[string]string{ - "name": "name ASC", "lastSeen": "last_seen DESC", "packetCount": "advert_count DESC", + "name": "name ASC NULLS FIRST", "lastSeen": "last_seen DESC NULLS LAST", "packetCount": "advert_count DESC NULLS LAST", } - order := "last_seen DESC" + order := "last_seen DESC NULLS LAST" if s, ok := sortMap[sortBy]; ok { order = s } @@ -1078,7 +1123,7 @@ func (db *DB) GetNodes(limit, offset int, role, search, before, lastHeard, sortB var total int db.conn.QueryRow(fmt.Sprintf("SELECT COUNT(*) FROM nodes %s", w), args...).Scan(&total) - querySQL := fmt.Sprintf("SELECT %s FROM nodes %s ORDER BY %s LIMIT ? OFFSET ?", db.nodeSelectCols(), w, order) + querySQL := fmt.Sprintf("SELECT %s FROM nodes %s ORDER BY %s LIMIT %s OFFSET %s", db.nodeSelectCols(), w, order, db.parameter(len(args)+1), db.parameter(len(args)+2)) qArgs := append(args, limit, offset) rows, err := db.conn.Query(querySQL, qArgs...) @@ -1104,7 +1149,7 @@ func (db *DB) SearchNodes(query string, limit int) ([]map[string]interface{}, er if limit <= 0 { limit = 10 } - rows, err := db.conn.Query(fmt.Sprintf("SELECT %s FROM nodes WHERE name LIKE ? OR public_key LIKE ? ORDER BY last_seen DESC LIMIT ?", db.nodeSelectCols()), + rows, err := db.conn.Query(fmt.Sprintf(db.nativeSQL("SELECT %s FROM nodes WHERE name LIKE ?1 OR public_key LIKE ?2 ORDER BY last_seen DESC NULLS LAST LIMIT ?3", "SELECT %s FROM nodes WHERE name ILIKE "+db.parameter(1)+" ESCAPE '' OR public_key ILIKE "+db.parameter(2)+" ESCAPE '' ORDER BY last_seen DESC NULLS LAST LIMIT "+db.parameter(3)), db.nodeSelectCols()), "%"+query+"%", query+"%", limit) if err != nil { return nil, err @@ -1142,7 +1187,7 @@ func (db *DB) GetNodeByPrefix(prefix string) (map[string]interface{}, bool, erro } } rows, err := db.conn.Query( - fmt.Sprintf("SELECT %s FROM nodes WHERE public_key LIKE ? LIMIT 2", db.nodeSelectCols()), + fmt.Sprintf(db.nativeSQL("SELECT %s FROM nodes WHERE public_key LIKE ?1 LIMIT 2", "SELECT %s FROM nodes WHERE public_key ILIKE "+db.parameter(1)+" ESCAPE '' LIMIT 2"), db.nodeSelectCols()), prefix+"%", ) if err != nil { @@ -1171,7 +1216,7 @@ func (db *DB) GetNodeByPrefix(prefix string) (map[string]interface{}, bool, erro // GetNodeByPubkey returns a single node. func (db *DB) GetNodeByPubkey(pubkey string) (map[string]interface{}, error) { - rows, err := db.conn.Query(fmt.Sprintf("SELECT %s FROM nodes WHERE public_key = ?", db.nodeSelectCols()), pubkey) + rows, err := db.conn.Query(fmt.Sprintf("SELECT %s FROM nodes WHERE public_key = "+db.parameter(1), db.nodeSelectCols()), pubkey) if err != nil { return nil, err } @@ -1198,7 +1243,7 @@ func (db *DB) GetRecentTransmissionsForNode(pubkey string, limit int) ([]map[str // #1345: order by ingest id, not first_seen (=rxTime). Buffered observer // uploads with old rxTime would otherwise displace fresh activity from // the "recent transmissions for node" list. - querySQL := fmt.Sprintf("SELECT %s FROM transmissions t %s WHERE t.from_pubkey = ? ORDER BY t.id DESC LIMIT ?", + querySQL := fmt.Sprintf("SELECT %s FROM transmissions t %s WHERE t.from_pubkey = "+db.parameter(1)+" ORDER BY t.id DESC LIMIT "+db.parameter(2), selectCols, observerJoin) args := []interface{}{pubkey, limit} @@ -1222,6 +1267,20 @@ func (db *DB) GetRecentTransmissionsForNode(pubkey string, limit int) ([]map[str } } + if err := rows.Err(); err != nil { + return nil, err + } + rows.Close() + masks, err := db.advertEvidenceForIDs(txIDs) + if err != nil { + return nil, err + } + for _, p := range packets { + if p["payload_type"] == 4 { + p["advert_kind"] = packetpath.AdvertKind(masks[p["id"].(int)]) + } + } + // Fetch observations for all transmissions if len(txIDs) > 0 { obsMap := db.getObservationsForTransmissions(txIDs) @@ -1249,18 +1308,23 @@ func (db *DB) getObservationsForTransmissions(txIDs []int) map[int][]map[string] placeholders := make([]string, len(txIDs)) args := make([]interface{}, len(txIDs)) for i, id := range txIDs { - placeholders[i] = "?" + placeholders[i] = fmt.Sprintf("%s", db.parameter(i+1)) args[i] = id } var querySQL string if db.isV3 { - querySQL = fmt.Sprintf(`SELECT o.transmission_id, o.id, obs.id AS observer_id, obs.name AS observer_name, COALESCE(obs.iata, '') AS observer_iata, + querySQL = fmt.Sprintf(db.nativeSQL(`SELECT o.transmission_id, o.id, obs.id AS observer_id, obs.name AS observer_name, COALESCE(obs.iata, '') AS observer_iata, o.direction, o.snr, o.rssi, o.path_json, strftime('%%Y-%%m-%%dT%%H:%%M:%%fZ', o.timestamp, 'unixepoch') AS obs_timestamp FROM observations o LEFT JOIN observers obs ON obs.rowid = o.observer_idx WHERE o.transmission_id IN (%s) - ORDER BY o.timestamp DESC`, strings.Join(placeholders, ",")) + ORDER BY o.timestamp DESC`, `SELECT o.transmission_id, o.id, obs.id AS observer_id, obs.name AS observer_name, COALESCE(obs.iata, '') AS observer_iata, + o.direction, o.snr, o.rssi, o.path_json, to_char(to_timestamp(o.timestamp) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"') AS obs_timestamp + FROM observations o + LEFT JOIN observers obs ON obs.rowid = o.observer_idx + WHERE o.transmission_id IN (%s) + ORDER BY o.timestamp DESC`), strings.Join(placeholders, ",")) } else { querySQL = fmt.Sprintf(`SELECT o.transmission_id, o.id, o.observer_id, o.observer_name, COALESCE(obs.iata, '') AS observer_iata, o.direction, o.snr, o.rssi, o.path_json, o.timestamp AS obs_timestamp @@ -1318,17 +1382,17 @@ func (db *DB) GetObservers() ([]Observer, error) { // and the server returns CanRelay=nil so the UI shows no badge. canRelayClause := "COALESCE(can_relay, 1)" canRelaySeenClause := "0" - if hasCol, _ := dbschema.TableHasColumn(db.conn, "observers", "can_relay"); !hasCol { + if hasCol, _ := db.tableHasColumn("observers", "can_relay"); !hasCol { canRelayClause = "1" } - if hasCol, _ := dbschema.TableHasColumn(db.conn, "observers", "can_relay_seen"); hasCol { + if hasCol, _ := db.tableHasColumn("observers", "can_relay_seen"); hasCol { canRelaySeenClause = "COALESCE(can_relay_seen, 0)" } rows, err := db.conn.Query(`SELECT id, name, iata, last_seen, first_seen, packet_count, model, firmware, client_version, radio, battery_mv, uptime_secs, noise_floor, last_packet_at, clock_skew_seconds, clock_skew_count_24h, clock_last_naive_at, ` + canRelayClause + `, ` + canRelaySeenClause + ` - FROM observers WHERE inactive IS NULL OR inactive = 0 ORDER BY last_seen DESC`) + FROM observers WHERE inactive IS NULL OR inactive = 0 ORDER BY last_seen DESC NULLS LAST`) if err != nil { return nil, err } @@ -1381,7 +1445,7 @@ func (db *DB) GetObservers() ([]Observer, error) { func (db *DB) GetNonRelayObserverPubkeys() ([]string, error) { // Graceful no-op when can_relay column is absent (legacy DB / older // test fixture). Avoids noisy schema-degradation log spam. - if hasCol, _ := dbschema.TableHasColumn(db.conn, "observers", "can_relay"); !hasCol { + if hasCol, _ := db.tableHasColumn("observers", "can_relay"); !hasCol { return nil, nil } rows, err := db.conn.Query(`SELECT LOWER(id) FROM observers @@ -1407,7 +1471,7 @@ func (db *DB) GetNonRelayObserverPubkeys() ([]string, error) { // this to render tri-state — observers NOT in this set are "unknown" // and the UI shows no badge. func (db *DB) GetCanRelaySeenObserverPubkeys() ([]string, error) { - if hasCol, _ := dbschema.TableHasColumn(db.conn, "observers", "can_relay_seen"); !hasCol { + if hasCol, _ := db.tableHasColumn("observers", "can_relay_seen"); !hasCol { return nil, nil } rows, err := db.conn.Query(`SELECT LOWER(id) FROM observers @@ -1436,17 +1500,17 @@ func (db *DB) GetObserverByID(id string) (*Observer, error) { var canRelay, canRelaySeen int canRelayClause := "COALESCE(can_relay, 1)" canRelaySeenClause := "0" - if hasCol, _ := dbschema.TableHasColumn(db.conn, "observers", "can_relay"); !hasCol { + if hasCol, _ := db.tableHasColumn("observers", "can_relay"); !hasCol { canRelayClause = "1" } - if hasCol, _ := dbschema.TableHasColumn(db.conn, "observers", "can_relay_seen"); hasCol { + if hasCol, _ := db.tableHasColumn("observers", "can_relay_seen"); hasCol { canRelaySeenClause = "COALESCE(can_relay_seen, 0)" } err := db.conn.QueryRow(`SELECT id, name, iata, last_seen, first_seen, packet_count, model, firmware, client_version, radio, battery_mv, uptime_secs, noise_floor, last_packet_at, clock_skew_seconds, clock_skew_count_24h, clock_last_naive_at, `+canRelayClause+`, `+canRelaySeenClause+` - FROM observers WHERE id = ?`, id). + FROM observers WHERE id = `+db.parameter(1), id). Scan(&o.ID, &o.Name, &o.IATA, &o.LastSeen, &o.FirstSeen, &o.PacketCount, &o.Model, &o.Firmware, &o.ClientVersion, &o.Radio, &batteryMv, &uptimeSecs, &noiseFloor, &o.LastPacketAt, &clockSkewSec, &clockSkewCount, &o.ClockLastNaiveAt, &canRelay, &canRelaySeen) @@ -1486,7 +1550,7 @@ func (db *DB) GetObserverIdsForRegion(regionParam string) ([]string, error) { placeholders := make([]string, len(codes)) args := make([]interface{}, len(codes)) for i, c := range codes { - placeholders[i] = "?" + placeholders[i] = fmt.Sprintf("%s", db.parameter(i+1)) args[i] = c } rows, err := db.conn.Query(fmt.Sprintf("SELECT id FROM observers WHERE UPPER(TRIM(iata)) IN (%s)", strings.Join(placeholders, ",")), args...) @@ -1605,22 +1669,35 @@ func (db *DB) GetNetworkStatus(healthThresholds HealthThresholds) (map[string]in func (db *DB) GetTraces(hash string) ([]map[string]interface{}, error) { var querySQL string if db.isV3 { - querySQL = `SELECT obs.id AS observer_id, obs.name AS observer_name, + querySQL = db.nativeSQL(`SELECT obs.id AS observer_id, obs.name AS observer_name, strftime('%Y-%m-%dT%H:%M:%fZ', o.timestamp, 'unixepoch') AS timestamp, o.snr, o.rssi, o.path_json FROM observations o JOIN transmissions t ON t.id = o.transmission_id LEFT JOIN observers obs ON obs.rowid = o.observer_idx - WHERE t.hash = ? - ORDER BY o.timestamp ASC` + WHERE t.hash = ?1 + ORDER BY o.timestamp ASC`, `SELECT obs.id AS observer_id, obs.name AS observer_name, + to_char(to_timestamp(o.timestamp) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"') AS timestamp, + o.snr, o.rssi, o.path_json + FROM observations o + JOIN transmissions t ON t.id = o.transmission_id + LEFT JOIN observers obs ON obs.rowid = o.observer_idx + WHERE t.hash = `+db.parameter(1)+` + ORDER BY o.timestamp ASC`) } else { - querySQL = `SELECT o.observer_id, o.observer_name, + querySQL = db.nativeSQL(`SELECT o.observer_id, o.observer_name, strftime('%Y-%m-%dT%H:%M:%fZ', o.timestamp, 'unixepoch') AS timestamp, o.snr, o.rssi, o.path_json FROM observations o JOIN transmissions t ON t.id = o.transmission_id - WHERE t.hash = ? - ORDER BY o.timestamp ASC` + WHERE t.hash = ?1 + ORDER BY o.timestamp ASC`, `SELECT o.observer_id, o.observer_name, + to_char(to_timestamp(o.timestamp) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"') AS timestamp, + o.snr, o.rssi, o.path_json + FROM observations o + JOIN transmissions t ON t.id = o.transmission_id + WHERE t.hash = `+db.parameter(1)+` + ORDER BY o.timestamp ASC`) } rows, err := db.conn.Query(querySQL, strings.ToLower(hash)) if err != nil { @@ -1703,128 +1780,141 @@ func (db *DB) GetChannels(region ...string) ([]map[string]interface{}, error) { return cached, nil } - regionCodes := normalizeRegionCodes(regionParam) + v, err, _ := db.channelsSF.Do(regionParam, func() (interface{}, error) { + // Double-check inside the flight: a previous winner may have stored a + // fresh result between this caller's read above and its arrival here. + if cached, ok := db.getChannelsCache(regionParam); ok { + return cached, nil + } - var querySQL string - args := make([]interface{}, 0, len(regionCodes)) + if db.channelsQueryHook != nil { + db.channelsQueryHook() + } - if len(regionCodes) > 0 { - placeholders := make([]string, len(regionCodes)) - for i, code := range regionCodes { - placeholders[i] = "?" - args = append(args, code) - } - regionPlaceholder := strings.Join(placeholders, ",") - // #1899: the sample_json subquery is region-scoped too, so its placeholders - // appear FIRST in the statement (it sits in the SELECT list, ahead of the - // WHERE). Bind the codes twice, subquery set first. - args = append(append(make([]interface{}, 0, len(regionCodes)*2), args...), args...) - if db.isV3 { - querySQL = fmt.Sprintf(`SELECT t.channel_hash, - COUNT(*) AS msg_count, - MAX(t.first_seen) AS last_activity, - (SELECT t2.decoded_json FROM transmissions t2 - JOIN observations o2 ON o2.transmission_id = t2.id - LEFT JOIN observers obs2 ON obs2.rowid = o2.observer_idx - WHERE t2.channel_hash = t.channel_hash AND t2.payload_type = 5 - AND obs2.rowid IS NOT NULL AND UPPER(TRIM(obs2.iata)) IN (%s) - ORDER BY t2.first_seen DESC LIMIT 1) AS sample_json - FROM transmissions t - JOIN observations o ON o.transmission_id = t.id - LEFT JOIN observers obs ON obs.rowid = o.observer_idx - WHERE t.payload_type = 5 - AND t.channel_hash IS NOT NULL - AND t.channel_hash NOT LIKE 'enc_%%' - AND obs.rowid IS NOT NULL AND UPPER(TRIM(obs.iata)) IN (%s) - GROUP BY t.channel_hash - ORDER BY last_activity DESC`, regionPlaceholder, regionPlaceholder) + regionCodes := normalizeRegionCodes(regionParam) + + var querySQL string + args := make([]interface{}, 0, len(regionCodes)) + + if len(regionCodes) > 0 { + placeholders := make([]string, len(regionCodes)) + for i, code := range regionCodes { + placeholders[i] = fmt.Sprintf("%s", db.parameter(i+1)) + args = append(args, code) + } + regionPlaceholder := strings.Join(placeholders, ",") + // Reuse the same native parameters for sample and channel region filters. + if db.isV3 { + querySQL = fmt.Sprintf(`SELECT t.channel_hash, + COUNT(*) AS msg_count, + MAX(t.first_seen) AS last_activity, + (SELECT t2.decoded_json FROM transmissions t2 + JOIN observations o2 ON o2.transmission_id = t2.id + LEFT JOIN observers obs2 ON obs2.rowid = o2.observer_idx + WHERE t2.channel_hash = t.channel_hash AND t2.payload_type = 5 + AND obs2.rowid IS NOT NULL AND UPPER(TRIM(obs2.iata)) IN (%s) + ORDER BY t2.first_seen DESC LIMIT 1) AS sample_json + FROM transmissions t + JOIN observations o ON o.transmission_id = t.id + LEFT JOIN observers obs ON obs.rowid = o.observer_idx + WHERE t.payload_type = 5 + AND t.channel_hash IS NOT NULL + AND t.channel_hash NOT LIKE 'enc_%%' + AND obs.rowid IS NOT NULL AND UPPER(TRIM(obs.iata)) IN (%s) + GROUP BY t.channel_hash + ORDER BY last_activity DESC NULLS LAST`, regionPlaceholder, regionPlaceholder) + } else { + querySQL = fmt.Sprintf(`SELECT t.channel_hash, + COUNT(*) AS msg_count, + MAX(t.first_seen) AS last_activity, + (SELECT t2.decoded_json FROM transmissions t2 + JOIN observations o2 ON o2.transmission_id = t2.id + WHERE t2.channel_hash = t.channel_hash AND t2.payload_type = 5 + AND EXISTS ( + SELECT 1 FROM observers obs2 + WHERE obs2.id = o2.observer_id + AND UPPER(TRIM(obs2.iata)) IN (%s) + ) + ORDER BY t2.first_seen DESC LIMIT 1) AS sample_json + FROM transmissions t + JOIN observations o ON o.transmission_id = t.id + WHERE t.payload_type = 5 + AND t.channel_hash IS NOT NULL + AND t.channel_hash NOT LIKE 'enc_%%' + AND EXISTS ( + SELECT 1 FROM observers obs + WHERE obs.id = o.observer_id + AND UPPER(TRIM(obs.iata)) IN (%s) + ) + GROUP BY t.channel_hash + ORDER BY last_activity DESC NULLS LAST`, regionPlaceholder, regionPlaceholder) + } } else { - querySQL = fmt.Sprintf(`SELECT t.channel_hash, + querySQL = `SELECT channel_hash, COUNT(*) AS msg_count, - MAX(t.first_seen) AS last_activity, + MAX(first_seen) AS last_activity, (SELECT t2.decoded_json FROM transmissions t2 - JOIN observations o2 ON o2.transmission_id = t2.id WHERE t2.channel_hash = t.channel_hash AND t2.payload_type = 5 - AND EXISTS ( - SELECT 1 FROM observers obs2 - WHERE obs2.id = o2.observer_id - AND UPPER(TRIM(obs2.iata)) IN (%s) - ) ORDER BY t2.first_seen DESC LIMIT 1) AS sample_json FROM transmissions t - JOIN observations o ON o.transmission_id = t.id - WHERE t.payload_type = 5 - AND t.channel_hash IS NOT NULL - AND t.channel_hash NOT LIKE 'enc_%%' - AND EXISTS ( - SELECT 1 FROM observers obs - WHERE obs.id = o.observer_id - AND UPPER(TRIM(obs.iata)) IN (%s) - ) - GROUP BY t.channel_hash - ORDER BY last_activity DESC`, regionPlaceholder, regionPlaceholder) + WHERE payload_type = 5 + AND channel_hash IS NOT NULL + AND channel_hash NOT LIKE 'enc_%%' + GROUP BY channel_hash + ORDER BY last_activity DESC NULLS LAST` } - } else { - querySQL = `SELECT channel_hash, - COUNT(*) AS msg_count, - MAX(first_seen) AS last_activity, - (SELECT t2.decoded_json FROM transmissions t2 - WHERE t2.channel_hash = t.channel_hash AND t2.payload_type = 5 - ORDER BY t2.first_seen DESC LIMIT 1) AS sample_json - FROM transmissions t - WHERE payload_type = 5 - AND channel_hash IS NOT NULL - AND channel_hash NOT LIKE 'enc_%%' - GROUP BY channel_hash - ORDER BY last_activity DESC` - } - rows, err := db.conn.Query(querySQL, args...) - if err != nil { - return nil, err - } - defer rows.Close() - - channels := make([]map[string]interface{}, 0) - for rows.Next() { - var chHash, lastActivity, sampleJSON sql.NullString - var msgCount int - if err := rows.Scan(&chHash, &msgCount, &lastActivity, &sampleJSON); err != nil { - continue - } - channelName := nullStr(chHash) - if channelName == "" { - continue + rows, err := db.conn.Query(querySQL, args...) + if err != nil { + return nil, err } + defer rows.Close() - var lastMessage, lastSender interface{} - if sampleJSON.Valid { - var decoded map[string]interface{} - if json.Unmarshal([]byte(sampleJSON.String), &decoded) == nil { - if text, ok := decoded["text"].(string); ok && text != "" { - idx := strings.Index(text, ": ") - if idx > 0 { - lastMessage = text[idx+2:] - } else { - lastMessage = text - } - if sender, ok := decoded["sender"].(string); ok { - lastSender = sender + channels := make([]map[string]interface{}, 0) + for rows.Next() { + var chHash, lastActivity, sampleJSON sql.NullString + var msgCount int + if err := rows.Scan(&chHash, &msgCount, &lastActivity, &sampleJSON); err != nil { + continue + } + channelName := nullStr(chHash) + if channelName == "" { + continue + } + + var lastMessage, lastSender interface{} + if sampleJSON.Valid { + var decoded map[string]interface{} + if json.Unmarshal([]byte(sampleJSON.String), &decoded) == nil { + if text, ok := decoded["text"].(string); ok && text != "" { + idx := strings.Index(text, ": ") + if idx > 0 { + lastMessage = text[idx+2:] + } else { + lastMessage = text + } + if sender, ok := decoded["sender"].(string); ok { + lastSender = sender + } } } } - } - channels = append(channels, map[string]interface{}{ - "hash": channelName, "name": channelName, - "lastMessage": lastMessage, "lastSender": lastSender, - "messageCount": msgCount, "lastActivity": nullStr(lastActivity), - }) - } + channels = append(channels, map[string]interface{}{ + "hash": channelName, "name": channelName, + "lastMessage": lastMessage, "lastSender": lastSender, + "messageCount": msgCount, "lastActivity": nullStr(lastActivity), + }) + } - db.setChannelsCache(regionParam, channels) + db.setChannelsCache(regionParam, channels) - return channels, nil + return channels, nil + }) + if err != nil { + return nil, err + } + return v.([]map[string]interface{}), nil } // GetEncryptedChannels returns channels where all messages are undecryptable (no key). @@ -1839,86 +1929,102 @@ func (db *DB) GetEncryptedChannels(region ...string) ([]map[string]interface{}, return cached, nil } - regionCodes := normalizeRegionCodes(regionParam) - - var querySQL string - args := make([]interface{}, 0, len(regionCodes)) + v, err, _ := db.encChannelsSF.Do(regionParam, func() (interface{}, error) { + // Double-check inside the flight: a previous winner may have stored a + // fresh result between this caller's read above and its arrival here. + if cached, ok := db.getEncChannelsCache(regionParam); ok { + return cached, nil + } - if len(regionCodes) > 0 { - placeholders := make([]string, len(regionCodes)) - for i, code := range regionCodes { - placeholders[i] = "?" - args = append(args, code) + if db.encChannelsQueryHook != nil { + db.encChannelsQueryHook() } - regionPlaceholder := strings.Join(placeholders, ",") - if db.isV3 { - querySQL = fmt.Sprintf(`SELECT t.channel_hash, - COUNT(*) AS msg_count, - MAX(t.first_seen) AS last_activity - FROM transmissions t - JOIN observations o ON o.transmission_id = t.id - LEFT JOIN observers obs ON obs.rowid = o.observer_idx - WHERE t.payload_type = 5 - AND t.channel_hash LIKE 'enc_%%' - AND obs.rowid IS NOT NULL AND UPPER(TRIM(obs.iata)) IN (%s) - GROUP BY t.channel_hash - ORDER BY last_activity DESC`, regionPlaceholder) + + regionCodes := normalizeRegionCodes(regionParam) + + var querySQL string + args := make([]interface{}, 0, len(regionCodes)) + + if len(regionCodes) > 0 { + placeholders := make([]string, len(regionCodes)) + for i, code := range regionCodes { + placeholders[i] = fmt.Sprintf("%s", db.parameter(i+1)) + args = append(args, code) + } + regionPlaceholder := strings.Join(placeholders, ",") + if db.isV3 { + querySQL = fmt.Sprintf(`SELECT t.channel_hash, + COUNT(*) AS msg_count, + MAX(t.first_seen) AS last_activity + FROM transmissions t + JOIN observations o ON o.transmission_id = t.id + LEFT JOIN observers obs ON obs.rowid = o.observer_idx + WHERE t.payload_type = 5 + AND t.channel_hash LIKE 'enc_%%' + AND obs.rowid IS NOT NULL AND UPPER(TRIM(obs.iata)) IN (%s) + GROUP BY t.channel_hash + ORDER BY last_activity DESC NULLS LAST`, regionPlaceholder) + } else { + querySQL = fmt.Sprintf(`SELECT t.channel_hash, + COUNT(*) AS msg_count, + MAX(t.first_seen) AS last_activity + FROM transmissions t + JOIN observations o ON o.transmission_id = t.id + WHERE t.payload_type = 5 + AND t.channel_hash LIKE 'enc_%%' + AND EXISTS ( + SELECT 1 FROM observers obs + WHERE obs.id = o.observer_id + AND UPPER(TRIM(obs.iata)) IN (%s) + ) + GROUP BY t.channel_hash + ORDER BY last_activity DESC NULLS LAST`, regionPlaceholder) + } } else { - querySQL = fmt.Sprintf(`SELECT t.channel_hash, + querySQL = `SELECT channel_hash, COUNT(*) AS msg_count, - MAX(t.first_seen) AS last_activity - FROM transmissions t - JOIN observations o ON o.transmission_id = t.id - WHERE t.payload_type = 5 - AND t.channel_hash LIKE 'enc_%%' - AND EXISTS ( - SELECT 1 FROM observers obs - WHERE obs.id = o.observer_id - AND UPPER(TRIM(obs.iata)) IN (%s) - ) - GROUP BY t.channel_hash - ORDER BY last_activity DESC`, regionPlaceholder) + MAX(first_seen) AS last_activity + FROM transmissions + WHERE payload_type = 5 + AND channel_hash LIKE 'enc_%%' + GROUP BY channel_hash + ORDER BY last_activity DESC NULLS LAST` } - } else { - querySQL = `SELECT channel_hash, - COUNT(*) AS msg_count, - MAX(first_seen) AS last_activity - FROM transmissions - WHERE payload_type = 5 - AND channel_hash LIKE 'enc_%%' - GROUP BY channel_hash - ORDER BY last_activity DESC` - } - rows, err := db.conn.Query(querySQL, args...) + rows, err := db.conn.Query(querySQL, args...) + if err != nil { + return nil, err + } + defer rows.Close() + + channels := make([]map[string]interface{}, 0) + for rows.Next() { + var chHash, lastActivity sql.NullString + var msgCount int + if err := rows.Scan(&chHash, &msgCount, &lastActivity); err != nil { + continue + } + fullHash := nullStrVal(chHash) // e.g. "enc_3A" + hexPart := strings.TrimPrefix(fullHash, "enc_") + channels = append(channels, map[string]interface{}{ + "hash": fullHash, + "name": "Encrypted (0x" + hexPart + ")", + "lastMessage": nil, + "lastSender": nil, + "messageCount": msgCount, + "lastActivity": nullStr(lastActivity), + "encrypted": true, + }) + } + + db.setEncChannelsCache(regionParam, channels) + + return channels, nil + }) if err != nil { return nil, err } - defer rows.Close() - - channels := make([]map[string]interface{}, 0) - for rows.Next() { - var chHash, lastActivity sql.NullString - var msgCount int - if err := rows.Scan(&chHash, &msgCount, &lastActivity); err != nil { - continue - } - fullHash := nullStrVal(chHash) // e.g. "enc_3A" - hexPart := strings.TrimPrefix(fullHash, "enc_") - channels = append(channels, map[string]interface{}{ - "hash": fullHash, - "name": "Encrypted (0x" + hexPart + ")", - "lastMessage": nil, - "lastSender": nil, - "messageCount": msgCount, - "lastActivity": nullStr(lastActivity), - "encrypted": true, - }) - } - - db.setEncChannelsCache(regionParam, channels) - - return channels, nil + return v.([]map[string]interface{}), nil } // getMsgCache/setMsgCache cache GetChannelMessages results, keyed by @@ -1979,7 +2085,7 @@ func (db *DB) GetChannelMessages(channelHash string, limit, offset int, region . if len(regionCodes) > 0 { placeholders := make([]string, len(regionCodes)) for i, code := range regionCodes { - placeholders[i] = "?" + placeholders[i] = fmt.Sprintf("%s", db.parameter(i+2)) regionArgs = append(regionArgs, code) } regionPlaceholders = strings.Join(placeholders, ",") @@ -2006,7 +2112,7 @@ func (db *DB) GetChannelMessages(channelHash string, limit, offset int, region . // 1) Total count (after region filter, before pagination). countSQL := `SELECT COUNT(*) FROM transmissions t - WHERE t.channel_hash = ? AND t.payload_type = 5` + regionFilter + WHERE t.channel_hash = ` + db.parameter(1) + ` AND t.payload_type = 5` + regionFilter countArgs := []interface{}{channelHash} countArgs = append(countArgs, regionArgs...) var total int @@ -2036,16 +2142,16 @@ func (db *DB) GetChannelMessages(channelHash string, limit, offset int, region . pageSQL := `SELECT t.id, COALESCE((SELECT MAX(timestamp) FROM observations WHERE transmission_id = t.id), 0) AS latest_obs_epoch FROM transmissions t - WHERE t.channel_hash = ? AND t.payload_type = 5 - ORDER BY latest_obs_epoch DESC, t.id DESC - LIMIT ? OFFSET ?` + WHERE t.channel_hash = ` + db.parameter(1) + ` AND t.payload_type = 5 + ORDER BY latest_obs_epoch DESC NULLS LAST, t.id DESC + LIMIT ` + fmt.Sprintf("%s OFFSET %s", db.parameter(len(regionArgs)+2), db.parameter(len(regionArgs)+3)) if len(regionCodes) > 0 { pageSQL = `SELECT t.id, COALESCE((SELECT MAX(timestamp) FROM observations WHERE transmission_id = t.id), 0) AS latest_obs_epoch FROM transmissions t - WHERE t.channel_hash = ? AND t.payload_type = 5` + regionFilter + ` - ORDER BY latest_obs_epoch DESC, t.id DESC - LIMIT ? OFFSET ?` + WHERE t.channel_hash = ` + db.parameter(1) + ` AND t.payload_type = 5` + regionFilter + ` + ORDER BY latest_obs_epoch DESC NULLS LAST, t.id DESC + LIMIT ` + fmt.Sprintf("%s OFFSET %s", db.parameter(len(regionArgs)+2), db.parameter(len(regionArgs)+3)) } pageArgs := []interface{}{channelHash} pageArgs = append(pageArgs, regionArgs...) @@ -2077,21 +2183,29 @@ func (db *DB) GetChannelMessages(channelHash string, limit, offset int, region . idPlaceholders := make([]string, len(pageIDs)) obsArgs := make([]interface{}, len(pageIDs)) for i, id := range pageIDs { - idPlaceholders[i] = "?" + idPlaceholders[i] = fmt.Sprintf("%s", db.parameter(i+1)) obsArgs[i] = id } + // #1851: scope_name lives on the transmission row, so appending it as the + // last selected column is safe for both query shapes. + scopeNameCol := "" + if db.hasScopeName { + scopeNameCol = ", t.scope_name" + } + // substr(t.raw_hex, 1, 12): packetpath.HashSize only reads the header, + // transport codes and path byte (6 bytes), not the whole packet. var obsSQL string if db.isV3 { - obsSQL = `SELECT o.id, t.id, t.hash, t.decoded_json, t.first_seen, - obs.id, obs.name, o.snr, o.path_json, o.timestamp + obsSQL = `SELECT o.id, t.id, t.hash, t.decoded_json, t.first_seen, substr(t.raw_hex, 1, 12), + obs.id, obs.name, o.snr, o.path_json, o.timestamp` + scopeNameCol + ` FROM observations o JOIN transmissions t ON t.id = o.transmission_id LEFT JOIN observers obs ON obs.rowid = o.observer_idx WHERE t.id IN (` + strings.Join(idPlaceholders, ",") + `) ORDER BY o.id ASC` } else { - obsSQL = `SELECT o.id, t.id, t.hash, t.decoded_json, t.first_seen, - o.observer_id, o.observer_name, o.snr, o.path_json, o.timestamp + obsSQL = `SELECT o.id, t.id, t.hash, t.decoded_json, t.first_seen, substr(t.raw_hex, 1, 12), + o.observer_id, o.observer_name, o.snr, o.path_json, o.timestamp` + scopeNameCol + ` FROM observations o JOIN transmissions t ON t.id = o.transmission_id WHERE t.id IN (` + strings.Join(idPlaceholders, ",") + `) @@ -2113,10 +2227,15 @@ func (db *DB) GetChannelMessages(channelHash string, limit, offset int, region . for rows.Next() { var pktID, txID int - var pktHash, dj, fs, obsID, obsName, pathJSON sql.NullString + var pktHash, dj, fs, rawHexHead, obsID, obsName, pathJSON sql.NullString var snr sql.NullFloat64 var obsTs sql.NullInt64 - rows.Scan(&pktID, &txID, &pktHash, &dj, &fs, &obsID, &obsName, &snr, &pathJSON, &obsTs) + var scopeName sql.NullString + scanArgs := []interface{}{&pktID, &txID, &pktHash, &dj, &fs, &rawHexHead, &obsID, &obsName, &snr, &pathJSON, &obsTs} + if db.hasScopeName { + scanArgs = append(scanArgs, &scopeName) + } + rows.Scan(scanArgs...) if !dj.Valid { continue } @@ -2167,6 +2286,8 @@ func (db *DB) GetChannelMessages(channelHash string, limit, offset int, region . "observers": []string{}, "hops": hops, "snr": nullFloat(snr), + "scope_name": nullStr(scopeName), + "path_hash_size": packetpath.HashSize(rawHexHead.String), }, Repeats: 1, } @@ -2223,13 +2344,17 @@ func (db *DB) GetChannelMessages(channelHash string, limit, offset int, region . return messages, total, nil } +// IDs are NOT NULL. Keep native index ordering so LIMIT stops the cursor +// after its page, without sorting every row beyond the cursor. +const newTransmissionsSinceSQL = `SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type, t.payload_type, t.payload_version, t.decoded_json + FROM transmissions t WHERE t.id > $1 ORDER BY t.id ASC LIMIT $2` + // GetNewTransmissionsSince returns new transmissions after a given ID for WebSocket polling. func (db *DB) GetNewTransmissionsSince(lastID int, limit int) ([]map[string]interface{}, error) { if limit <= 0 { limit = 100 } - rows, err := db.conn.Query(`SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type, t.payload_type, t.payload_version, t.decoded_json - FROM transmissions t WHERE t.id > ? ORDER BY t.id ASC LIMIT ?`, lastID, limit) + rows, err := db.conn.Query(newTransmissionsSinceSQL, lastID, limit) if err != nil { return nil, err } @@ -2246,6 +2371,7 @@ func (db *DB) GetNewTransmissionsSince(lastID int, limit int) ([]map[string]inte "raw_hex": nullStr(rawHex), "hash": nullStr(hash), "first_seen": nullStr(firstSeen), + "timestamp": nullStr(firstSeen), "route_type": nullInt(routeType), "payload_type": nullInt(payloadType), "payload_version": nullInt(payloadVersion), @@ -2269,23 +2395,25 @@ func (db *DB) GetMaxObservationID() int { return maxID } -// GetObserverPacketCounts returns packetsLastHour for all observers (batch query). -func (db *DB) GetObserverPacketCounts(sinceEpoch int64) map[string]int { - counts := make(map[string]int) - var rows *sql.Rows - var err error +// observerPacketCountsSQL is shared with the native planner regression. +func (db *DB) observerPacketCountsSQL() string { if db.isV3 { - rows, err = db.conn.Query(`SELECT obs.id, COUNT(*) as cnt + return `SELECT obs.id, COUNT(*) as cnt FROM observations o JOIN observers obs ON obs.rowid = o.observer_idx - WHERE o.timestamp > ? - GROUP BY obs.id`, sinceEpoch) - } else { - rows, err = db.conn.Query(`SELECT o.observer_id, COUNT(*) as cnt - FROM observations o - WHERE o.observer_id IS NOT NULL AND o.timestamp > ? - GROUP BY o.observer_id`, sinceEpoch) + WHERE o.timestamp > ` + db.parameter(1) + ` + GROUP BY obs.id` } + return `SELECT o.observer_id, COUNT(*) as cnt + FROM observations o + WHERE o.observer_id IS NOT NULL AND o.timestamp > ` + db.parameter(1) + ` + GROUP BY o.observer_id` +} + +// GetObserverPacketCounts returns packetsLastHour for all observers (batch query). +func (db *DB) GetObserverPacketCounts(sinceEpoch int64) map[string]int { + counts := make(map[string]int) + rows, err := db.conn.Query(db.observerPacketCountsSQL(), sinceEpoch) if err != nil { return counts } @@ -2331,7 +2459,7 @@ func (db *DB) GetNodeLocationsByKeys(keys []string) map[string]map[string]interf placeholders := make([]string, len(keys)) args := make([]interface{}, len(keys)) for i, k := range keys { - placeholders[i] = "?" + placeholders[i] = fmt.Sprintf("%s", db.parameter(i+1)) args[i] = strings.ToLower(k) } // #1481 P0-3: drop LOWER(public_key) — that wrap is non-sargable and @@ -2359,6 +2487,7 @@ func (db *DB) GetNodeLocationsByKeys(keys []string) map[string]map[string]interf // QueryMultiNodePackets returns transmissions referencing any of the given pubkeys. func (db *DB) QueryMultiNodePackets(pubkeys []string, limit, offset int, order, since, until string) (*PacketResult, error) { + offset = max(0, offset) if len(pubkeys) == 0 { return &PacketResult{Packets: []map[string]interface{}{}, Total: 0}, nil } @@ -2376,17 +2505,17 @@ func (db *DB) QueryMultiNodePackets(pubkeys []string, limit, offset int, order, for _, pk := range pubkeys { resolved := db.resolveNodePubkey(pk) args = append(args, resolved) - placeholders = append(placeholders, "?") + placeholders = append(placeholders, fmt.Sprintf("%s", db.parameter(len(args)))) } pkWhere := "t.from_pubkey IN (" + strings.Join(placeholders, ",") + ")" var timeFilters []string if since != "" { - timeFilters = append(timeFilters, "t.first_seen >= ?") + timeFilters = append(timeFilters, fmt.Sprintf("t.first_seen >= %s", db.parameter(len(args)+1))) args = append(args, since) } if until != "" { - timeFilters = append(timeFilters, "t.first_seen <= ?") + timeFilters = append(timeFilters, fmt.Sprintf("t.first_seen <= %s", db.parameter(len(args)+1))) args = append(args, until) } @@ -2400,8 +2529,8 @@ func (db *DB) QueryMultiNodePackets(pubkeys []string, limit, offset int, order, selectCols, observerJoin := db.transmissionBaseSQL() // #1345: order by ingest id (see QueryPackets comment above). - querySQL := fmt.Sprintf("SELECT %s FROM transmissions t %s %s ORDER BY t.id %s LIMIT ? OFFSET ?", - selectCols, observerJoin, w, order) + querySQL := fmt.Sprintf("SELECT %s FROM transmissions t %s %s ORDER BY t.id %s LIMIT %s OFFSET %s", + selectCols, observerJoin, w, order, db.parameter(len(args)+1), db.parameter(len(args)+2)) qArgs := make([]interface{}, len(args)) copy(qArgs, args) @@ -2428,8 +2557,8 @@ func (db *DB) QueryMultiNodePackets(pubkeys []string, limit, offset int, order, func scanPacketRow(rows *sql.Rows) map[string]interface{} { var id int var rawHex, ts, obsID, obsName, direction, hash, pathJSON, decodedJSON, createdAt sql.NullString - var snr, rssi sql.NullFloat64 - var score, routeType, payloadType, payloadVersion sql.NullInt64 + var snr, rssi, score sql.NullFloat64 + var routeType, payloadType, payloadVersion sql.NullInt64 if err := rows.Scan(&id, &rawHex, &ts, &obsID, &obsName, &direction, &snr, &rssi, &score, &hash, &routeType, &payloadType, &payloadVersion, &pathJSON, &decodedJSON, &createdAt); err != nil { return nil @@ -2443,7 +2572,7 @@ func scanPacketRow(rows *sql.Rows) map[string]interface{} { "direction": nullStr(direction), "snr": nullFloat(snr), "rssi": nullFloat(rssi), - "score": nullInt(score), + "score": nullFloat(score), "hash": nullStr(hash), "route_type": nullInt(routeType), "payload_type": nullInt(payloadType), @@ -2465,11 +2594,15 @@ func (db *DB) scanNodeRow(rows *sql.Rows) map[string]interface{} { var temperatureC sql.NullFloat64 var foreign sql.NullInt64 var defaultScope sql.NullString + var configuredScope, configuredScopeAt sql.NullString scanArgs := []interface{}{&pk, &name, &role, &lat, &lon, &lastSeen, &firstSeen, &advertCount, &batteryMv, &temperatureC, &foreign} if db.hasDefaultScope { scanArgs = append(scanArgs, &defaultScope) } + if db.hasConfiguredScope { + scanArgs = append(scanArgs, &configuredScope, &configuredScopeAt) + } if err := rows.Scan(scanArgs...); err != nil { return nil } @@ -2500,6 +2633,10 @@ func (db *DB) scanNodeRow(rows *sql.Rows) map[string]interface{} { if db.hasDefaultScope { m["default_scope"] = nullStr(defaultScope) } + if db.hasConfiguredScope { + m["configured_scope"] = nullStr(configuredScope) + m["configured_scope_at"] = nullStr(configuredScopeAt) + } return m } @@ -2552,7 +2689,7 @@ func nullInt(ni sql.NullInt64) interface{} { // PruneOldPackets, PruneOldMetrics, and RemoveStaleObservers were // removed in #1283 — they are write operations and now live on the // ingestor's *Store (cmd/ingestor/maintenance.go and cmd/ingestor/db.go). -// The server is the read path; it must not hold the SQLite write lock. +// The server is the read path; it must not write telemetry. // MetricsSample represents a single row from observer_metrics with computed deltas. type MetricsSample struct { @@ -2604,37 +2741,47 @@ func (db *DB) GetObserverMetrics(observerID, since, until, resolution string, sa // Use LAST value per bucket (latest timestamp) instead of MAX to preserve // reboot semantics: if a device reboots mid-bucket, the last sample is the // post-reboot baseline, not the pre-reboot high-water mark. - query = `SELECT ts, noise_floor, tx_air_secs, rx_air_secs, recv_errors, battery_mv, packets_sent, packets_recv FROM ( + query = db.nativeSQL(`SELECT ts, noise_floor, tx_air_secs, rx_air_secs, recv_errors, battery_mv, packets_sent, packets_recv FROM ( SELECT strftime('%Y-%m-%dT%H:00:00Z', timestamp) as ts, noise_floor, tx_air_secs, rx_air_secs, recv_errors, battery_mv, packets_sent, packets_recv, ROW_NUMBER() OVER (PARTITION BY observer_id, strftime('%Y-%m-%dT%H:00:00Z', timestamp) ORDER BY timestamp DESC) as rn - FROM observer_metrics WHERE observer_id = ?` + FROM observer_metrics WHERE observer_id = ?1`, `SELECT ts, noise_floor, tx_air_secs, rx_air_secs, recv_errors, battery_mv, packets_sent, packets_recv FROM ( + SELECT + to_char(timestamp::timestamptz AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:00:00"Z"') as ts, + noise_floor, tx_air_secs, rx_air_secs, recv_errors, battery_mv, packets_sent, packets_recv, + ROW_NUMBER() OVER (PARTITION BY observer_id, to_char(timestamp::timestamptz AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:00:00"Z"') ORDER BY timestamp DESC) as rn + FROM observer_metrics WHERE observer_id = `+db.parameter(1)) case "1d": bucketSizeSec = 86400 - query = `SELECT ts, noise_floor, tx_air_secs, rx_air_secs, recv_errors, battery_mv, packets_sent, packets_recv FROM ( + query = db.nativeSQL(`SELECT ts, noise_floor, tx_air_secs, rx_air_secs, recv_errors, battery_mv, packets_sent, packets_recv FROM ( SELECT strftime('%Y-%m-%dT00:00:00Z', timestamp) as ts, noise_floor, tx_air_secs, rx_air_secs, recv_errors, battery_mv, packets_sent, packets_recv, ROW_NUMBER() OVER (PARTITION BY observer_id, strftime('%Y-%m-%dT00:00:00Z', timestamp) ORDER BY timestamp DESC) as rn - FROM observer_metrics WHERE observer_id = ?` + FROM observer_metrics WHERE observer_id = ?1`, `SELECT ts, noise_floor, tx_air_secs, rx_air_secs, recv_errors, battery_mv, packets_sent, packets_recv FROM ( + SELECT + to_char(timestamp::timestamptz AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"00:00:00"Z"') as ts, + noise_floor, tx_air_secs, rx_air_secs, recv_errors, battery_mv, packets_sent, packets_recv, + ROW_NUMBER() OVER (PARTITION BY observer_id, to_char(timestamp::timestamptz AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"00:00:00"Z"') ORDER BY timestamp DESC) as rn + FROM observer_metrics WHERE observer_id = `+db.parameter(1)) default: // "5m" or raw query = `SELECT timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors, battery_mv, packets_sent, packets_recv - FROM observer_metrics WHERE observer_id = ?` + FROM observer_metrics WHERE observer_id = ` + db.parameter(1) } if since != "" { - query += " AND timestamp >= ?" + query += fmt.Sprintf(" AND timestamp >= %s", db.parameter(len(args)+1)) args = append(args, since) } if until != "" { - query += " AND timestamp <= ?" + query += fmt.Sprintf(" AND timestamp <= %s", db.parameter(len(args)+1)) args = append(args, until) } switch resolution { case "1h", "1d": - query += ") WHERE rn = 1 ORDER BY ts ASC" + query += ") WHERE rn = 1 ORDER BY ts ASC NULLS FIRST" default: query += " ORDER BY timestamp ASC" } @@ -2804,7 +2951,7 @@ func (db *DB) GetMetricsSummary(since string) ([]MetricsSummaryRow, error) { SELECT observer_id, noise_floor, battery_mv, ROW_NUMBER() OVER (PARTITION BY observer_id ORDER BY timestamp DESC) as rn FROM observer_metrics - WHERE timestamp >= ? + WHERE timestamp >= ` + db.parameter(1) + ` ) SELECT m.observer_id, o.name, COALESCE(o.iata, '') as iata, r.noise_floor as current_nf, @@ -2815,9 +2962,9 @@ func (db *DB) GetMetricsSummary(since string) ([]MetricsSummaryRow, error) { FROM observer_metrics m LEFT JOIN observers o ON o.id = m.observer_id LEFT JOIN ranked r ON r.observer_id = m.observer_id AND r.rn = 1 - WHERE m.timestamp >= ? - GROUP BY m.observer_id - ORDER BY max_nf DESC + WHERE m.timestamp >= ` + db.parameter(2) + ` + GROUP BY m.observer_id, o.name, o.iata, r.noise_floor, r.battery_mv + ORDER BY max_nf DESC NULLS LAST ` rows, err := db.conn.Query(query, since, since) if err != nil { @@ -2840,7 +2987,7 @@ func (db *DB) GetMetricsSummary(since string) ([]MetricsSummaryRow, error) { // Fetch sparkline data (noise_floor series) for all observers in one query if len(result) > 0 { sparkQuery := `SELECT observer_id, noise_floor FROM observer_metrics - WHERE timestamp >= ? ORDER BY observer_id, timestamp ASC` + WHERE timestamp >= ` + db.parameter(1) + ` ORDER BY observer_id, timestamp ASC` sparkRows, err := db.conn.Query(sparkQuery, since) if err != nil { return nil, err @@ -2882,17 +3029,17 @@ func (db *DB) GetDroppedPackets(limit int, observerID, nodePubkey string) ([]map var conditions []string var args []interface{} if observerID != "" { - conditions = append(conditions, "observer_id = ?") + conditions = append(conditions, fmt.Sprintf("observer_id = %s", db.parameter(len(args)+1))) args = append(args, observerID) } if nodePubkey != "" { - conditions = append(conditions, "node_pubkey = ?") + conditions = append(conditions, fmt.Sprintf("node_pubkey = %s", db.parameter(len(args)+1))) args = append(args, nodePubkey) } if len(conditions) > 0 { query += " WHERE " + strings.Join(conditions, " AND ") } - query += " ORDER BY dropped_at DESC LIMIT ?" + query += fmt.Sprintf(" ORDER BY dropped_at DESC NULLS LAST LIMIT %s", db.parameter(len(args)+1)) args = append(args, limit) rows, err := db.conn.Query(query, args...) @@ -2990,16 +3137,16 @@ func (db *DB) GetScopeStats(window string) (*ScopeStatsResponse, error) { case "1h": since = time.Now().Add(-1 * time.Hour).UTC().Format(time.RFC3339) // 5-minute buckets - bucketExpr = `strftime('%Y-%m-%dT%H:', first_seen) || printf('%02d', (CAST(strftime('%M', first_seen) AS INTEGER) / 5) * 5) || ':00Z'` + bucketExpr = db.nativeSQL(`strftime('%Y-%m-%dT%H:', first_seen) || printf('%02d', (CAST(strftime('%M', first_seen) AS INTEGER) / 5) * 5) || ':00Z'`, `to_char(date_bin('5 minutes', (CASE WHEN pg_input_is_valid(first_seen, 'timestamp with time zone') THEN first_seen::timestamptz END), '2000-01-01T00:00:00Z'::timestamptz) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"')`) case "7d": since = time.Now().Add(-7 * 24 * time.Hour).UTC().Format(time.RFC3339) // 6-hour buckets - bucketExpr = `strftime('%Y-%m-%dT', first_seen) || printf('%02d', (CAST(strftime('%H', first_seen) AS INTEGER) / 6) * 6) || ':00:00Z'` + bucketExpr = db.nativeSQL(`strftime('%Y-%m-%dT', first_seen) || printf('%02d', (CAST(strftime('%H', first_seen) AS INTEGER) / 6) * 6) || ':00:00Z'`, `to_char(date_bin('6 hours', (CASE WHEN pg_input_is_valid(first_seen, 'timestamp with time zone') THEN first_seen::timestamptz END), '2000-01-01T00:00:00Z'::timestamptz) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"')`) default: // "24h" window = "24h" since = time.Now().Add(-24 * time.Hour).UTC().Format(time.RFC3339) // 1-hour buckets - bucketExpr = `strftime('%Y-%m-%dT%H:00:00Z', first_seen)` + bucketExpr = db.nativeSQL(`strftime('%Y-%m-%dT%H:00:00Z', first_seen)`, `to_char((CASE WHEN pg_input_is_valid(first_seen, 'timestamp with time zone') THEN first_seen::timestamptz END) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:00:00"Z"')`) } resp := &ScopeStatsResponse{Window: window} @@ -3012,7 +3159,7 @@ func (db *DB) GetScopeStats(window string) (*ScopeStatsResponse, error) { COALESCE(SUM(CASE WHEN scope_name IS NULL THEN 1 ELSE 0 END), 0) AS unscoped, COALESCE(SUM(CASE WHEN scope_name = '' THEN 1 ELSE 0 END), 0) AS unknown_scope FROM transmissions - WHERE `+routeTypeTransportSQL+` AND first_seen >= ? + WHERE `+routeTypeTransportSQL+` AND first_seen >= `+db.parameter(1)+` `, since) if err := row.Scan( &resp.Summary.TransportTotal, @@ -3030,7 +3177,7 @@ func (db *DB) GetScopeStats(window string) (*ScopeStatsResponse, error) { var nonTransportUnscoped int if err := db.conn.QueryRow(` SELECT COUNT(*) FROM transmissions - WHERE `+routeTypeNonTransportSQL+` AND first_seen >= ? + WHERE `+routeTypeNonTransportSQL+` AND first_seen >= `+db.parameter(1)+` `, since).Scan(&nonTransportUnscoped); err != nil { return nil, fmt.Errorf("scope non-transport count query: %w", err) } @@ -3040,9 +3187,9 @@ func (db *DB) GetScopeStats(window string) (*ScopeStatsResponse, error) { rows, err := db.conn.Query(` SELECT scope_name, COUNT(*) AS cnt FROM transmissions - WHERE `+routeTypeTransportSQL+` AND scope_name IS NOT NULL AND scope_name != '' AND first_seen >= ? + WHERE `+routeTypeTransportSQL+` AND scope_name IS NOT NULL AND scope_name != '' AND first_seen >= `+db.parameter(1)+` GROUP BY scope_name - ORDER BY cnt DESC + ORDER BY cnt DESC NULLS LAST `, since) if err != nil { return nil, fmt.Errorf("scope byRegion query: %w", err) @@ -3067,7 +3214,7 @@ func (db *DB) GetScopeStats(window string) (*ScopeStatsResponse, error) { COUNT(scope_name) AS scoped, SUM(CASE WHEN scope_name IS NULL THEN 1 ELSE 0 END) AS unscoped FROM transmissions - WHERE `+routeTypeTransportSQL+` AND first_seen >= ? + WHERE `+routeTypeTransportSQL+` AND first_seen >= `+db.parameter(1)+` GROUP BY bucket ORDER BY bucket `, bucketExpr) @@ -3089,6 +3236,38 @@ func (db *DB) GetScopeStats(window string) (*ScopeStatsResponse, error) { resp.TimeSeries = []ScopeTimePoint{} } + // #1979: flood adverts by sender role, split by the three scope_name + // states. Flood routes only (TRANSPORT_FLOOD 0, FLOOD 1): zero-hop adverts + // go out as DIRECT/TRANSPORT_DIRECT (firmware Mesh::sendZeroHop) and are + // not flooded. Role is the sender's current nodes.role. + // The unary + on payload_type keeps the planner on the first_seen range + // index; the payload_type index would walk every advert ever stored. + roleRows, err := db.conn.Query(` + SELECT COALESCE(NULLIF(n.role, ''), 'unknown') AS role, + SUM(CASE WHEN t.scope_name IS NULL THEN 1 ELSE 0 END) AS unscoped, + SUM(CASE WHEN t.scope_name = '' THEN 1 ELSE 0 END) AS unknown_scope, + SUM(CASE WHEN t.scope_name != '' THEN 1 ELSE 0 END) AS named + FROM transmissions t + LEFT JOIN nodes n ON n.public_key = t.from_pubkey + WHERE +t.payload_type = `+db.parameter(1)+` AND t.route_type IN (0, 1) AND t.first_seen >= `+db.parameter(2)+` + GROUP BY 1 + ORDER BY COUNT(*) DESC NULLS LAST, 1 + `, payloadTypeAdvert, since) + if err != nil { + return nil, fmt.Errorf("scope advertsByRole query: %w", err) + } + defer roleRows.Close() + resp.AdvertsByRole = []ScopeAdvertRoleCount{} + for roleRows.Next() { + var rc ScopeAdvertRoleCount + if roleRows.Scan(&rc.Role, &rc.Unscoped, &rc.UnknownScope, &rc.Named) == nil { + resp.AdvertsByRole = append(resp.AdvertsByRole, rc) + } + } + if err := roleRows.Err(); err != nil { + return nil, fmt.Errorf("scope advertsByRole iteration: %w", err) + } + return resp, nil } diff --git a/cmd/server/db_channel_messages_perf_test.go b/cmd/server/db_channel_messages_perf_test.go index b96fc8d42..9a7a0d1dd 100644 --- a/cmd/server/db_channel_messages_perf_test.go +++ b/cmd/server/db_channel_messages_perf_test.go @@ -37,39 +37,15 @@ func TestGetChannelMessagesPerfLargeChannel(t *testing.T) { const numTx = 1500 const obsPerTx = 50 - tx, err := db.conn.Begin() - if err != nil { - t.Fatal(err) - } - txStmt, err := tx.Prepare(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash) - VALUES (?, ?, ?, 1, 5, ?, '#perf')`) - if err != nil { - t.Fatal(err) - } - obsStmt, err := tx.Prepare(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (?, 1, 10.0, -90, '[]', ?)`) - if err != nil { - t.Fatal(err) - } base := time.Now().UTC().Add(-24 * time.Hour) - for i := 0; i < numTx; i++ { - ts := base.Add(time.Duration(i) * time.Second).Format(time.RFC3339) - hash := fmt.Sprintf("perfhash%08d", i) + copyTestRows(t, db.conn, "transmissions", []string{"id", "raw_hex", "hash", "first_seen", "route_type", "payload_type", "decoded_json", "channel_hash"}, numTx, func(i int) []any { body := fmt.Sprintf(`{"type":"CHAN","channel":"#perf","text":"Sender%d: msg %d","sender":"Sender%d"}`, i%10, i, i%10) - res, err := txStmt.Exec(fmt.Sprintf("%04X", i), hash, ts, body) - if err != nil { - t.Fatal(err) - } - txID, _ := res.LastInsertId() - for o := 0; o < obsPerTx; o++ { - if _, err := obsStmt.Exec(txID, base.Unix()+int64(i*100+o)); err != nil { - t.Fatal(err) - } - } - } - if err := tx.Commit(); err != nil { - t.Fatal(err) - } + return []any{i + 1, fmt.Sprintf("%04X", i), fmt.Sprintf("perfhash%08d", i), base.Add(time.Duration(i) * time.Second).Format(time.RFC3339), 1, 5, body, "#perf"} + }) + copyTestRows(t, db.conn, "observations", []string{"id", "transmission_id", "observer_idx", "snr", "rssi", "path_json", "timestamp"}, numTx*obsPerTx, func(n int) []any { + i, o := n/obsPerTx, n%obsPerTx + return []any{n + 1, i + 1, 1, 10.0, -90.0, "[]", base.Unix() + int64(i*100+o)} + }) // Warm-up call to amortize first-run prepare cost. if _, _, err := db.GetChannelMessages("#perf", 50, 0); err != nil { diff --git a/cmd/server/db_channels_singleflight_test.go b/cmd/server/db_channels_singleflight_test.go new file mode 100644 index 000000000..f4e38c947 --- /dev/null +++ b/cmd/server/db_channels_singleflight_test.go @@ -0,0 +1,129 @@ +package main + +import ( + "sync" + "sync/atomic" + "testing" + "time" +) + +// TestGetChannels_SingleflightCoalescesQueries (issue #2029) asserts that N +// concurrent cache-miss callers trigger at most ONE real query, using a call +// counter rather than timing. Timing alone ("finish within Nms of each +// other") passes on a fast machine even without coalescing, since it doesn't +// distinguish "one shared execution" from "N independent executions that all +// happened to be fast" — same rationale as TestEnsureNeighborGraph_Singleflight +// (#1203 Pair A) and statsSF (#1910). +// +// Anti-tautology: revert channelsSF.Do back to a bare call and this test +// fails (it observes N instead of 1). +func TestGetChannels_SingleflightCoalescesQueries(t *testing.T) { + db := setupTestDB(t) + defer db.Close() + seedTestData(t, db) + + var calls int32 + db.channelsQueryHook = func() { + atomic.AddInt32(&calls, 1) + time.Sleep(50 * time.Millisecond) // ensure callers actually overlap + } + + var wg sync.WaitGroup + const N = 10 + errs := make(chan error, N) + wg.Add(N) + for i := 0; i < N; i++ { + go func() { + defer wg.Done() + if _, err := db.GetChannels(); err != nil { + errs <- err + } + }() + } + wg.Wait() + close(errs) + for err := range errs { + t.Errorf("concurrent GetChannels: %v", err) + } + + got := atomic.LoadInt32(&calls) + if got != 1 { + // got==0 would mean the hook never fired (a broken test, not a + // broken fix). got>1 means the query ran once per caller, i.e. + // singleflight is missing or keyed wrong. Both are caught by !=1. + t.Fatalf("expected exactly 1 real query under singleflight, got %d", got) + } +} + +// TestGetEncryptedChannels_SingleflightCoalescesQueries mirrors the above for +// GetEncryptedChannels/encChannelsSF, which has the identical bug shape and +// fix (see #2029). +func TestGetEncryptedChannels_SingleflightCoalescesQueries(t *testing.T) { + db := setupTestDB(t) + defer db.Close() + seedTestData(t, db) + + var calls int32 + db.encChannelsQueryHook = func() { + atomic.AddInt32(&calls, 1) + time.Sleep(50 * time.Millisecond) + } + + var wg sync.WaitGroup + const N = 10 + errs := make(chan error, N) + wg.Add(N) + for i := 0; i < N; i++ { + go func() { + defer wg.Done() + if _, err := db.GetEncryptedChannels(); err != nil { + errs <- err + } + }() + } + wg.Wait() + close(errs) + for err := range errs { + t.Errorf("concurrent GetEncryptedChannels: %v", err) + } + + got := atomic.LoadInt32(&calls) + if got != 1 { + t.Fatalf("expected exactly 1 real query under singleflight, got %d", got) + } +} + +// TestGetChannels_SingleflightPerRegion asserts channelsSF is keyed per +// region, not a single shared slot — two different regions queried +// concurrently must each get their own query, or a caller for region A would +// wrongly receive region B's coalesced result. +func TestGetChannels_SingleflightPerRegion(t *testing.T) { + db := setupTestDB(t) + defer db.Close() + seedTestData(t, db) + + var calls int32 + db.channelsQueryHook = func() { + atomic.AddInt32(&calls, 1) + time.Sleep(30 * time.Millisecond) + } + + var wg sync.WaitGroup + regions := []string{"SAT", "DFW"} + wg.Add(len(regions) * 5) + for _, region := range regions { + region := region + for i := 0; i < 5; i++ { + go func() { + defer wg.Done() + _, _ = db.GetChannels(region) + }() + } + } + wg.Wait() + + got := atomic.LoadInt32(&calls) + if got != 2 { + t.Fatalf("expected exactly 1 real query per distinct region key (2 regions), got %d", got) + } +} diff --git a/cmd/server/db_test.go b/cmd/server/db_test.go index 51bb10ca1..895e48a27 100644 --- a/cmd/server/db_test.go +++ b/cmd/server/db_test.go @@ -6,25 +6,30 @@ import ( "errors" "os" "path/filepath" + "reflect" + "strconv" "strings" "testing" "time" - - _ "modernc.org/sqlite" ) // setupTestDB creates an in-memory SQLite database with the v3 schema. -func setupTestDB(t *testing.T) *DB { +func setupTestDB(t testing.TB) *DB { + return setupTestDBAtURL(t, postgresTestDSN(t)) +} + +func setupTestDBAtURL(t testing.TB, dsn string) *DB { t.Helper() - conn, err := sql.Open("sqlite", ":memory:") + conn, err := openFixtureSQL(dsn) if err != nil { t.Fatal(err) } - // Force single connection so all goroutines share the same in-memory DB + t.Cleanup(func() { conn.Close() }) + // One fixture connection keeps write and assertion order deterministic. conn.SetMaxOpenConns(1) // Create schema matching MeshCore Analyzer v3 - schema := ` + schema := testNativeSQL(` CREATE TABLE nodes ( public_key TEXT PRIMARY KEY, name TEXT, @@ -136,12 +141,122 @@ func setupTestDB(t *testing.T) *DB { CREATE INDEX IF NOT EXISTS idx_observations_timestamp ON observations(timestamp); CREATE INDEX IF NOT EXISTS idx_observations_tx_ts ON observations(transmission_id, timestamp); CREATE INDEX IF NOT EXISTS idx_transmissions_channel_hash ON transmissions(channel_hash); - ` + `, ` + CREATE TABLE nodes ( + public_key TEXT PRIMARY KEY, + name TEXT, + role TEXT, + lat DOUBLE PRECISION, + lon DOUBLE PRECISION, + last_seen TEXT, + first_seen TEXT, + advert_count INTEGER DEFAULT 0, + battery_mv INTEGER, + temperature_c DOUBLE PRECISION, + foreign_advert INTEGER DEFAULT 0 + ); + + CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY UNIQUE, + id TEXT PRIMARY KEY, + name TEXT, + iata TEXT, + last_seen TEXT, + first_seen TEXT, + packet_count INTEGER DEFAULT 0, + model TEXT, + firmware TEXT, + client_version TEXT, + radio TEXT, + battery_mv INTEGER, + uptime_secs INTEGER, + noise_floor DOUBLE PRECISION, + inactive INTEGER DEFAULT 0, + last_packet_at TEXT DEFAULT NULL, + clock_skew_seconds INTEGER DEFAULT NULL, + clock_skew_count_24h INTEGER DEFAULT 0, + clock_last_naive_at TEXT DEFAULT NULL + ); + + CREATE TABLE transmissions ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + raw_hex TEXT NOT NULL, + hash TEXT NOT NULL UNIQUE, + first_seen TEXT NOT NULL, + route_type INTEGER, + payload_type INTEGER, + payload_version INTEGER, + decoded_json TEXT, + channel_hash TEXT DEFAULT NULL, + from_pubkey TEXT DEFAULT NULL, + created_at TEXT DEFAULT (to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"')) + ); + + CREATE TABLE observations ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + transmission_id INTEGER NOT NULL REFERENCES transmissions(id), + observer_idx INTEGER, + direction TEXT, + snr DOUBLE PRECISION, + rssi DOUBLE PRECISION, + score INTEGER, + path_json TEXT, + timestamp INTEGER NOT NULL, + resolved_path TEXT, + raw_hex TEXT + ); + + CREATE TABLE IF NOT EXISTS observer_metrics ( + observer_id TEXT NOT NULL, + timestamp TEXT NOT NULL, + noise_floor DOUBLE PRECISION, + tx_air_secs INTEGER, + rx_air_secs INTEGER, + recv_errors INTEGER, + battery_mv INTEGER, + packets_sent INTEGER, + packets_recv INTEGER, + PRIMARY KEY (observer_id, timestamp) + ); + + CREATE INDEX IF NOT EXISTS idx_observer_metrics_timestamp ON observer_metrics(timestamp); + + -- Auto-populate from_pubkey for ADVERT rows so existing test fixtures + -- (which only set decoded_json) still attribute correctly under #1143's + -- exact-match column. Production migration handles legacy data; the + -- ingestor sets the column at write time. + -- + -- m4 alignment: prod ingest leaves from_pubkey NULL when pubKey is + -- missing or empty (cmd/ingestor/db.go ~1289 guards PubKey != empty-string). + -- The trigger mirrors that: only assign when json_extract yields a + -- non-empty string. json_extract returns NULL for missing keys, so + -- the explicit IS NOT NULL AND <> empty-string guard catches the empty-string + -- case too. UPDATE only when we have something to write. + CREATE FUNCTION test_from_pubkey_advert() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN + IF NEW.from_pubkey IS NULL AND NEW.payload_type = 4 AND NEW.decoded_json IS JSON OBJECT THEN + NEW.from_pubkey := NULLIF(NEW.decoded_json::json ->> 'pubKey', ''); + END IF; + RETURN NEW; + END $$; + CREATE TRIGGER test_from_pubkey_advert BEFORE INSERT ON transmissions + FOR EACH ROW EXECUTE FUNCTION test_from_pubkey_advert(); + CREATE INDEX IF NOT EXISTS idx_transmissions_from_pubkey ON transmissions(from_pubkey); + + -- Mirror prod indexes from internal/dbschema/dbschema.go so query plans + -- in tests match prod. idx_observations_transmission_id is required by + -- GetChannelMessages's grouped MAX(timestamp) per tx aggregate + -- (issue #1366 / PR #1368): without it the perf test on 1500 tx × 50 obs + -- blows the 1.5s budget under -race. + CREATE INDEX IF NOT EXISTS idx_observations_transmission_id ON observations(transmission_id); + CREATE INDEX IF NOT EXISTS idx_observations_timestamp ON observations(timestamp); + CREATE INDEX IF NOT EXISTS idx_observations_tx_ts ON observations(transmission_id, timestamp); + CREATE INDEX IF NOT EXISTS idx_transmissions_channel_hash ON transmissions(channel_hash); + `) if _, err := conn.Exec(schema); err != nil { t.Fatal(err) } - return &DB{conn: conn, isV3: true, hasResolvedPath: true} + return &DB{backend: testBackendValue(), conn: conn, path: dsn, stateDir: t.TempDir(), isV3: true, hasResolvedPath: true} } func seedTestData(t *testing.T, db *DB) { @@ -156,37 +271,64 @@ func seedTestData(t *testing.T, db *DB) { // Seed observers db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obs1', 'Observer One', 'SJC', ?, '2026-01-01T00:00:00Z', 100)`, recent) + VALUES ('obs1', 'Observer One', 'SJC', $1, '2026-01-01T00:00:00Z', 100)`, recent) db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obs2', 'Observer Two', 'SFO', ?, '2026-01-01T00:00:00Z', 50)`, yesterday) + VALUES ('obs2', 'Observer Two', 'SFO', $1, '2026-01-01T00:00:00Z', 50)`, yesterday) // Seed nodes db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES ('aabbccdd11223344', 'TestRepeater', 'repeater', 37.5, -122.0, ?, '2026-01-01T00:00:00Z', 50)`, recent) + VALUES ('aabbccdd11223344', 'TestRepeater', 'repeater', 37.5, -122.0, $1, '2026-01-01T00:00:00Z', 50)`, recent) db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES ('eeff00112233aabb', 'TestCompanion', 'companion', 37.6, -122.1, ?, '2026-01-01T00:00:00Z', 10)`, yesterday) + VALUES ('eeff00112233aabb', 'TestCompanion', 'companion', 37.6, -122.1, $1, '2026-01-01T00:00:00Z', 10)`, yesterday) db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES ('1122334455667788', 'TestRoom', 'room', 37.4, -121.9, ?, '2026-01-01T00:00:00Z', 5)`, twoDaysAgo) + VALUES ('1122334455667788', 'TestRoom', 'room', 37.4, -121.9, $1, '2026-01-01T00:00:00Z', 5)`, twoDaysAgo) // Seed transmissions db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash, from_pubkey) - VALUES ('AABB', 'abc123def4567890', ?, 1, 4, '{"pubKey":"aabbccdd11223344","name":"TestRepeater","type":"ADVERT","timestamp":1700000000,"timestampISO":"2023-11-14T22:13:20.000Z","signature":"abcdef","flags":{"isRepeater":true},"lat":37.5,"lon":-122.0}', '#test', 'aabbccdd11223344')`, recent) + VALUES ('AABB', 'abc123def4567890', $1, 1, 4, '{"pubKey":"aabbccdd11223344","name":"TestRepeater","type":"ADVERT","timestamp":1700000000,"timestampISO":"2023-11-14T22:13:20.000Z","signature":"abcdef","flags":{"isRepeater":true},"lat":37.5,"lon":-122.0}', '#test', 'aabbccdd11223344')`, recent) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash) - VALUES ('CCDD', '1234567890abcdef', ?, 1, 5, '{"type":"CHAN","channel":"#test","text":"Hello: World","sender":"TestUser"}', '#test')`, yesterday) + VALUES ('CCDD', '1234567890abcdef', $1, 1, 5, '{"type":"CHAN","channel":"#test","text":"Hello: World","sender":"TestUser"}', '#test')`, yesterday) // Second ADVERT for same node with different hash_size (raw_hex byte 0x1F → hs=1 vs 0xBB → hs=3) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, from_pubkey) - VALUES ('AA1F', 'def456abc1230099', ?, 1, 4, '{"pubKey":"aabbccdd11223344","name":"TestRepeater","type":"ADVERT","timestamp":1700000100,"timestampISO":"2023-11-14T22:14:40.000Z","signature":"fedcba","flags":{"isRepeater":true},"lat":37.5,"lon":-122.0}', 'aabbccdd11223344')`, yesterday) + VALUES ('AA1F', 'def456abc1230099', $1, 1, 4, '{"pubKey":"aabbccdd11223344","name":"TestRepeater","type":"ADVERT","timestamp":1700000100,"timestampISO":"2023-11-14T22:14:40.000Z","signature":"fedcba","flags":{"isRepeater":true},"lat":37.5,"lon":-122.0}', 'aabbccdd11223344')`, yesterday) // Seed observations (use unix timestamps) // resolved_path contains full pubkeys parallel to path_json hops db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp, resolved_path) - VALUES (1, 1, 12.5, -90, '["aa","bb"]', ?, '["aabbccdd11223344","eeff00112233aabb"]')`, recentEpoch) + VALUES (1, 1, 12.5, -90, '["aa","bb"]', $1, '["aabbccdd11223344","eeff00112233aabb"]')`, recentEpoch) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp, resolved_path) - VALUES (1, 2, 8.0, -95, '["aa"]', ?, '["aabbccdd11223344"]')`, recentEpoch-100) + VALUES (1, 2, 8.0, -95, '["aa"]', $1, '["aabbccdd11223344"]')`, recentEpoch-100) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (2, 1, 15.0, -85, '[]', ?)`, yesterdayEpoch) + VALUES (2, 1, 15.0, -85, '[]', $1)`, yesterdayEpoch) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp, resolved_path) - VALUES (3, 1, 10.0, -92, '["cc"]', ?, '["1122334455667788"]')`, yesterdayEpoch) + VALUES (3, 1, 10.0, -92, '["cc"]', $1, '["1122334455667788"]')`, yesterdayEpoch) +} + +// The topology endpoint takes uniqueNodes from CountActiveNodes, so it must +// count what GetStats reports as totalNodes: nodes seen in the last 7 days. +func TestCountActiveNodes(t *testing.T) { + db := setupTestDB(t) + defer db.Close() + now := time.Now().UTC() + for i, seen := range []time.Time{now.Add(-time.Hour), now.Add(-6 * 24 * time.Hour), now.Add(-8 * 24 * time.Hour)} { + db.conn.Exec(`INSERT INTO nodes (public_key, name, last_seen) VALUES (`+db.parameter(1)+`, `+db.parameter(2)+`, `+db.parameter(3)+`)`, + "pk"+strconv.Itoa(i), "n"+strconv.Itoa(i), seen.Format(time.RFC3339)) + } + + got, err := db.CountActiveNodes() + if err != nil { + t.Fatal(err) + } + if got != 2 { + t.Errorf("CountActiveNodes = %d, want 2 (the node seen 8 days ago is not active)", got) + } + stats, err := db.GetStats() + if err != nil { + t.Fatal(err) + } + if got != stats.TotalNodes { + t.Errorf("CountActiveNodes = %d, GetStats().TotalNodes = %d", got, stats.TotalNodes) + } } func TestGetStats(t *testing.T) { @@ -237,8 +379,8 @@ func TestGetDBSizeStats(t *testing.T) { stats := db.GetDBSizeStats() // In-memory DB has dbSizeMB=0 and walSizeMB=0 - if stats["dbSizeMB"] != float64(0) { - t.Errorf("expected dbSizeMB=0 for in-memory DB, got %v", stats["dbSizeMB"]) + if stats["dbSizeMB"].(float64) < 0 || (testBackendValue() == "postgres" && stats["dbSizeMB"].(float64) == 0) { + t.Errorf("expected positive PostgreSQL database size, got %v", stats["dbSizeMB"]) } rows, ok := stats["rows"].(map[string]int) @@ -258,17 +400,15 @@ func TestGetDBSizeStats(t *testing.T) { t.Errorf("expected 2 observers rows, got %d", rows["observers"]) } - // Verify new PRAGMA-based fields - if _, ok := stats["freelistMB"]; !ok { - t.Error("expected freelistMB in DB size stats") - } - walPages, ok := stats["walPages"].(map[string]interface{}) - if !ok { - t.Fatal("expected walPages object in DB size stats") + if stats["engine"] != testNativeSQL("sqlite", "postgresql") { + t.Fatal("database engine not identified") } - for _, key := range []string{"total", "checkpointed", "busy"} { - if _, ok := walPages[key]; !ok { - t.Errorf("expected %s in walPages", key) + for _, obsolete := range []string{"freelistMB", "walPages", "walSizeMB"} { + if testBackendValue() == "sqlite" { + continue + } + if _, ok := stats[obsolete]; ok { + t.Fatalf("invented SQLite metric %s", obsolete) } } } @@ -412,7 +552,7 @@ func TestGetObservers_ExcludesInactive(t *testing.T) { defer db.Close() seedTestData(t, db) // Mark obs2 inactive — soft delete simulating a stale-observer prune. - if _, err := db.conn.Exec(`UPDATE observers SET inactive = 1 WHERE id = ?`, "obs2"); err != nil { + if _, err := db.conn.Exec(`UPDATE observers SET inactive = 1 WHERE id = $1`, "obs2"); err != nil { t.Fatalf("update inactive: %v", err) } observers, err := db.GetObservers() @@ -454,7 +594,7 @@ func TestGetObserverLastPacketAt(t *testing.T) { // Set last_packet_at for obs1 ts := "2026-04-24T12:00:00Z" - db.conn.Exec(`UPDATE observers SET last_packet_at = ? WHERE id = ?`, ts, "obs1") + db.conn.Exec(`UPDATE observers SET last_packet_at = $1 WHERE id = $2`, ts, "obs1") // Verify via GetObservers observers, err := db.GetObservers() @@ -766,7 +906,7 @@ func TestGetObserverIdsForRegion(t *testing.T) { t.Run("case and trim normalization", func(t *testing.T) { db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obs3', 'Observer Three', ' sjc ', ?, '2026-01-01T00:00:00Z', 1)`, time.Now().UTC().Format(time.RFC3339)) + VALUES ('obs3', 'Observer Three', ' sjc ', $1, '2026-01-01T00:00:00Z', 1)`, time.Now().UTC().Format(time.RFC3339)) ids, err := db.GetObserverIdsForRegion(" sjc ") if err != nil { t.Fatal(err) @@ -852,15 +992,15 @@ func TestGetChannelMessagesRegionFiltering(t *testing.T) { db.conn.Exec(`INSERT INTO observers (id, name, iata) VALUES ('obs1', 'Observer One', 'SJC')`) db.conn.Exec(`INSERT INTO observers (id, name, iata) VALUES ('obs2', 'Observer Two', ' sfo ')`) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash) - VALUES ('AA', 'chanregion0001', ?, 1, 5, + VALUES ('AA', 'chanregion0001', $1, 1, 5, '{"type":"CHAN","channel":"#region","text":"SjcUser: One","sender":"SjcUser"}', '#region')`, ts1) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash) - VALUES ('BB', 'chanregion0002', ?, 1, 5, + VALUES ('BB', 'chanregion0002', $1, 1, 5, '{"type":"CHAN","channel":"#region","text":"SfoUser: Two","sender":"SfoUser"}', '#region')`, ts2) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 10.0, -90, '[]', ?)`, epoch1) + VALUES (1, 1, 10.0, -90, '[]', $1)`, epoch1) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (2, 2, 9.0, -91, '[]', ?)`, epoch2) + VALUES (2, 2, 9.0, -91, '[]', $1)`, epoch2) msgsSJC, totalSJC, err := db.GetChannelMessages("#region", 100, 0, " sjc ") if err != nil { @@ -1043,7 +1183,7 @@ func TestGetNodesFiltering(t *testing.T) { seedTestData(t, db) t.Run("role filter", func(t *testing.T) { - nodes, total, _, err := db.GetNodes(50, 0, "repeater", "", "", "", "", "") + nodes, total, _, err := db.GetNodes(NodeQuery{Limit: 50, Role: "repeater"}) if err != nil { t.Fatal(err) } @@ -1056,7 +1196,7 @@ func TestGetNodesFiltering(t *testing.T) { }) t.Run("search filter", func(t *testing.T) { - nodes, _, _, err := db.GetNodes(50, 0, "", "Companion", "", "", "", "") + nodes, _, _, err := db.GetNodes(NodeQuery{Limit: 50, Search: "Companion"}) if err != nil { t.Fatal(err) } @@ -1066,7 +1206,7 @@ func TestGetNodesFiltering(t *testing.T) { }) t.Run("sort by name", func(t *testing.T) { - nodes, _, _, err := db.GetNodes(50, 0, "", "", "", "", "name", "") + nodes, _, _, err := db.GetNodes(NodeQuery{Limit: 50, SortBy: "name"}) if err != nil { t.Fatal(err) } @@ -1076,7 +1216,7 @@ func TestGetNodesFiltering(t *testing.T) { }) t.Run("sort by packetCount", func(t *testing.T) { - nodes, _, _, err := db.GetNodes(50, 0, "", "", "", "", "packetCount", "") + nodes, _, _, err := db.GetNodes(NodeQuery{Limit: 50, SortBy: "packetCount"}) if err != nil { t.Fatal(err) } @@ -1086,7 +1226,7 @@ func TestGetNodesFiltering(t *testing.T) { }) t.Run("sort by lastSeen", func(t *testing.T) { - nodes, _, _, err := db.GetNodes(50, 0, "", "", "", "", "lastSeen", "") + nodes, _, _, err := db.GetNodes(NodeQuery{Limit: 50, SortBy: "lastSeen"}) if err != nil { t.Fatal(err) } @@ -1098,7 +1238,7 @@ func TestGetNodesFiltering(t *testing.T) { t.Run("lastHeard filter 30d", func(t *testing.T) { // The filter works by computing since = now - 30d; seed data last_seen may or may not match. // Just verify the filter runs without error. - _, _, _, err := db.GetNodes(50, 0, "", "", "", "30d", "", "") + _, _, _, err := db.GetNodes(NodeQuery{Limit: 50, LastHeard: "30d"}) if err != nil { t.Fatal(err) } @@ -1106,7 +1246,7 @@ func TestGetNodesFiltering(t *testing.T) { t.Run("lastHeard filter various", func(t *testing.T) { for _, lh := range []string{"1h", "6h", "24h", "7d", "30d", "invalid"} { - _, _, _, err := db.GetNodes(50, 0, "", "", "", lh, "", "") + _, _, _, err := db.GetNodes(NodeQuery{Limit: 50, LastHeard: lh}) if err != nil { t.Fatalf("lastHeard=%s failed: %v", lh, err) } @@ -1114,7 +1254,7 @@ func TestGetNodesFiltering(t *testing.T) { }) t.Run("default limit", func(t *testing.T) { - nodes, _, _, err := db.GetNodes(0, 0, "", "", "", "", "", "") + nodes, _, _, err := db.GetNodes(NodeQuery{Limit: 0}) if err != nil { t.Fatal(err) } @@ -1124,7 +1264,7 @@ func TestGetNodesFiltering(t *testing.T) { }) t.Run("before filter", func(t *testing.T) { - _, total, _, err := db.GetNodes(50, 0, "", "", "2026-01-02T00:00:00Z", "", "", "") + _, total, _, err := db.GetNodes(NodeQuery{Limit: 50, Before: "2026-01-02T00:00:00Z"}) if err != nil { t.Fatal(err) } @@ -1134,7 +1274,7 @@ func TestGetNodesFiltering(t *testing.T) { }) t.Run("offset", func(t *testing.T) { - nodes, total, _, err := db.GetNodes(1, 1, "", "", "", "", "", "") + nodes, total, _, err := db.GetNodes(NodeQuery{Limit: 1, Offset: 1}) if err != nil { t.Fatal(err) } @@ -1147,7 +1287,7 @@ func TestGetNodesFiltering(t *testing.T) { }) t.Run("region filter SJC", func(t *testing.T) { - nodes, total, _, err := db.GetNodes(50, 0, "", "", "", "", "", "SJC") + nodes, total, _, err := db.GetNodes(NodeQuery{Limit: 50, Region: "SJC"}) if err != nil { t.Fatal(err) } @@ -1163,7 +1303,7 @@ func TestGetNodesFiltering(t *testing.T) { }) t.Run("region filter SFO", func(t *testing.T) { - _, total, _, err := db.GetNodes(50, 0, "", "", "", "", "", "SFO") + _, total, _, err := db.GetNodes(NodeQuery{Limit: 50, Region: "SFO"}) if err != nil { t.Fatal(err) } @@ -1173,7 +1313,7 @@ func TestGetNodesFiltering(t *testing.T) { }) t.Run("region filter multi", func(t *testing.T) { - _, total, _, err := db.GetNodes(50, 0, "", "", "", "", "", "SJC,SFO") + _, total, _, err := db.GetNodes(NodeQuery{Limit: 50, Region: "SJC,SFO"}) if err != nil { t.Fatal(err) } @@ -1183,7 +1323,7 @@ func TestGetNodesFiltering(t *testing.T) { }) t.Run("region filter unknown", func(t *testing.T) { - _, total, _, err := db.GetNodes(50, 0, "", "", "", "", "", "AMS") + _, total, _, err := db.GetNodes(NodeQuery{Limit: 50, Region: "AMS"}) if err != nil { t.Fatal(err) } @@ -1197,13 +1337,13 @@ func TestGetNodesFiltering(t *testing.T) { // where observations use observer_id TEXT instead of observer_idx INTEGER. func setupTestDBV2(t *testing.T) *DB { t.Helper() - conn, err := sql.Open("sqlite", ":memory:") + conn, err := openPostgresTestDB(t) if err != nil { t.Fatal(err) } conn.SetMaxOpenConns(1) - schema := ` + schema := testNativeSQL(` CREATE TABLE nodes ( public_key TEXT PRIMARY KEY, name TEXT, @@ -1225,7 +1365,8 @@ func setupTestDBV2(t *testing.T) *DB { last_seen TEXT, first_seen TEXT, packet_count INTEGER DEFAULT 0, - last_packet_at TEXT DEFAULT NULL + last_packet_at TEXT DEFAULT NULL, + inactive INTEGER DEFAULT 0 ); CREATE TABLE transmissions ( @@ -1268,12 +1409,76 @@ func setupTestDBV2(t *testing.T) *DB { WHERE id = NEW.id; END; CREATE INDEX IF NOT EXISTS idx_transmissions_from_pubkey ON transmissions(from_pubkey); - ` + `, ` + CREATE TABLE nodes ( + public_key TEXT PRIMARY KEY, + name TEXT, + role TEXT, + lat DOUBLE PRECISION, + lon DOUBLE PRECISION, + last_seen TEXT, + first_seen TEXT, + advert_count INTEGER DEFAULT 0, + battery_mv INTEGER, + temperature_c DOUBLE PRECISION, + foreign_advert INTEGER DEFAULT 0 + ); + + CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY UNIQUE, + id TEXT PRIMARY KEY, + name TEXT, + iata TEXT, + last_seen TEXT, + first_seen TEXT, + packet_count INTEGER DEFAULT 0, + last_packet_at TEXT DEFAULT NULL, + inactive INTEGER DEFAULT 0 + ); + + CREATE TABLE transmissions ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + raw_hex TEXT NOT NULL, + hash TEXT NOT NULL UNIQUE, + first_seen TEXT NOT NULL, + route_type INTEGER, + payload_type INTEGER, + payload_version INTEGER, + decoded_json TEXT, + channel_hash TEXT DEFAULT NULL, + from_pubkey TEXT DEFAULT NULL, + created_at TEXT DEFAULT (to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"')) + ); + + CREATE TABLE observations ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + transmission_id INTEGER NOT NULL REFERENCES transmissions(id), + observer_id TEXT, + observer_name TEXT, + direction TEXT, + snr DOUBLE PRECISION, + rssi DOUBLE PRECISION, + score INTEGER, + path_json TEXT, + timestamp INTEGER NOT NULL, + raw_hex TEXT + ); + + CREATE FUNCTION test_from_pubkey_advert() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN + IF NEW.from_pubkey IS NULL AND NEW.payload_type = 4 AND NEW.decoded_json IS JSON OBJECT THEN + NEW.from_pubkey := NULLIF(NEW.decoded_json::json ->> 'pubKey', ''); + END IF; + RETURN NEW; + END $$; + CREATE TRIGGER test_from_pubkey_advert BEFORE INSERT ON transmissions + FOR EACH ROW EXECUTE FUNCTION test_from_pubkey_advert(); + CREATE INDEX IF NOT EXISTS idx_transmissions_from_pubkey ON transmissions(from_pubkey); + `) if _, err := conn.Exec(schema); err != nil { t.Fatal(err) } - return &DB{conn: conn, isV3: false} + return &DB{backend: testBackendValue(), conn: conn, isV3: false} } func TestGetNodesRegionFilterV2(t *testing.T) { @@ -1286,22 +1491,22 @@ func TestGetNodesRegionFilterV2(t *testing.T) { // Seed observer with IATA code db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obs-v2-1', 'V2 Observer', 'LAX', ?, '2026-01-01T00:00:00Z', 10)`, recent) + VALUES ('obs-v2-1', 'V2 Observer', 'LAX', $1, '2026-01-01T00:00:00Z', 10)`, recent) // Seed a node db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES ('v2pubkey11223344', 'V2Node', 'repeater', 34.0, -118.0, ?, '2026-01-01T00:00:00Z', 5)`, recent) + VALUES ('v2pubkey11223344', 'V2Node', 'repeater', 34.0, -118.0, $1, '2026-01-01T00:00:00Z', 5)`, recent) // Seed an ADVERT transmission for the node db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('AABB', 'v2hash0001', ?, 1, 4, '{"pubKey":"v2pubkey11223344","name":"V2Node","type":"ADVERT"}')`, recent) + VALUES ('AABB', 'v2hash0001', $1, 1, 4, '{"pubKey":"v2pubkey11223344","name":"V2Node","type":"ADVERT"}')`, recent) // Seed v2-style observation: observer_id references observers.id directly db.conn.Exec(`INSERT INTO observations (transmission_id, observer_id, observer_name, snr, rssi, path_json, timestamp) - VALUES (1, 'obs-v2-1', 'V2 Observer', 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (1, 'obs-v2-1', 'V2 Observer', 10.0, -90, '[]', $1)`, recentEpoch) t.Run("v2 region filter match", func(t *testing.T) { - nodes, total, _, err := db.GetNodes(50, 0, "", "", "", "", "", "LAX") + nodes, total, _, err := db.GetNodes(NodeQuery{Limit: 50, Region: "LAX"}) if err != nil { t.Fatal(err) } @@ -1317,7 +1522,7 @@ func TestGetNodesRegionFilterV2(t *testing.T) { }) t.Run("v2 region filter no match", func(t *testing.T) { - _, total, _, err := db.GetNodes(50, 0, "", "", "", "", "", "JFK") + _, total, _, err := db.GetNodes(NodeQuery{Limit: 50, Region: "JFK"}) if err != nil { t.Fatal(err) } @@ -1409,9 +1614,9 @@ func TestGetNetworkStatusDateFormats(t *testing.T) { // Insert nodes with different date formats db.conn.Exec(`INSERT INTO nodes (public_key, name, role, last_seen) - VALUES ('node1111', 'NodeRFC', 'repeater', ?)`, time.Now().Format(time.RFC3339)) + VALUES ('node1111', 'NodeRFC', 'repeater', $1)`, time.Now().Format(time.RFC3339)) db.conn.Exec(`INSERT INTO nodes (public_key, name, role, last_seen) - VALUES ('node2222', 'NodeSQL', 'companion', ?)`, time.Now().Format("2006-01-02 15:04:05")) + VALUES ('node2222', 'NodeSQL', 'companion', $1)`, time.Now().Format("2006-01-02 15:04:05")) db.conn.Exec(`INSERT INTO nodes (public_key, name, role, last_seen) VALUES ('node3333', 'NodeNull', 'room', NULL)`) db.conn.Exec(`INSERT INTO nodes (public_key, name, role, last_seen) @@ -1449,23 +1654,23 @@ func TestGetNetworkStatusDateFormats(t *testing.T) { func TestOpenDBValid(t *testing.T) { // Create a real SQLite database file - dir := t.TempDir() - dbPath := filepath.Join(dir, "test.db") + dbPath := postgresTestDSN(t) // Create DB with a table using a writable connection first - conn, err := sql.Open("sqlite", dbPath) + conn, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } - _, err = conn.Exec(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY, hash TEXT)`) + _, err = conn.Exec(testNativeSQL(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY AUTOINCREMENT, hash TEXT)`, `CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, hash TEXT)`)) if err != nil { conn.Close() t.Fatal(err) } + ensurePreparable(t, conn) conn.Close() // Now test OpenDB (read-only) - database, err := OpenDB(dbPath) + database, err := openFixtureReader(t, dbPath) if err != nil { t.Fatalf("OpenDB failed: %v", err) } @@ -1490,16 +1695,15 @@ func TestOpenDBInvalidPath(t *testing.T) { // The existing ScopeStats tests set these flags manually — this test ensures // the flag-setting code itself is covered. func TestDetectSchemaScopeName(t *testing.T) { - dir := t.TempDir() - dbPath := filepath.Join(dir, "detect.db") + dbPath := postgresTestDSN(t) // Create file-based DB with the scope_name and default_scope columns. - conn, err := sql.Open("sqlite", dbPath) + conn, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } conn.SetMaxOpenConns(1) - if _, err := conn.Exec(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY, hash TEXT, scope_name TEXT)`); err != nil { + if _, err := conn.Exec(testNativeSQL(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY AUTOINCREMENT, hash TEXT, scope_name TEXT)`, `CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, hash TEXT, scope_name TEXT)`)); err != nil { conn.Close() t.Fatalf("create transmissions: %v", err) } @@ -1507,13 +1711,14 @@ func TestDetectSchemaScopeName(t *testing.T) { conn.Close() t.Fatalf("create nodes: %v", err) } - if _, err := conn.Exec(`CREATE TABLE observations (id INTEGER PRIMARY KEY)`); err != nil { + if _, err := conn.Exec(testNativeSQL(`CREATE TABLE observations (id INTEGER PRIMARY KEY AUTOINCREMENT)`, `CREATE TABLE observations (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY)`)); err != nil { conn.Close() t.Fatalf("create observations: %v", err) } + ensurePreparable(t, conn) conn.Close() - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatalf("OpenDB: %v", err) } @@ -1527,18 +1732,19 @@ func TestDetectSchemaScopeName(t *testing.T) { } // Verify the flags stay false when the columns are absent. - dbPath2 := filepath.Join(dir, "detect2.db") - conn2, err := sql.Open("sqlite", dbPath2) + dbPath2 := postgresTestDSN(t) + conn2, err := openFixtureSQL(dbPath2) if err != nil { t.Fatal(err) } conn2.SetMaxOpenConns(1) - conn2.Exec(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY, hash TEXT)`) + conn2.Exec(testNativeSQL(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY AUTOINCREMENT, hash TEXT)`, `CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, hash TEXT)`)) conn2.Exec(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY)`) - conn2.Exec(`CREATE TABLE observations (id INTEGER PRIMARY KEY)`) + conn2.Exec(testNativeSQL(`CREATE TABLE observations (id INTEGER PRIMARY KEY AUTOINCREMENT)`, `CREATE TABLE observations (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY)`)) + ensurePreparable(t, conn2) conn2.Close() - db2, err := OpenDB(dbPath2) + db2, err := openFixtureReader(t, dbPath2) if err != nil { t.Fatalf("OpenDB2: %v", err) } @@ -1784,7 +1990,7 @@ func TestNodeTelemetryFields(t *testing.T) { } // Test via GetNodes - nodes, _, _, err := db.GetNodes(50, 0, "sensor", "", "", "", "", "") + nodes, _, _, err := db.GetNodes(NodeQuery{Limit: 50, Role: "sensor"}) if err != nil { t.Fatal(err) } @@ -1808,6 +2014,9 @@ func TestNodeTelemetryFields(t *testing.T) { } func TestMain(m *testing.M) { + // Server fixtures need stable build metadata, not a Git subprocess per + // instance. TestResolveCommit still exercises the actual fallback. + Commit = "unit-test" os.Exit(m.Run()) } @@ -1820,13 +2029,13 @@ func TestGetObserverMetrics(t *testing.T) { t2 := now.Add(-1 * time.Hour).Format(time.RFC3339) t3 := now.Format(time.RFC3339) - db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors, battery_mv) VALUES (?, ?, ?, ?, ?, ?, ?)", + db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors, battery_mv) VALUES ($1, $2, $3, $4, $5, $6, $7)", "obs1", t1, -112.5, 100, 500, 3, 3720) - db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors) VALUES (?, ?, ?, ?, ?, ?)", + db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors) VALUES ($1, $2, $3, $4, $5, $6)", "obs1", t2, -110.0, 200, 800, 5) - db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors) VALUES (?, ?, ?, ?, ?, ?)", + db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors) VALUES ($1, $2, $3, $4, $5, $6)", "obs1", t3, -108.0, 300, 1100, 8) - db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor) VALUES (?, ?, ?)", + db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor) VALUES ($1, $2, $3)", "obs2", t1, -115.0) // Query all for obs1 @@ -1877,11 +2086,11 @@ func TestGetMetricsSummary(t *testing.T) { t1 := now.Add(-2 * time.Hour).Format(time.RFC3339) t2 := now.Add(-1 * time.Hour).Format(time.RFC3339) - db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, battery_mv) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, battery_mv) VALUES ($1, $2, $3, $4)", "obs1", t1, -112.0, 3720) - db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor) VALUES (?, ?, ?)", + db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor) VALUES ($1, $2, $3)", "obs1", t2, -108.0) - db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor) VALUES (?, ?, ?)", + db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor) VALUES ($1, $2, $3)", "obs2", t1, -115.0) since := now.Add(-24 * time.Hour).Format(time.RFC3339) @@ -1927,7 +2136,7 @@ func TestObserverMetricsAPIEndpoints(t *testing.T) { now := time.Now().UTC() t1 := now.Add(-1 * time.Hour).Format(time.RFC3339) - db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor) VALUES (?, ?, ?)", + db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor) VALUES ($1, $2, $3)", "obs1", t1, -112.0) // Query directly to verify @@ -2051,13 +2260,13 @@ func TestGetObserverMetricsResolution(t *testing.T) { db := setupTestDB(t) seedTestData(t, db) - db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs) VALUES ($1, $2, $3, $4)", "obs1", "2026-04-05T00:00:00Z", -112.0, 100) - db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs) VALUES ($1, $2, $3, $4)", "obs1", "2026-04-05T00:05:00Z", -110.0, 200) - db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs) VALUES ($1, $2, $3, $4)", "obs1", "2026-04-05T01:00:00Z", -108.0, 500) - db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs) VALUES ($1, $2, $3, $4)", "obs1", "2026-04-05T01:05:00Z", -106.0, 600) // 5m resolution: all 4 rows @@ -2094,9 +2303,9 @@ func TestHourlyResolutionDeltasNotNull(t *testing.T) { // Two hourly buckets, each with one sample. With old MAX+hardcoded gap threshold, // the 3600s gap would exceed sampleInterval*2 (600s) and deltas would be null. - db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors, packets_sent, packets_recv) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors, packets_sent, packets_recv) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)", "obs_hr", "2026-04-05T10:00:00Z", -110.0, 100, 200, 5, 50, 100) - db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors, packets_sent, packets_recv) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors, packets_sent, packets_recv) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)", "obs_hr", "2026-04-05T11:00:00Z", -108.0, 200, 400, 10, 80, 200) m, _, err := db.GetObserverMetrics("obs_hr", "2026-04-04T00:00:00Z", "", "1h", 300) @@ -2119,15 +2328,15 @@ func TestLastValuePreservesReboot(t *testing.T) { // Hour bucket with two samples: pre-reboot (high) and post-reboot (low). // With MAX(), the pre-reboot value wins and the reboot is hidden. // With LAST (latest timestamp), the post-reboot value wins. - db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors, packets_sent, packets_recv) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors, packets_sent, packets_recv) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)", "obs_rb", "2026-04-05T10:00:00Z", -110.0, 1000, 2000, 500, 400, 800) // pre-reboot baseline - db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors, packets_sent, packets_recv) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors, packets_sent, packets_recv) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)", "obs_rb", "2026-04-05T10:20:00Z", -110.0, 5000, 6000, 900, 700, 1200) // pre-reboot peak - db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors, packets_sent, packets_recv) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors, packets_sent, packets_recv) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)", "obs_rb", "2026-04-05T10:40:00Z", -110.0, 10, 20, 1, 5, 10) // post-reboot (counter reset) // Next hour bucket - db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors, packets_sent, packets_recv) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors, packets_sent, packets_recv) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)", "obs_rb", "2026-04-05T11:00:00Z", -108.0, 100, 120, 5, 20, 50) m, reboots, err := db.GetObserverMetrics("obs_rb", "2026-04-04T00:00:00Z", "", "1h", 300) @@ -2191,14 +2400,14 @@ func TestPerObservationRawHexEnrich(t *testing.T) { // Insert transmission with raw_hex txHex := "deadbeef" - db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen) VALUES (?, 'hash1', '2026-04-21T10:00:00Z')`, txHex) + db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen) VALUES ($1, 'hash1', '2026-04-21T10:00:00Z')`, txHex) // Insert two observations: A has its own raw_hex, B has NULL (historical) obsAHex := "c0ffee01" db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp, raw_hex) - VALUES (1, ?, -5.0, -90.0, '[]', 1745236800, ?)`, rowA, obsAHex) + VALUES (1, $1, -5.0, -90.0, '[]', 1745236800, $2)`, rowA, obsAHex) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, ?, -3.0, -85.0, '["aabb"]', 1745236801)`, rowB) + VALUES (1, $1, -3.0, -85.0, '["aabb"]', 1745236801)`, rowB) store := NewPacketStore(db, nil) if err := store.Load(); err != nil { @@ -2215,7 +2424,7 @@ func TestPerObservationRawHexEnrich(t *testing.T) { // Check enriched observations for _, obs := range tx.Observations { - m := store.enrichObs(obs) + m := store.enrichObsWithTx(obs, store.byTxID[obs.TransmissionID]) rh, _ := m["raw_hex"].(string) if obs.RawHex != "" { // Observer A: should get per-observation raw_hex @@ -2232,35 +2441,41 @@ func TestPerObservationRawHexEnrich(t *testing.T) { } func TestGetScopeStats(t *testing.T) { - conn, err := sql.Open("sqlite", ":memory:") + conn, err := openPostgresTestDB(t) if err != nil { t.Fatalf("sql.Open: %v", err) } conn.SetMaxOpenConns(1) - db := &DB{conn: conn} + db := &DB{backend: testBackendValue(), conn: conn} defer db.conn.Close() // Create minimal schema - db.conn.Exec(`CREATE TABLE IF NOT EXISTS transmissions ( + db.conn.Exec(testNativeSQL(`CREATE TABLE IF NOT EXISTS transmissions ( id INTEGER PRIMARY KEY AUTOINCREMENT, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT, - scope_name TEXT DEFAULT NULL - )`) + scope_name TEXT DEFAULT NULL, from_pubkey TEXT DEFAULT NULL + )`, `CREATE TABLE IF NOT EXISTS transmissions ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, + payload_type INTEGER, payload_version INTEGER, decoded_json TEXT, + scope_name TEXT DEFAULT NULL, from_pubkey TEXT DEFAULT NULL + )`)) + db.conn.Exec(`CREATE TABLE IF NOT EXISTS nodes (public_key TEXT PRIMARY KEY, role TEXT)`) // Manually set hasScopeName since we bypassed the detector db.hasScopeName = true now := time.Now().UTC().Format(time.RFC3339) // Transport scoped, known region - db.conn.Exec(`INSERT INTO transmissions (hash, first_seen, route_type, scope_name) VALUES ('a', ?, 0, '#belgium')`, now) + db.conn.Exec(`INSERT INTO transmissions (hash, first_seen, route_type, scope_name) VALUES ('a', $1, 0, '#belgium')`, now) // Transport scoped, unknown - db.conn.Exec(`INSERT INTO transmissions (hash, first_seen, route_type, scope_name) VALUES ('b', ?, 0, '')`, now) + db.conn.Exec(`INSERT INTO transmissions (hash, first_seen, route_type, scope_name) VALUES ('b', $1, 0, '')`, now) // Transport unscoped (NULL) - db.conn.Exec(`INSERT INTO transmissions (hash, first_seen, route_type, scope_name) VALUES ('c', ?, 0, NULL)`, now) + db.conn.Exec(`INSERT INTO transmissions (hash, first_seen, route_type, scope_name) VALUES ('c', $1, 0, NULL)`, now) // Non-transport FLOOD (route_type=1) — inherently unscoped per MeshCore protocol (#1838) - db.conn.Exec(`INSERT INTO transmissions (hash, first_seen, route_type, scope_name) VALUES ('d', ?, 1, NULL)`, now) + db.conn.Exec(`INSERT INTO transmissions (hash, first_seen, route_type, scope_name) VALUES ('d', $1, 1, NULL)`, now) // Non-transport DIRECT (route_type=2) — inherently unscoped per MeshCore protocol (#1838) - db.conn.Exec(`INSERT INTO transmissions (hash, first_seen, route_type, scope_name) VALUES ('e', ?, 2, NULL)`, now) + db.conn.Exec(`INSERT INTO transmissions (hash, first_seen, route_type, scope_name) VALUES ('e', $1, 2, NULL)`, now) stats, err := db.GetScopeStats("24h") if err != nil { @@ -2286,6 +2501,119 @@ func TestGetScopeStats(t *testing.T) { } } +// TestGetScopeStatsAdvertsByRole covers the #1979 breakdown: flood adverts in +// the window, grouped by the sender's nodes.role, split by the three +// scope_name states (NULL is unscoped, an empty string is unknown scope, a +// name is a named scope). +func TestGetScopeStatsAdvertsByRole(t *testing.T) { + conn, err := openPostgresTestDB(t) + if err != nil { + t.Fatalf("sql.Open: %v", err) + } + conn.SetMaxOpenConns(1) + db := &DB{backend: testBackendValue(), conn: conn, hasScopeName: true} + defer db.conn.Close() + + for _, stmt := range []string{ + testNativeSQL(`CREATE TABLE transmissions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, + scope_name TEXT DEFAULT NULL, from_pubkey TEXT DEFAULT NULL + )`, `CREATE TABLE transmissions ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, + scope_name TEXT DEFAULT NULL, from_pubkey TEXT DEFAULT NULL + )`), + `CREATE TABLE nodes (public_key TEXT PRIMARY KEY, role TEXT)`, + `INSERT INTO nodes (public_key, role) VALUES ('rpt', 'repeater'), ('cmp', 'companion'), ('sns', 'sensor'), ('norole', '')`, + } { + if _, err := db.conn.Exec(stmt); err != nil { + t.Fatalf("schema: %v", err) + } + } + + now := time.Now().UTC().Format(time.RFC3339) + old := time.Now().UTC().Add(-48 * time.Hour).Format(time.RFC3339) + rows := []struct { + hash string + seen string + route int + ptype int + scope interface{} + fromKey interface{} + }{ + // Totals per role differ from alphabetical order so the test pins + // ORDER BY total: repeater 4, unknown 3, companion 2, sensor 2. + // companion and sensor tie, which pins the role-name tie-break. + // Repeater: every scope state on flood routes, two named. + {"r1", now, 1, 4, nil, "rpt"}, + {"r2", now, 0, 4, "", "rpt"}, + {"r3", now, 0, 4, "#belgium", "rpt"}, + {"r8", now, 1, 4, "#belgium", "rpt"}, + // Excluded: zero-hop adverts are DIRECT/TRANSPORT_DIRECT, not flood. + {"r4", now, 2, 4, nil, "rpt"}, + {"r5", now, 3, 4, "#belgium", "rpt"}, + // Excluded: not an advert. + {"r6", now, 0, 5, "#belgium", "rpt"}, + // Excluded: outside the window. + {"r7", old, 1, 4, nil, "rpt"}, + // Companion: one unscoped, one named. + {"c1", now, 1, 4, nil, "cmp"}, + {"c3", now, 0, 4, "#belgium", "cmp"}, + // Sensor: one unscoped, one unknown scope. + {"s1", now, 1, 4, nil, "sns"}, + {"s2", now, 0, 4, "", "sns"}, + // No node row, and a node row without a role: both "unknown". + {"u1", now, 1, 4, nil, "stranger"}, + {"u2", now, 0, 4, "", "norole"}, + {"u3", now, 0, 4, "#belgium", nil}, + } + for _, r := range rows { + if _, err := db.conn.Exec( + `INSERT INTO transmissions (hash, first_seen, route_type, payload_type, scope_name, from_pubkey) VALUES ($1, $2, $3, $4, $5, $6)`, + r.hash, r.seen, r.route, r.ptype, r.scope, r.fromKey, + ); err != nil { + t.Fatalf("seed %s: %v", r.hash, err) + } + } + + stats, err := db.GetScopeStats("24h") + if err != nil { + t.Fatalf("GetScopeStats: %v", err) + } + // Ordered by total adverts descending, then role name. + want := []ScopeAdvertRoleCount{ + {Role: "repeater", Unscoped: 1, UnknownScope: 1, Named: 2}, + {Role: "unknown", Unscoped: 1, UnknownScope: 1, Named: 1}, + {Role: "companion", Unscoped: 1, UnknownScope: 0, Named: 1}, + {Role: "sensor", Unscoped: 1, UnknownScope: 1, Named: 0}, + } + if !reflect.DeepEqual(stats.AdvertsByRole, want) { + t.Errorf("AdvertsByRole = %+v, want %+v", stats.AdvertsByRole, want) + } +} + +func TestGetScopeStatsAdvertsByRoleEmpty(t *testing.T) { + conn, err := openPostgresTestDB(t) + if err != nil { + t.Fatalf("sql.Open: %v", err) + } + conn.SetMaxOpenConns(1) + db := &DB{backend: testBackendValue(), conn: conn, hasScopeName: true} + defer db.conn.Close() + db.conn.Exec(testNativeSQL(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY AUTOINCREMENT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, scope_name TEXT, from_pubkey TEXT)`, `CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, first_seen TEXT, route_type INTEGER, payload_type INTEGER, scope_name TEXT, from_pubkey TEXT)`)) + db.conn.Exec(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY, role TEXT)`) + + stats, err := db.GetScopeStats("1h") + if err != nil { + t.Fatalf("GetScopeStats: %v", err) + } + // Array fields are [] when empty, never null (docs/api-spec.md conventions). + if stats.AdvertsByRole == nil || len(stats.AdvertsByRole) != 0 { + t.Errorf("AdvertsByRole = %#v, want empty non-nil slice", stats.AdvertsByRole) + } +} + // TestLoadIndexesRelayHopsFromResolvedPath verifies that after Load(), relay // nodes that appear only in resolved_path (not in decoded_json) are indexed // in byNode. Regression for #692: indexByNode was called before observations @@ -2304,12 +2632,12 @@ func TestLoadIndexesRelayHopsFromResolvedPath(t *testing.T) { const senderPubkey = "sender00112233445566" db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('FF01', 'relaytest0001hash', ?, 1, 4, ?)`, + VALUES ('FF01', 'relaytest0001hash', $1, 1, 4, $2)`, now, `{"pubKey":"`+senderPubkey+`","name":"Sender","type":"ADVERT"}`) // Observer hears the packet via the relay node. db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp, resolved_path) - VALUES (1, 1, 10.0, -90, '["rr"]', ?, ?)`, + VALUES (1, 1, 10.0, -90, '["rr"]', $1, $2)`, epoch, `["`+relayPubkey+`"]`) store := NewPacketStore(db, nil) @@ -2340,7 +2668,7 @@ type failingQuerier struct { } func (f *failingQuerier) QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) { - if f.failOn != "" && strings.Contains(query, f.failOn) { + if f.failOn != "" && (strings.Contains(query, f.failOn) || (len(args) > 0 && args[0] == f.failOn)) { return nil, f.err } return f.real.QueryContext(ctx, query, args...) @@ -2351,28 +2679,28 @@ func (f *failingQuerier) QueryContext(ctx context.Context, query string, args .. // instead of being silently swallowed and leaving the server in v2 mode against // a v3 database for the whole process lifetime. func TestDetectSchemaFailsLoudOnProbeError(t *testing.T) { - dbPath := filepath.Join(t.TempDir(), "v3.db") - conn, err := sql.Open("sqlite", dbPath) + dbPath := postgresTestDSN(t) + conn, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } conn.SetMaxOpenConns(1) // v3-shaped: observations carries observer_idx. - if _, err := conn.Exec(`CREATE TABLE observations (id INTEGER PRIMARY KEY, observer_idx INTEGER)`); err != nil { + if _, err := conn.Exec(testNativeSQL(`CREATE TABLE observations (id INTEGER PRIMARY KEY AUTOINCREMENT, observer_idx INTEGER)`, `CREATE TABLE observations (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, observer_idx INTEGER)`)); err != nil { conn.Close() t.Fatal(err) } - conn.Exec(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY)`) + conn.Exec(testNativeSQL(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY AUTOINCREMENT)`, `CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY)`)) conn.Exec(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY)`) conn.Close() - real, err := sql.Open("sqlite", "file:"+dbPath+"?mode=ro") + real, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } defer real.Close() - q := &failingQuerier{real: real, failOn: "table_info(observations)", err: errors.New("injected probe failure")} + q := &failingQuerier{real: real, failOn: "observations", err: errors.New("injected probe failure")} db := &DB{} if err := db.detectSchema(context.Background(), q); err == nil { t.Fatal("detectSchema must return an error when a schema probe fails (#1901)") @@ -2385,19 +2713,19 @@ func TestDetectSchemaFailsLoudOnProbeError(t *testing.T) { // TestDetectSchemaV3AndV2 verifies detection sets isV3 correctly for both schema // shapes via the real OpenDB path, and that both open cleanly. func TestDetectSchemaV3AndV2(t *testing.T) { - dir := t.TempDir() - v3 := filepath.Join(dir, "v3.db") - c, err := sql.Open("sqlite", v3) + v3 := postgresTestDSN(t) + c, err := openFixtureSQL(v3) if err != nil { t.Fatal(err) } c.SetMaxOpenConns(1) - c.Exec(`CREATE TABLE observations (id INTEGER PRIMARY KEY, observer_idx INTEGER)`) - c.Exec(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY, hash TEXT)`) + c.Exec(testNativeSQL(`CREATE TABLE observations (id INTEGER PRIMARY KEY AUTOINCREMENT, observer_idx INTEGER)`, `CREATE TABLE observations (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, observer_idx INTEGER)`)) + c.Exec(testNativeSQL(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY AUTOINCREMENT, hash TEXT)`, `CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, hash TEXT)`)) c.Exec(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY)`) + ensurePreparable(t, c) c.Close() - db, err := OpenDB(v3) + db, err := openFixtureReader(t, v3) if err != nil { t.Fatalf("OpenDB v3: %v", err) } @@ -2406,17 +2734,18 @@ func TestDetectSchemaV3AndV2(t *testing.T) { } db.Close() - v2 := filepath.Join(dir, "v2.db") - c2, err := sql.Open("sqlite", v2) + v2 := postgresTestDSN(t) + c2, err := openFixtureSQL(v2) if err != nil { t.Fatal(err) } c2.SetMaxOpenConns(1) - c2.Exec(`CREATE TABLE observations (id INTEGER PRIMARY KEY, observer_id INTEGER)`) - c2.Exec(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY, hash TEXT)`) + c2.Exec(testNativeSQL(`CREATE TABLE observations (id INTEGER PRIMARY KEY AUTOINCREMENT, observer_id INTEGER)`, `CREATE TABLE observations (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, observer_id INTEGER)`)) + c2.Exec(testNativeSQL(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY AUTOINCREMENT, hash TEXT)`, `CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, hash TEXT)`)) c2.Exec(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY)`) + ensurePreparable(t, c2) c2.Close() - db2, err := OpenDB(v2) + db2, err := openFixtureReader(t, v2) if err != nil { t.Fatalf("OpenDB v2: %v", err) } diff --git a/cmd/server/decoder.go b/cmd/server/decoder.go index 8f7981ef5..875e6052a 100644 --- a/cmd/server/decoder.go +++ b/cmd/server/decoder.go @@ -1,7 +1,6 @@ package main import ( - "crypto/sha256" "encoding/binary" "encoding/hex" "encoding/json" @@ -669,55 +668,7 @@ func DecodePacket(hexString string, validateSignatures bool) (*DecodedPacket, er // route-independent identifier for the same logical packet. For TRACE packets, // path_len is included in the hash to match firmware behavior. func ComputeContentHash(rawHex string) string { - buf, err := hex.DecodeString(rawHex) - if err != nil || len(buf) < 2 { - if len(rawHex) >= 16 { - return rawHex[:16] - } - return rawHex - } - - headerByte := buf[0] - offset := 1 - if isTransportRoute(int(headerByte & 0x03)) { - offset += 4 - } - if offset >= len(buf) { - if len(rawHex) >= 16 { - return rawHex[:16] - } - return rawHex - } - pathByte := buf[offset] - offset++ - hashSize := int((pathByte>>6)&0x3) + 1 - hashCount := int(pathByte & 0x3F) - pathBytes := hashSize * hashCount - - payloadStart := offset + pathBytes - if payloadStart > len(buf) { - if len(rawHex) >= 16 { - return rawHex[:16] - } - return rawHex - } - - payload := buf[payloadStart:] - - // Hash payload-type byte only (bits 2-5 of header), not the full header. - // Firmware: SHA256(payload_type + [path_len for TRACE] + payload) - // Using the full header caused different hashes for the same logical packet - // when route type or version bits differed. See issue #786. - payloadType := (headerByte >> 2) & 0x0F - toHash := []byte{payloadType} - if int(payloadType) == PayloadTRACE { - // Firmware uses uint16_t path_len (2 bytes, little-endian) - toHash = append(toHash, pathByte, 0x00) - } - toHash = append(toHash, payload...) - - h := sha256.Sum256(toHash) - return hex.EncodeToString(h[:])[:16] + return packetpath.ContentHash(rawHex) } // PayloadJSON serializes the payload to JSON for DB storage. diff --git a/cmd/server/direct_heard.go b/cmd/server/direct_heard.go new file mode 100644 index 000000000..9118a2d65 --- /dev/null +++ b/cmd/server/direct_heard.go @@ -0,0 +1,297 @@ +// Package main: direct-RF attribution for the node-health "Heard By" card. +// +// An observer "heard" a node when it received that node's own transmission +// off the air. That is a narrower relation than "saw a packet this node was +// involved in", which is what the node-health card reported before this +// file existed, and it is the only one for which the SNR and RSSI printed +// next to an observer belong to the node the row names. +// +// The rule follows the firmware: +// +// - Flood routes build the path up as they travel: a forwarding repeater +// appends its own hash before retransmitting (Mesh.cpp:349). The last +// hop is therefore the node whose transmission the observer received. +// - Direct routes carry the REMAINING route, not the travelled one. A +// forwarder matches itself against the head of the path and calls +// removeSelfFromPath (Mesh.cpp:89,103) before retransmitting, so the +// node the observer heard is not in the path at all. Direct routes +// therefore never attribute. +// - An empty flood path means the observer received the originator's own +// transmission. Only ADVERTs carry the originator's pubkey in the +// clear, so other payload types with an empty path attribute to nobody. +// - A hop prefix that matches more than one node attributes to nobody. +// Path hop sizes are chosen by the originator (Packet.h:83, +// Mesh.cpp:649) and default to one byte, so a hop often matches many +// candidates. resolveWithContext guesses in that case; this file does +// not. The gate matches resolvePathForObsColdLoad: under-attribute +// rather than credit the wrong node. +package main + +import ( + "sort" + "strings" +) + +// directHeardAgg accumulates one observer's direct receptions of one node. +type directHeardAgg struct { + ObserverName string + Count int + SNRSum float64 + SNRCount int + RSSISum float64 + RSSICount int +} + +// directHeardIndex maps node pubkey (lowercase) to observer id to aggregate. +type directHeardIndex map[string]map[string]*directHeardAgg + +// HealthObserverRow is one row of the node-health "Heard By" table. +// +// Field names and null semantics are unchanged from the map-based rows this +// replaced, so the frontend needs no migration: avgSnr and avgRssi are null +// when the observer contributed no sample, and can_relay is tri-state (null +// = no repeat field ever reported, see PR #1624). +type HealthObserverRow struct { + ObserverID string `json:"observer_id"` + ObserverName string `json:"observer_name"` + AvgSNR *float64 `json:"avgSnr"` + AvgRSSI *float64 `json:"avgRssi"` + PacketCount int `json:"packetCount"` + CanRelay *bool `json:"can_relay"` +} + +// lastPathHop returns the last quoted token of a path_json array, or "" when +// there is none. It scans backwards rather than unmarshalling: the compute +// pass runs over every observation in the store (2.9M on the reference +// deployment) and only ever needs the final element. +func lastPathHop(pathJSON string) string { + end := -1 + for i := len(pathJSON) - 1; i >= 0; i-- { + if pathJSON[i] != '"' { + continue + } + if end < 0 { + end = i + continue + } + return pathJSON[i+1 : end] + } + return "" +} + +// advertOriginPubkey returns the pubkey an ADVERT announces, or "" when the +// transmission is not an ADVERT or carries no decodable pubkey. Mirrors the +// field probing in trackAdvertPubkey. +func advertOriginPubkey(tx *StoreTx) string { + if tx.PayloadType == nil || *tx.PayloadType != PayloadADVERT || tx.DecodedJSON == "" { + return "" + } + d := tx.ParsedDecoded() + if d == nil { + return "" + } + if v, ok := d["pubKey"].(string); ok && v != "" { + return strings.ToLower(v) + } + if v, ok := d["public_key"].(string); ok && v != "" { + return strings.ToLower(v) + } + return "" +} + +// directHeardNode returns the lowercase pubkey of the node whose +// transmission this observation received off the air, or "" when that cannot +// be established. See the package comment for the rule and its firmware +// grounding. +func directHeardNode(tx *StoreTx, obs *StoreObs, pm *prefixMap) string { + if tx == nil || obs == nil || tx.RouteType == nil { + return "" + } + hop := lastPathHop(obs.PathJSON) + switch *tx.RouteType { + case RouteFlood, RouteTransportFlood: + // Handled below: empty path means the originator, otherwise the last + // hop is whoever was heard. + case RouteDirect, RouteTransportDirect: + // A direct route normally says nothing about the transmitter, because + // the forwarder calls removeSelfFromPath before retransmitting + // (firmware Mesh.cpp) and the path left behind is the REMAINING route. + // + // The exception is a zero hop. Mesh::sendZeroHop sets ROUTE_TYPE_DIRECT + // (its transport overload ROUTE_TYPE_TRANSPORT_DIRECT) and path_len = 0, + // commented there as "path_len of zero means Zero Hop". Repeaters send + // their periodic local advert that way (simple_repeater/MyMesh.cpp, the + // next_local_advert branch and sendSelfAdvertisement with flood=false), + // and so do companions. An ADVERT arriving on a direct route with an + // empty path therefore cannot have been forwarded: the observer heard + // the advertiser's own transmission, and the advert carries its pubkey + // in the clear. That is the strongest direct-RF evidence there is. + // + // Anything else direct still says nothing: a non-empty path is the + // remaining route, and advertOriginPubkey returns "" for every payload + // type other than ADVERT. + if hop == "" { + return advertOriginPubkey(tx) + } + return "" + default: + return "" + } + if hop == "" { + return advertOriginPubkey(tx) + } + if pm == nil { + return "" + } + candidates := pm.relayCandidates(hop) + if len(candidates) != 1 { + return "" + } + return strings.ToLower(candidates[0].PublicKey) +} + +// buildDirectHeardIndex folds every observation of every transmission into +// the node-to-observer aggregate. Pure over its arguments so it can be +// tested and benchmarked without a store. +func buildDirectHeardIndex(packets []*StoreTx, pm *prefixMap) directHeardIndex { + idx := make(directHeardIndex, 256) + for _, tx := range packets { + for _, obs := range tx.Observations { + if obs.ObserverID == "" { + continue + } + pk := directHeardNode(tx, obs, pm) + if pk == "" { + continue + } + byObs := idx[pk] + if byObs == nil { + byObs = make(map[string]*directHeardAgg, 4) + idx[pk] = byObs + } + agg := byObs[obs.ObserverID] + if agg == nil { + agg = &directHeardAgg{ObserverName: obs.ObserverName} + byObs[obs.ObserverID] = agg + } + agg.Count++ + if obs.SNR != nil { + agg.SNRSum += *obs.SNR + agg.SNRCount++ + } + if obs.RSSI != nil { + agg.RSSISum += *obs.RSSI + agg.RSSICount++ + } + } + } + return idx +} + +// computeDirectHeard rebuilds the whole index from the current store. Run by +// a background recomputer rather than per request: the reference deployment +// holds 232,928 transmissions and 2,887,861 observations, and one node's +// byNode slice alone can hold 1.45M observations. +// +// Rebuilding wholesale also means eviction needs no bookkeeping — a pass +// simply does not see transmissions that are gone. +func (s *PacketStore) computeDirectHeard() directHeardIndex { + s.mu.RLock() + defer s.mu.RUnlock() + _, pm := s.getCachedNodesAndPM() + return buildDirectHeardIndex(s.packets, pm) +} + +// publishDirectHeard installs a snapshot for readers. Called by the +// recomputer after each pass; a nil-safe no-op for callers that have nothing +// to publish. +func (s *PacketStore) publishDirectHeard(idx directHeardIndex) { + if idx == nil { + idx = directHeardIndex{} + } + s.directHeardSnap.Store(idx) +} + +// loadDirectHeard returns the latest snapshot, or nil before the first +// compute has published one. A nil index simply yields empty direct rows: +// the card degrades to "nobody hears this node" until the first pass lands, +// never to a wrong attribution. +func (s *PacketStore) loadDirectHeard() directHeardIndex { + idx, _ := s.directHeardSnap.Load().(directHeardIndex) + return idx +} + +// canRelaySets fetches the two inputs behind the can_relay tri-state badge +// (#1290, PR #1624): the observers that reported repeat:off, and the +// observers we have any repeat field for at all. Both are lowercase to match +// pm.nonRelay and GetNonRelayObserverPubkeys; two case conventions on the +// same upstream string would be a latent regression. A read failure degrades +// to "no badge" rather than a wrong badge. +func (s *PacketStore) canRelaySets() (nonRelay, seen map[string]struct{}) { + nonRelay = map[string]struct{}{} + seen = map[string]struct{}{} + if s.db == nil || s.db.conn == nil { + return nonRelay, seen + } + if pks, err := s.db.GetNonRelayObserverPubkeys(); err == nil { + for _, pk := range pks { + nonRelay[strings.ToLower(pk)] = struct{}{} + } + } + if pks, err := s.db.GetCanRelaySeenObserverPubkeys(); err == nil { + for _, pk := range pks { + seen[strings.ToLower(pk)] = struct{}{} + } + } + return nonRelay, seen +} + +// relayOnlyObserverCount counts observers that saw traffic involving the node +// without hearing it on air. seenObservers is the set the health builders +// already collect from each transmission's representative observation. +func relayOnlyObserverCount(seenObservers map[string]struct{}, direct map[string]*directHeardAgg) int { + n := 0 + for id := range seenObservers { + if _, isDirect := direct[id]; !isDirect { + n++ + } + } + return n +} + +// buildDirectObserverRows renders one node's aggregate as sorted API rows. +// nonRelay and seen carry the can_relay tri-state; both may be nil. +func buildDirectObserverRows(byObs map[string]*directHeardAgg, nonRelay, seen map[string]struct{}) []HealthObserverRow { + rows := make([]HealthObserverRow, 0, len(byObs)) + for id, agg := range byObs { + row := HealthObserverRow{ + ObserverID: id, + ObserverName: agg.ObserverName, + PacketCount: agg.Count, + } + if agg.SNRCount > 0 { + v := agg.SNRSum / float64(agg.SNRCount) + row.AvgSNR = &v + } + if agg.RSSICount > 0 { + v := agg.RSSISum / float64(agg.RSSICount) + row.AvgRSSI = &v + } + idLower := strings.ToLower(id) + if _, ok := seen[idLower]; ok { + _, isListener := nonRelay[idLower] + canRelay := !isListener + row.CanRelay = &canRelay + } + rows = append(rows, row) + } + // Packet count descending, observer id ascending as a deterministic + // tiebreak so repeated requests return a stable order. + sort.Slice(rows, func(i, j int) bool { + if rows[i].PacketCount != rows[j].PacketCount { + return rows[i].PacketCount > rows[j].PacketCount + } + return rows[i].ObserverID < rows[j].ObserverID + }) + return rows +} diff --git a/cmd/server/direct_heard_test.go b/cmd/server/direct_heard_test.go new file mode 100644 index 000000000..b5101ec38 --- /dev/null +++ b/cmd/server/direct_heard_test.go @@ -0,0 +1,310 @@ +package main + +import ( + "testing" +) + +// Coverage for the direct-RF attribution rule behind the node detail +// "Heard By" card. +// +// The card used to credit every observer that saw traffic *involving* a +// node — originated by it, addressed to it, or relayed through it — and +// printed an SNR/RSSI next to each. Those signal numbers belong to +// whichever node last transmitted the copy the observer received, not to +// the node the row names. +// +// The rule these tests pin, derived from the firmware: +// +// - Only flood routes carry a travelled path. For ROUTE_TYPE_DIRECT the +// forwarder removes itself from the front before retransmitting +// (firmware Mesh.cpp:103 removeSelfFromPath), so path_json is the +// REMAINING route and says nothing about who was heard. +// - On a flood, the last hop is the node the observer heard on air +// (firmware Mesh.cpp:349 — a repeater appends its own hash before +// retransmitting). +// - An empty flood path means the originator was heard directly. Only +// ADVERTs carry the originator's pubkey in the clear. +// - An ambiguous hop prefix credits nobody. Same gate as +// resolvePathForObsColdLoad: under-attribute rather than guess. +// +// Fixture pubkeys are lowercase hex placeholders only (AGENTS.md PII rule). + +// directHeardNodes are three repeaters, two of which collide on the 1-byte +// prefix "a4" — the shape that produced the original report, where a 433 MHz +// repeater was credited with 868 MHz traffic it could not have relayed. +var directHeardNodes = []nodeInfo{ + {PublicKey: "a433ec0000000000000000000000000000000000000000000000000000000001", Role: "repeater", Name: "collideA"}, + {PublicKey: "a4ef4b0000000000000000000000000000000000000000000000000000000002", Role: "repeater", Name: "collideB"}, + {PublicKey: "bb11220000000000000000000000000000000000000000000000000000000003", Role: "repeater", Name: "unique"}, +} + +const ( + dhCollideA = "a433ec0000000000000000000000000000000000000000000000000000000001" + dhUnique = "bb11220000000000000000000000000000000000000000000000000000000003" +) + +func dhRoute(rt int) *int { return &rt } + +func dhTx(routeType int, payloadType int, decoded string) *StoreTx { + return &StoreTx{ + RouteType: dhRoute(routeType), + PayloadType: dhRoute(payloadType), + DecodedJSON: decoded, + } +} + +func TestLastPathHop(t *testing.T) { + cases := []struct { + name, in, want string + }{ + {"empty array", "[]", ""}, + {"empty string", "", ""}, + {"single hop", `["A4"]`, "A4"}, + {"two hops", `["A4","F1"]`, "F1"}, + {"two-byte hops", `["A433","1403"]`, "1403"}, + {"long path", `["66","E8","EA","DE","7C","CA"]`, "CA"}, + {"unterminated", `["A4`, ""}, + {"not json", "garbage", ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := lastPathHop(tc.in); got != tc.want { + t.Fatalf("lastPathHop(%q) = %q, want %q", tc.in, got, tc.want) + } + }) + } +} + +func TestDirectHeardNode(t *testing.T) { + pm := buildPrefixMap(directHeardNodes) + advert := `{"type":"ADVERT","pubKey":"` + dhUnique + `"}` + + cases := []struct { + name string + tx *StoreTx + pathJSON string + want string + }{ + { + name: "flood, unique two-byte last hop, credits that node", + tx: dhTx(RouteFlood, PayloadTXT_MSG, ""), + pathJSON: `["A433","BB11"]`, + want: dhUnique, + }, + { + name: "transport flood counts as flood", + tx: dhTx(RouteTransportFlood, PayloadTXT_MSG, ""), + pathJSON: `["BB11"]`, + want: dhUnique, + }, + { + name: "flood, node is an earlier hop, not the last one, credits nobody here", + tx: dhTx(RouteFlood, PayloadTXT_MSG, ""), + pathJSON: `["BB11","A433"]`, + want: dhCollideA, + }, + { + name: "ambiguous one-byte last hop credits nobody", + tx: dhTx(RouteFlood, PayloadTXT_MSG, ""), + pathJSON: `["66","E8","A4"]`, + want: "", + }, + { + name: "unknown last hop credits nobody", + tx: dhTx(RouteFlood, PayloadTXT_MSG, ""), + pathJSON: `["9999"]`, + want: "", + }, + { + name: "direct route never credits, even when the node is the last entry", + tx: dhTx(RouteDirect, PayloadTXT_MSG, ""), + pathJSON: `["A433","BB11"]`, + want: "", + }, + { + name: "transport direct never credits", + tx: dhTx(RouteTransportDirect, PayloadTXT_MSG, ""), + pathJSON: `["BB11"]`, + want: "", + }, + { + // Mesh::sendZeroHop sets ROUTE_TYPE_DIRECT and path_len = 0, and + // repeaters send their periodic local advert that way. An ADVERT + // arriving direct with an empty path cannot have been forwarded, + // so the observer heard the advertiser itself. + name: "zero-hop advert on a direct route credits the advertiser", + tx: dhTx(RouteDirect, PayloadADVERT, advert), + pathJSON: "", + want: dhUnique, + }, + { + name: "zero-hop advert on a transport-direct route credits the advertiser", + tx: dhTx(RouteTransportDirect, PayloadADVERT, advert), + pathJSON: `[]`, + want: dhUnique, + }, + { + // The zero-hop exception is only about adverts. Any other payload + // arriving direct with an empty path still identifies nobody. + name: "direct route, empty path, not an advert, credits nobody", + tx: dhTx(RouteDirect, PayloadTXT_MSG, ""), + pathJSON: "", + want: "", + }, + { + // A direct route with a path is the REMAINING route, so the last + // entry is where the packet is going, not who transmitted it. + name: "direct advert with a non-empty path credits nobody", + tx: dhTx(RouteDirect, PayloadADVERT, advert), + pathJSON: `["A433","BB11"]`, + want: "", + }, + { + name: "flood advert with empty path credits the originator", + tx: dhTx(RouteFlood, PayloadADVERT, advert), + pathJSON: `[]`, + want: dhUnique, + }, + { + name: "flood non-advert with empty path credits nobody: originator unknown", + tx: dhTx(RouteFlood, PayloadTXT_MSG, ""), + pathJSON: `[]`, + want: "", + }, + { + name: "missing route type credits nobody", + tx: &StoreTx{PayloadType: dhRoute(PayloadTXT_MSG)}, + pathJSON: `["BB11"]`, + want: "", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + obs := &StoreObs{ObserverID: "obs1", PathJSON: tc.pathJSON} + if got := directHeardNode(tc.tx, obs, pm); got != tc.want { + t.Fatalf("directHeardNode = %q, want %q", got, tc.want) + } + }) + } +} + +// TestDirectHeardNode_AmbiguousPrefixRegression pins the reported case: a +// long 868 MHz flood path whose one-byte "A4" hop matches two repeaters must +// not credit either of them, however plausible the geo/affinity tiers of +// resolveWithContext would find one. +func TestDirectHeardNode_AmbiguousPrefixRegression(t *testing.T) { + pm := buildPrefixMap(directHeardNodes) + tx := dhTx(RouteFlood, PayloadTXT_MSG, "") + obs := &StoreObs{ + ObserverID: "obs868", + PathJSON: `["A4","F1","AE","6A","77","5C","29","ED","4D","3F","6A","E7","6A","C2","68"]`, + } + if got := directHeardNode(tx, obs, pm); got != "" { + t.Fatalf("ambiguous A4 hop credited %q; the prefix has %d candidates and must credit nobody", + got, len(pm.relayCandidates("a4"))) + } +} + +// TestDirectHeardNode_ListenerNeverCredited: an observer that reported +// repeat:off cannot have retransmitted the packet, so it must not survive as +// a last-hop candidate (#1290 parity — relayCandidates already filters it). +func TestDirectHeardNode_ListenerNeverCredited(t *testing.T) { + pm := buildPrefixMap(directHeardNodes) + pm.markNonRelay([]string{dhUnique}) + tx := dhTx(RouteFlood, PayloadTXT_MSG, "") + obs := &StoreObs{ObserverID: "obs1", PathJSON: `["BB11"]`} + if got := directHeardNode(tx, obs, pm); got != "" { + t.Fatalf("listener-only node credited as last hop: %q", got) + } +} + +func TestComputeDirectHeardAggregates(t *testing.T) { + snr := func(v float64) *float64 { return &v } + tx := dhTx(RouteFlood, PayloadTXT_MSG, "") + tx.Observations = []*StoreObs{ + {ObserverID: "obsA", ObserverName: "A", PathJSON: `["BB11"]`, SNR: snr(10), RSSI: snr(-50)}, + {ObserverID: "obsA", ObserverName: "A", PathJSON: `["BB11"]`, SNR: snr(20), RSSI: snr(-70)}, + {ObserverID: "obsB", ObserverName: "B", PathJSON: `["BB11","A433"]`, SNR: snr(5)}, + // Ambiguous last hop: contributes to nobody. + {ObserverID: "obsC", ObserverName: "C", PathJSON: `["A4"]`, SNR: snr(1)}, + } + + idx := buildDirectHeardIndex([]*StoreTx{tx}, buildPrefixMap(directHeardNodes)) + + byObs := idx[dhUnique] + if len(byObs) != 1 { + t.Fatalf("unique node has %d direct observers, want 1: %#v", len(byObs), byObs) + } + a := byObs["obsA"] + if a == nil || a.Count != 2 { + t.Fatalf("obsA aggregate = %#v, want Count 2", a) + } + if a.SNRCount != 2 || a.SNRSum != 30 { + t.Fatalf("obsA SNR = %v over %d, want 30 over 2", a.SNRSum, a.SNRCount) + } + if a.RSSICount != 2 || a.RSSISum != -120 { + t.Fatalf("obsA RSSI = %v over %d, want -120 over 2", a.RSSISum, a.RSSICount) + } + + if got := len(idx[dhCollideA]); got != 1 { + t.Fatalf("collideA has %d direct observers, want 1 (obsB heard it as last hop)", got) + } + for pk, m := range idx { + if _, ok := m["obsC"]; ok { + t.Fatalf("observer with an ambiguous last hop was credited to %s", pk) + } + } +} + +func TestBuildDirectObserverRowsSortedAndAveraged(t *testing.T) { + byObs := map[string]*directHeardAgg{ + "low": {ObserverName: "low", Count: 1, SNRSum: 3, SNRCount: 1}, + "high": {ObserverName: "high", Count: 9, SNRSum: 18, SNRCount: 2, RSSISum: -100, RSSICount: 2}, + "none": {ObserverName: "none", Count: 4}, + } + rows := buildDirectObserverRows(byObs, nil, nil) + if len(rows) != 3 { + t.Fatalf("got %d rows, want 3", len(rows)) + } + if rows[0].ObserverID != "high" || rows[1].ObserverID != "none" || rows[2].ObserverID != "low" { + t.Fatalf("rows not sorted by packet count desc: %#v", rows) + } + if rows[0].AvgSNR == nil || *rows[0].AvgSNR != 9 { + t.Fatalf("avgSnr = %v, want 9", rows[0].AvgSNR) + } + if rows[0].AvgRSSI == nil || *rows[0].AvgRSSI != -50 { + t.Fatalf("avgRssi = %v, want -50", rows[0].AvgRSSI) + } + if rows[1].AvgSNR != nil || rows[1].AvgRSSI != nil { + t.Fatalf("observer with no signal samples must report null, got %#v", rows[1]) + } +} + +// TestBuildDirectObserverRows_CanRelayTriState mirrors the nodes.js badge: +// nil means "no repeat field ever seen", false means listener, true means +// confirmed repeater (PR #1624). +func TestBuildDirectObserverRows_CanRelayTriState(t *testing.T) { + byObs := map[string]*directHeardAgg{ + "unknown": {Count: 3}, + "listener": {Count: 2}, + "repeater": {Count: 1}, + } + seen := map[string]struct{}{"listener": {}, "repeater": {}} + nonRelay := map[string]struct{}{"listener": {}} + rows := buildDirectObserverRows(byObs, nonRelay, seen) + + got := map[string]*bool{} + for _, r := range rows { + got[r.ObserverID] = r.CanRelay + } + if got["unknown"] != nil { + t.Fatalf("unknown observer must report nil can_relay, got %v", *got["unknown"]) + } + if got["listener"] == nil || *got["listener"] { + t.Fatalf("listener must report can_relay false, got %v", got["listener"]) + } + if got["repeater"] == nil || !*got["repeater"] { + t.Fatalf("repeater must report can_relay true, got %v", got["repeater"]) + } +} diff --git a/cmd/server/distance_lock_contention_test.go b/cmd/server/distance_lock_contention_test.go index f04e332a8..a508bc638 100644 --- a/cmd/server/distance_lock_contention_test.go +++ b/cmd/server/distance_lock_contention_test.go @@ -17,10 +17,22 @@ import ( // continuously, while the test goroutine measures how long it takes to // complete W bare mu.Lock()/mu.Unlock() cycles. Each writer cycle must // wait for ALL currently-holding RLocks to release. Pre-fix, every reader -// holds RLock for the entire compute (~ms), so each writer cycle waits -// behind an active reader → avg cycle hundreds of microseconds to -// milliseconds. Post-fix, readers hold RLock only long enough to grab -// slice headers (microseconds), so writer cycles complete unimpeded. +// holds RLock for the entire compute, so each writer cycle waits behind an +// active reader. Post-fix, readers hold RLock only long enough to grab +// slice headers, so writer cycles complete unimpeded. +// +// The threshold comes from measurement, and the gap it has to straddle is +// enormous (issue #2038). On CI runners: +// +// healthy 156µs, 222µs, 402µs (three readings, three commits) +// regressed 201203µs (RLock deliberately held across the compute) +// +// The old limit was a flat 150µs, which sits *inside* the healthy band, so +// it failed on two consecutive master commits that passed on re-run without +// a byte changed. 5ms is 12x above the worst healthy reading and 40x below +// the regression, which is the widest possible separation from both. The +// scale of the gap is the point: a #1239 regression is not marginal, it +// serializes a millisecond-scale compute behind every writer. func TestComputeAnalyticsDistanceLockHoldDuration(t *testing.T) { if testing.Short() { t.Skip("skipping concurrency timing test in -short mode") @@ -118,12 +130,11 @@ func TestComputeAnalyticsDistanceLockHoldDuration(t *testing.T) { t.Logf("avg writer Lock/Unlock cycle: %dµs over %d cycles (total %v) with %d concurrent readers, %d hops, %d paths", avgMicros, WriterCycles, elapsed, Readers, N, len(paths)) - // If readers hold the main RLock for their entire compute, every - // writer Lock cycle waits for an active reader to release: avg cycle - // >> 100µs at this data scale. After the refactor, readers hold the - // main RLock only long enough to snapshot slice headers (<1µs), so - // writer cycles complete in tens of microseconds. - const MaxAvgMicros = 150 + // See the measurements in the doc comment: healthy runs land in the + // hundreds of microseconds, a regression two orders of magnitude above + // that. Anything in between is a genuine change in lock-hold behaviour + // and deserves to be looked at, not re-run. + const MaxAvgMicros = 5000 if avgMicros > MaxAvgMicros { t.Fatalf("avg writer Lock/Unlock cycle %dµs exceeds %dµs threshold — computeAnalyticsDistance is holding the main RLock for too long and blocking writers (issue #1239)", avgMicros, MaxAvgMicros) diff --git a/cmd/server/encrypted_channels_test.go b/cmd/server/encrypted_channels_test.go index d8632b886..2c19b296b 100644 --- a/cmd/server/encrypted_channels_test.go +++ b/cmd/server/encrypted_channels_test.go @@ -16,15 +16,15 @@ func seedEncryptedChannelData(t *testing.T, db *DB) { // Two encrypted GRP_TXT packets on channel hash "A1B2" db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash) - VALUES ('EE01', 'enc_hash_001', ?, 1, 5, '{"type":"GRP_TXT","channelHashHex":"A1B2","decryptionStatus":"no_key"}', 'enc_A1B2')`, recent) + VALUES ('EE01', 'enc_hash_001', $1, 1, 5, '{"type":"GRP_TXT","channelHashHex":"A1B2","decryptionStatus":"no_key"}', 'enc_A1B2')`, recent) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash) - VALUES ('EE02', 'enc_hash_002', ?, 1, 5, '{"type":"GRP_TXT","channelHashHex":"A1B2","decryptionStatus":"no_key"}', 'enc_A1B2')`, recent) + VALUES ('EE02', 'enc_hash_002', $1, 1, 5, '{"type":"GRP_TXT","channelHashHex":"A1B2","decryptionStatus":"no_key"}', 'enc_A1B2')`, recent) // Observations for both db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES ((SELECT id FROM transmissions WHERE hash='enc_hash_001'), 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES ((SELECT id FROM transmissions WHERE hash='enc_hash_001'), 1, 10.0, -90, '[]', $1)`, recentEpoch) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES ((SELECT id FROM transmissions WHERE hash='enc_hash_002'), 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES ((SELECT id FROM transmissions WHERE hash='enc_hash_002'), 1, 10.0, -90, '[]', $1)`, recentEpoch) } func TestGetEncryptedChannels(t *testing.T) { diff --git a/cmd/server/evict_order_test.go b/cmd/server/evict_order_test.go new file mode 100644 index 000000000..e0350ed32 --- /dev/null +++ b/cmd/server/evict_order_test.go @@ -0,0 +1,82 @@ +package main + +import ( + "fmt" + "testing" + "time" +) + +// s.packets is declared "sorted by first_seen ASC (oldest first; newest at +// tail)" (store.go:177), and retention eviction depends on it: it walks from +// the head and stops at the first transmission inside the window, so a slice +// out of order is under-evicted silently rather than noisily wrong. +// +// The background chunk loader broke that. Chunks are windowed on last_seen, so +// a transmission first heard weeks ago and heard again recently rides in with +// a recent chunk carrying its old first_seen; the chunk was put in front with +// `append(localPackets, s.packets...)` and never re-sorted, so a later, older +// chunk prepended in front of it left that ancient row behind newer ones. +// +// Not a corner case: on a production database 2071 of the 236080 transmissions +// in a 14 day window have a first_seen more than a day older than their +// last_seen, 1848 of them more than a week. + +// mergeChunkIntoPackets is what keeps that invariant true, so pin it directly. +func TestMergeChunkIntoPackets_KeepsFirstSeenOrder(t *testing.T) { + at := func(h int) string { return time.Now().UTC().Add(-time.Duration(h) * time.Hour).Format(time.RFC3339) } + + existing := []*StoreTx{ + {ID: 2, Hash: "b", FirstSeen: at(30)}, + {ID: 4, Hash: "d", FirstSeen: at(10)}, + } + // Interleaves with what is already loaded, and is deliberately not in + // order: the chunk query sorts, but the merge must not depend on it. + chunk := []*StoreTx{ + {ID: 3, Hash: "c", FirstSeen: at(20)}, + {ID: 1, Hash: "a", FirstSeen: at(40)}, + {ID: 5, Hash: "e", FirstSeen: at(5)}, + } + + merged := mergeChunkIntoPackets(chunk, existing) + + if len(merged) != 5 { + t.Fatalf("merged %d packets, want 5", len(merged)) + } + got := "" + for _, tx := range merged { + got += tx.Hash + } + if got != "abcde" { + t.Fatalf("merge order = %q, want %q", got, "abcde") + } + for i := 1; i < len(merged); i++ { + if merged[i-1].FirstSeen > merged[i].FirstSeen { + t.Fatalf("merge left the slice out of order at index %d", i) + } + } +} + +// The merge runs under s.mu once per chunk, so it must stay linear. This +// guards against it being "simplified" back into a sort of the whole slice, +// which on a loaded instance means sorting hundreds of thousands of packets +// while ingest waits for the lock. +func BenchmarkMergeChunkIntoPackets(b *testing.B) { + base := time.Now().UTC().Add(-14 * 24 * time.Hour) + mk := func(n, stride, off int) []*StoreTx { + out := make([]*StoreTx, n) + for i := 0; i < n; i++ { + out[i] = &StoreTx{ + ID: i, + Hash: fmt.Sprintf("h%06d", i), + FirstSeen: base.Add(time.Duration(i*stride+off) * time.Second).Format(time.RFC3339), + } + } + return out + } + existing := mk(200000, 6, 0) + chunk := mk(20000, 60, 3) + b.ResetTimer() + for i := 0; i < b.N; i++ { + mergeChunkIntoPackets(chunk, existing) + } +} diff --git a/cmd/server/eviction_test.go b/cmd/server/eviction_test.go index e83aa0dd7..e16014616 100644 --- a/cmd/server/eviction_test.go +++ b/cmd/server/eviction_test.go @@ -87,7 +87,7 @@ func makeTestStore(count int, startTime time.Time, intervalMin int) *PacketStore addTxToSubpathIndex(store.spIndex, tx) // Track bytes for self-accounting - store.trackedBytes += estimateStoreTxBytes(tx) + store.trackedBytes += rechargeTx(tx) for _, obs := range tx.Observations { store.trackedBytes += estimateStoreObsBytes(obs) } @@ -295,9 +295,9 @@ func TestEvictStale_CleansResolvedPathNodeIndexes(t *testing.T) { tx.Observations = append(tx.Observations, obs) // Insert into DB so on-demand SQL fetch works during eviction - db.conn.Exec("INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (?, '', ?, ?)", + db.conn.Exec("INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES ($1, '', $2, $3)", txID, tx.Hash, tx.FirstSeen) - db.conn.Exec("INSERT INTO observations (id, transmission_id, observer_idx, path_json, timestamp, resolved_path) VALUES (?, ?, 1, ?, ?, ?)", + db.conn.Exec("INSERT INTO observations (id, transmission_id, observer_idx, path_json, timestamp, resolved_path) VALUES ($1, $2, 1, $3, $4, $5)", obsID, txID, `["aa"]`, now.Add(-48*time.Hour).Unix(), `["`+relayPK+`"]`) store.packets = append(store.packets, tx) @@ -460,7 +460,7 @@ func TestTrackedBytes_MatchesExpectedAfterMixedInsertEvict(t *testing.T) { var evictedBytes int64 for i := 0; i < 50; i++ { tx := store.packets[i] - evictedBytes += estimateStoreTxBytes(tx) + evictedBytes += tx.accountedBytes for _, obs := range tx.Observations { evictedBytes += estimateStoreObsBytes(obs) } @@ -568,6 +568,7 @@ func TestEstimateStoreTxBytes(t *testing.T) { // Manual calculation: base + string lengths + index entries + perTxMaps + path hops + subpaths hops := int64(len(txGetParsedPath(tx))) manualCalc := int64(storeTxBaseBytes) + int64(len(tx.RawHex)+len(tx.Hash)+len(tx.DecodedJSON)+len(tx.PathJSON)) + int64(numIndexesPerTx*indexEntryBytes) + manualCalc += int64(decodedCacheFactor * len(tx.DecodedJSON)) manualCalc += perTxMapsBytes manualCalc += hops * perPathHopBytes if hops > 1 { @@ -576,8 +577,8 @@ func TestEstimateStoreTxBytes(t *testing.T) { if est != manualCalc { t.Fatalf("estimateStoreTxBytes = %d, want %d (manual calc)", est, manualCalc) } - if est < 600 || est > 1200 { - t.Fatalf("estimateStoreTxBytes = %d, expected in range [600, 1200]", est) + if est < 600 || est > 1300 { + t.Fatalf("estimateStoreTxBytes = %d, expected in range [600, 1300]", est) } } @@ -587,8 +588,9 @@ func TestEstimateStoreObsBytes(t *testing.T) { PathJSON: `["aa"]`, } est := estimateStoreObsBytes(obs) - // storeObsBaseBytes(192) + len(ObserverID=6) + len(PathJSON=6) + 2*48(96) = 300 - expected := int64(192 + 6 + 6 + 2*48) + // storeObsBaseBytes(192) + len(ObserverID=6) + len(PathJSON=6) + 2*48(96) = 300, + // plus the obsKeys dedup entry: obsKeyEntryBytes(65) + 6 + 6 = 77 + expected := int64(192 + 6 + 6 + 2*48 + 77) if est != expected { t.Fatalf("estimateStoreObsBytes = %d, want %d", est, expected) } diff --git a/cmd/server/first_seen_1166_test.go b/cmd/server/first_seen_1166_test.go index f056c1274..1c7f386ba 100644 --- a/cmd/server/first_seen_1166_test.go +++ b/cmd/server/first_seen_1166_test.go @@ -24,7 +24,7 @@ func TestFirstSeen_1166_HandleNodesSurface(t *testing.T) { last := time.Now().UTC().Format("2006-01-02T15:04:05.000Z") if _, err := db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, 'rpt', 'repeater', 37.5, -122.0, ?, ?, 5)`, + VALUES ($1, 'rpt', 'repeater', 37.5, -122.0, $2, $3, 5)`, pk, last, first); err != nil { t.Fatal(err) } diff --git a/cmd/server/go.mod b/cmd/server/go.mod index ae93a43fc..8cec7a049 100644 --- a/cmd/server/go.mod +++ b/cmd/server/go.mod @@ -1,13 +1,12 @@ module github.com/corescope/server -go 1.22 +go 1.25.0 require ( github.com/gorilla/mux v1.8.1 github.com/gorilla/websocket v1.5.3 github.com/meshcore-analyzer/geofilter v0.0.0 github.com/meshcore-analyzer/sigvalidate v0.0.0 - modernc.org/sqlite v1.34.5 ) replace github.com/meshcore-analyzer/geofilter => ../../internal/geofilter @@ -34,25 +33,57 @@ require github.com/meshcore-analyzer/lora v0.0.0 replace github.com/meshcore-analyzer/lora => ../../internal/lora +require github.com/meshcore-analyzer/prunequeue v0.0.0 + +replace github.com/meshcore-analyzer/prunequeue => ../../internal/prunequeue + +require ( + github.com/meshcore-analyzer/mbcapqueue v0.0.0 + golang.org/x/sync v0.13.0 +) + +replace github.com/meshcore-analyzer/mbcapqueue => ../../internal/mbcapqueue + +require ( + github.com/meshcore-analyzer/mailer v0.0.0-00010101000000-000000000000 + github.com/meshcore-analyzer/users v0.0.0 +) + +replace github.com/meshcore-analyzer/users => ../../internal/users + +require ( + github.com/jackc/pgx/v5 v5.7.6 + github.com/meshcore-analyzer/channel v0.0.0 + modernc.org/sqlite v1.34.5 +) + require ( github.com/dustin/go-humanize v1.0.1 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/ncruces/go-strftime v0.1.9 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect - golang.org/x/sys v0.22.0 // indirect + golang.org/x/text v0.24.0 // indirect modernc.org/libc v1.55.3 // indirect modernc.org/mathutil v1.6.0 // indirect modernc.org/memory v1.8.0 // indirect ) -require github.com/meshcore-analyzer/prunequeue v0.0.0 - -replace github.com/meshcore-analyzer/prunequeue => ../../internal/prunequeue +replace github.com/meshcore-analyzer/channel => ../../internal/channel require ( - github.com/meshcore-analyzer/mbcapqueue v0.0.0 - golang.org/x/sync v0.10.0 + github.com/mattn/go-sqlite3 v1.14.52 + github.com/meshcore-analyzer/pgutil v0.0.0 + github.com/meshcore-analyzer/sqliteutil v0.0.0 + golang.org/x/crypto v0.37.0 // indirect + golang.org/x/sys v0.32.0 // indirect ) -replace github.com/meshcore-analyzer/mbcapqueue => ../../internal/mbcapqueue +replace github.com/meshcore-analyzer/mailer => ../../internal/mailer + +replace github.com/meshcore-analyzer/pgutil => ../../internal/pgutil + +replace github.com/meshcore-analyzer/sqliteutil => ../../internal/sqliteutil diff --git a/cmd/server/go.sum b/cmd/server/go.sum index 45b74210f..c370cf1a4 100644 --- a/cmd/server/go.sum +++ b/cmd/server/go.sum @@ -1,3 +1,6 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo= @@ -8,21 +11,46 @@ github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.7.6 h1:rWQc5FwZSPX58r1OQmkuaNicxdmExaEz5A2DO2hUuTk= +github.com/jackc/pgx/v5 v5.7.6/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-sqlite3 v1.14.52 h1:wVbm2Qnf4OXkqhBTSPuCRZDRnxfbVrrmiCEroVdog8U= +github.com/mattn/go-sqlite3 v1.14.52/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w= github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= -golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= -golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= +golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= +golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= -golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw= -golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc= +golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= +golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= +golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= modernc.org/cc/v4 v4.21.4 h1:3Be/Rdo1fpr8GrQ7IVw9OHtplU4gWbb+wNgeoBMmGLQ= modernc.org/cc/v4 v4.21.4/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ= modernc.org/ccgo/v4 v4.19.2 h1:lwQZgvboKD0jBwdaeVCTouxhxAyN6iawF3STraAal8Y= diff --git a/cmd/server/grouped_history_page_test.go b/cmd/server/grouped_history_page_test.go new file mode 100644 index 000000000..374a5fb7e --- /dev/null +++ b/cmd/server/grouped_history_page_test.go @@ -0,0 +1,157 @@ +package main + +import ( + "encoding/json" + "fmt" + "strings" + "testing" + "time" +) + +func TestGroupedHistoryPagePreservesFiltersAndDetails(t *testing.T) { + db, writer := openPostgresValueFixture(t) + now := time.Date(2026, 10, 12, 12, 0, 0, 0, time.UTC) + exec := func(query string, args ...any) { + t.Helper() + if _, err := writer.Exec(query, args...); err != nil { + t.Fatal(err) + } + } + exec(`INSERT INTO observers(rowid,id,name,iata) VALUES(1,'history-a','Observer A','AAA'),(2,'history-b','Observer B','BBB')`) + for id, age := range []time.Duration{48 * time.Hour, time.Minute, 2 * time.Minute, 12 * time.Hour} { + exec(`INSERT INTO transmissions(id,raw_hex,hash,first_seen,last_seen,payload_type,scope_name) VALUES($1,'AA',$2,$3,0,4,'history-scope')`, id+1, fmt.Sprintf("history-%d", id+1), now.Add(-age).Format(time.RFC3339)) + } + exec(`INSERT INTO observations(id,transmission_id,observer_idx,path_json,timestamp,snr) VALUES + (1,1,1,'["aa","bb","cc"]',$1,3), + (2,1,2,'["dd"]',$2,2), + (3,1,1,'["aa"]',$3,1), + (4,2,1,'[]',$4,4), + (5,4,1,'[]',$2,5)`, now.Add(-25*time.Hour).Unix(), now.Add(-5*time.Minute).Unix(), now.Add(-26*time.Hour).Unix(), now.Add(-72*time.Hour).Unix()) + q := PacketQuery{Since: now.Add(-24 * time.Hour).Format(time.RFC3339), Observer: "history-a", Limit: 1, Offset: 1} + result, err := db.QueryGroupedPackets(q) + if err != nil { + t.Fatal(err) + } + if result.Total != 2 || len(result.Packets) != 1 { + t.Fatalf("filtered page/total = %#v", result) + } + row := result.Packets[0] + // Equal latest timestamps are paged deterministically by descending ID. + // The since/observer predicates match independently, as before; details + // still include every observation, including ones outside that window. + if row["hash"] != "history-1" || row["count"] != 3 || row["observer_count"] != 2 || row["observer_id"] != "history-a" || row["path_json"] != `["aa","bb","cc"]` || row["scope_name"] != "history-scope" { + t.Fatalf("history details changed or tie page unstable: %#v", row) + } + if got := strings.Join(row["distinct_iatas"].([]string), ","); got != "AAA,BBB" { + t.Fatalf("IATAs restricted to filter window: %q", got) + } + if row["latest"] != now.Add(-5*time.Minute).Format("2006-01-02T15:04:05.000Z") { + t.Fatalf("latest changed: %#v", row["latest"]) + } + all, err := db.QueryGroupedPackets(PacketQuery{Limit: 10}) + if err != nil || all.Total != 4 || len(all.Packets) != 4 { + t.Fatalf("all history: %#v / %v", all, err) + } + if all.Packets[0]["hash"] != "history-3" || all.Packets[0]["latest"] != now.Add(-2*time.Minute).Format(time.RFC3339) { + t.Fatalf("no-observation first_seen fallback changed: %#v", all.Packets[0]) + } + if all.Packets[3]["hash"] != "history-2" { + t.Fatalf("ordered by first_seen/denormalized last_seen instead of observations: %#v", all.Packets) + } + end, err := db.QueryGroupedPackets(PacketQuery{Limit: 1, Offset: 10}) + if err != nil || end.Total != 4 || len(end.Packets) != 0 { + t.Fatalf("empty page lost total: %#v / %v", end, err) + } +} + +func TestPostgresGroupedHistoryHydratesOnlyPage(t *testing.T) { + postgresOnly(t) + db := startupMemoryFixture(t, 1000, 65, false) + sql := db.groupedPacketSelectSQL("", 0) + var raw []byte + if err := db.conn.QueryRow("EXPLAIN (ANALYZE, VERBOSE, FORMAT JSON) "+sql, 2, 10).Scan(&raw); err != nil { + t.Fatal(err) + } + type planNode struct { + Type string `json:"Node Type"` + Rows float64 `json:"Actual Rows"` + Loops float64 `json:"Actual Loops"` + Output []string `json:"Output"` + SortKey []string `json:"Sort Key"` + Relation string `json:"Relation Name"` + Direction string `json:"Scan Direction"` + Condition string `json:"Index Cond"` + Plans []planNode `json:"Plans"` + } + var plans []struct{ Plan planNode } + if err := json.Unmarshal(raw, &plans); err != nil || len(plans) != 1 { + t.Fatalf("invalid plan: %v", err) + } + pending := []planNode{plans[0].Plan} + foundBestPath := false + foundLatestSeek := false + for len(pending) > 0 { + n := pending[0] + pending = append(pending[1:], n.Plans...) + if strings.Contains(strings.Join(n.SortKey, " "), "length(") { + foundBestPath = true + if n.Loops > 2 { + t.Fatalf("two-row page hydrated %g best-path groups", n.Loops) + } + } + if n.Type == "Aggregate" && strings.Contains(strings.Join(n.Output, " "), "max(to_char(") { + t.Fatal("latest still formats every observation before aggregating") + } + if n.Relation == "observations" { + t.Logf("observation scan: %s direction=%s rows=%g loops=%g condition=%s", n.Type, n.Direction, n.Rows, n.Loops, n.Condition) + } + condition := strings.ReplaceAll(n.Condition, `"`, "") + if n.Relation == "observations" && strings.Contains(n.Type, "Index") && n.Direction == "Backward" && strings.Contains(condition, "timestamp IS NOT NULL") { + foundLatestSeek = true + if n.Rows != 1 || !strings.Contains(condition, "transmission_id") { + t.Fatalf("latest lookup scanned observations instead of seeking one: rows=%g, condition=%s", n.Rows, n.Condition) + } + } + } + if !foundBestPath { + t.Fatal("plan did not exercise best-path selection") + } + if !foundLatestSeek { + t.Fatal("latest lookup did not use the existing timestamp index") + } +} + +func TestGroupedHistoryLegacyV2(t *testing.T) { + dsn := postgresTestDSN(t) + createTestDBAt(t, dsn, 2) + db, err := openFixtureReader(t, dsn) + if err != nil { + t.Fatal(err) + } + defer db.Close() + result, err := db.QueryGroupedPackets(PacketQuery{Limit: 1}) + if err != nil || result.Total != 2 || len(result.Packets) != 1 { + t.Fatalf("legacy page: %#v / %v", result, err) + } + row := result.Packets[0] + if row["hash"] != "h0002" || row["latest"] != "2026-01-01T00:02:00Z" || row["count"] != 1 || row["observer_id"] != "obs1" { + t.Fatalf("legacy timestamp or observer changed: %#v", row) + } +} + +func BenchmarkGroupedPacketHistory(b *testing.B) { + db := startupMemoryFixture(b, 6000, 65, false) + for _, limit := range []int{1, 5000} { + b.Run(fmt.Sprintf("page-%d", limit), func(b *testing.B) { + q := PacketQuery{Since: time.Now().UTC().Add(-24 * time.Hour).Format(time.RFC3339), Limit: limit} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + result, err := db.QueryGroupedPackets(q) + if err != nil || result.Total != 6000 || len(result.Packets) != limit { + b.Fatalf("history query: %v / %v", result, err) + } + } + }) + } +} diff --git a/cmd/server/hash_migrate.go b/cmd/server/hash_migrate.go index a85d58bc1..1dd52090c 100644 --- a/cmd/server/hash_migrate.go +++ b/cmd/server/hash_migrate.go @@ -5,132 +5,34 @@ import ( "time" ) -// migrateContentHashesAsync recomputes content hashes in batches after the -// server is already serving HTTP. Packets whose hash changes are updated in -// both the DB and the in-memory byHash index. The migration is idempotent: -// once all hashes match the current formula it completes instantly. -func migrateContentHashesAsync(store *PacketStore, batchSize int, yieldDuration time.Duration) { - // #1856: every DB failure below continues to the next batch, so the loop - // always reaches the deferred completion. Setting the flag unconditionally - // therefore reported success after migrating nothing, which is exactly what - // happens on the read-only handle the server holds since #1283: begin, - // prepare and commit all fail, every batch is skipped, and /api/stats then - // answers hashMigrationComplete: true. - failedBatches := 0 - defer func() { - if r := recover(); r != nil { - log.Printf("[hash-migrate] panic recovered: %v", r) - failedBatches++ - } - if failedBatches > 0 { - log.Printf("[hash-migrate] INCOMPLETE: %d batch(es) could not be written; "+ - "hashMigrationComplete stays false. On a read-only DB handle this is expected "+ - "and the migration belongs in the ingestor (#1856).", failedBatches) - return - } - store.hashMigrationComplete.Store(true) - }() - - // Snapshot the packet slice length under lock (packets only grow). +// verifyContentHashesAsync reports whether the loaded hashes are current. The +// server never rewrites telemetry or its hash index independently of storage. +// Legacy hash repair is a separate offline operation; import preserves hashes. +func verifyContentHashesAsync(store *PacketStore, batchSize int, yieldDuration time.Duration) { + if batchSize <= 0 { + batchSize = 5000 + } store.mu.RLock() total := len(store.packets) store.mu.RUnlock() - - migrated := 0 for offset := 0; offset < total; offset += batchSize { - end := offset + batchSize - if end > total { - end = total - } - - // Collect stale hashes in this batch under RLock. - type hashUpdate struct { - tx *StoreTx - oldHash string - newHash string - } - var updates []hashUpdate - store.mu.RLock() - for _, tx := range store.packets[offset:end] { - if tx.RawHex == "" { - continue - } - newHash := ComputeContentHash(tx.RawHex) - if newHash != tx.Hash { - updates = append(updates, hashUpdate{tx: tx, oldHash: tx.Hash, newHash: newHash}) + end := min(offset+batchSize, min(total, len(store.packets))) + stale := false + for _, tx := range store.packets[min(offset, end):end] { + if tx.RawHex != "" && ComputeContentHash(tx.RawHex) != tx.Hash { + stale = true + break } } store.mu.RUnlock() - - if len(updates) == 0 { - continue - } - - // Write batch to DB in a single transaction. - dbTx, err := store.db.conn.Begin() - if err != nil { - log.Printf("[hash-migrate] begin tx: %v", err) - failedBatches++ - continue - } - stmt, err := dbTx.Prepare("UPDATE transmissions SET hash = ? WHERE id = ?") - if err != nil { - log.Printf("[hash-migrate] prepare: %v", err) - dbTx.Rollback() - failedBatches++ - continue - } - - for _, u := range updates { - if _, err := stmt.Exec(u.newHash, u.tx.ID); err != nil { - // UNIQUE constraint = two old hashes map to the same new hash (duplicate). - // Merge observations to the surviving tx, delete the duplicate. - log.Printf("[hash-migrate] tx %d collides — merging duplicate", u.tx.ID) - var survID int - if err2 := dbTx.QueryRow("SELECT id FROM transmissions WHERE hash = ?", u.newHash).Scan(&survID); err2 == nil { - dbTx.Exec("UPDATE observations SET transmission_id = ? WHERE transmission_id = ?", survID, u.tx.ID) - dbTx.Exec("DELETE FROM transmissions WHERE id = ?", u.tx.ID) - u.newHash = "" // mark for in-memory removal only - } - } + if stale { + log.Print("[hash-verify] stale content hashes found; perform explicit offline hash repair on a working copy of the legacy snapshot before import. The importer preserves existing hashes") + return } - stmt.Close() - - if err := dbTx.Commit(); err != nil { - log.Printf("[hash-migrate] commit: %v", err) - failedBatches++ - continue + if yieldDuration > 0 { + time.Sleep(yieldDuration) } - - // Update in-memory index under write lock. - store.mu.Lock() - for _, u := range updates { - delete(store.byHash, u.oldHash) - if u.newHash == "" { - // Merged duplicate — remove from packets slice and indexes. - delete(store.byTxID, u.tx.ID) - // Move observations to survivor if present. - if surv := store.byHash[ComputeContentHash(u.tx.RawHex)]; surv != nil { - for _, obs := range u.tx.Observations { - surv.Observations = append(surv.Observations, obs) - surv.ObservationCount++ - } - } - } else { - u.tx.Hash = u.newHash - store.byHash[u.newHash] = u.tx - } - } - store.mu.Unlock() - - migrated += len(updates) - - // Yield to let HTTP handlers run. - time.Sleep(yieldDuration) - } - - if migrated > 0 { - log.Printf("[hash-migrate] Migrated %d content hashes to new formula", migrated) } + store.hashMigrationComplete.Store(true) } diff --git a/cmd/server/hash_migrate_test.go b/cmd/server/hash_migrate_test.go index 0cc7f21e6..3df5c46f1 100644 --- a/cmd/server/hash_migrate_test.go +++ b/cmd/server/hash_migrate_test.go @@ -5,7 +5,7 @@ import ( "time" ) -func TestMigrateContentHashesAsync(t *testing.T) { +func TestVerifyContentHashesDoesNotWrite(t *testing.T) { db := setupTestDBv2(t) store := NewPacketStore(db, nil) @@ -14,8 +14,9 @@ func TestMigrateContentHashesAsync(t *testing.T) { correctHash := ComputeContentHash(rawHex) wrongHash := "deadbeef12345678" - _, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type) - VALUES (?, ?, datetime('now'), 0, 2)`, rawHex, wrongHash) + _, err := db.conn.Exec(testNativeSQL(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type) + VALUES ($1, $2, strftime('%Y-%m-%dT%H:%M:%SZ','now'), 0, 2)`, `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type) + VALUES ($1, $2, to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"'), 0, 2)`), rawHex, wrongHash) if err != nil { t.Fatal(err) } @@ -28,25 +29,25 @@ func TestMigrateContentHashesAsync(t *testing.T) { t.Fatal("expected packet under wrong hash before migration") } - migrateContentHashesAsync(store, 100, time.Millisecond) + verifyContentHashesAsync(store, 100, time.Millisecond) - if !store.hashMigrationComplete.Load() { - t.Error("expected hashMigrationComplete to be true") + if store.hashMigrationComplete.Load() { + t.Error("stale hashes must not report migration complete") } - if store.byHash[wrongHash] != nil { - t.Error("old hash should be removed from index") + if store.byHash[wrongHash] == nil { + t.Error("read-only verification must preserve the old hash index") } - if store.byHash[correctHash] == nil { - t.Error("new hash should be in index") + if store.byHash[correctHash] != nil { + t.Error("verification must not invent a new hash") } var dbHash string - err = db.conn.QueryRow("SELECT hash FROM transmissions WHERE raw_hex = ?", rawHex).Scan(&dbHash) + err = db.conn.QueryRow("SELECT hash FROM transmissions WHERE raw_hex = $1", rawHex).Scan(&dbHash) if err != nil { t.Fatal(err) } - if dbHash != correctHash { - t.Errorf("DB hash = %s, want %s", dbHash, correctHash) + if dbHash != wrongHash { + t.Errorf("DB hash = %s, want unchanged %s", dbHash, wrongHash) } } @@ -57,8 +58,9 @@ func TestMigrateContentHashesAsync_NoOp(t *testing.T) { rawHex := "0A00D69FD7A5A7475DB07337749AE61FA53A4788E976" correctHash := ComputeContentHash(rawHex) - _, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type) - VALUES (?, ?, datetime('now'), 0, 2)`, rawHex, correctHash) + _, err := db.conn.Exec(testNativeSQL(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type) + VALUES ($1, $2, strftime('%Y-%m-%dT%H:%M:%SZ','now'), 0, 2)`, `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type) + VALUES ($1, $2, to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"'), 0, 2)`), rawHex, correctHash) if err != nil { t.Fatal(err) } @@ -67,7 +69,7 @@ func TestMigrateContentHashesAsync_NoOp(t *testing.T) { t.Fatal(err) } - migrateContentHashesAsync(store, 100, time.Millisecond) + verifyContentHashesAsync(store, 100, time.Millisecond) if !store.hashMigrationComplete.Load() { t.Error("expected hashMigrationComplete to be true") @@ -93,8 +95,9 @@ func TestMigrateContentHashesAsyncDoesNotClaimCompletionWhenWritesFail(t *testin rawHex := "0A00D69FD7A5A7475DB07337749AE61FA53A4788E976" wrongHash := "deadbeef12345678" - if _, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type) - VALUES (?, ?, datetime('now'), 0, 2)`, rawHex, wrongHash); err != nil { + if _, err := db.conn.Exec(testNativeSQL(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type) + VALUES ($1, $2, strftime('%Y-%m-%dT%H:%M:%SZ','now'), 0, 2)`, `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type) + VALUES ($1, $2, to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"'), 0, 2)`), rawHex, wrongHash); err != nil { t.Fatal(err) } if err := store.Load(); err != nil { @@ -109,7 +112,7 @@ func TestMigrateContentHashesAsyncDoesNotClaimCompletionWhenWritesFail(t *testin t.Fatal(err) } - migrateContentHashesAsync(store, 100, time.Millisecond) + verifyContentHashesAsync(store, 100, time.Millisecond) if store.hashMigrationComplete.Load() { t.Error("hashMigrationComplete must stay false when no batch could be written; " + diff --git a/cmd/server/hexgrid.go b/cmd/server/hexgrid.go index 93c25f3b7..689661d8c 100644 --- a/cmd/server/hexgrid.go +++ b/cmd/server/hexgrid.go @@ -8,7 +8,9 @@ import ( ) // Pure-Go hexagonal binning for RX coverage display. We deliberately avoid the -// CGO-based uber/h3-go (this project builds with CGO_ENABLED=0). Points are +// CGO-based uber/h3-go. (That predates the SQLite driver move to cgo; this +// stays pure Go because it needs no C library, not because cgo is unavailable.) +// Points are // projected to Web Mercator and snapped to a pointy-top hex grid whose size // depends on the display resolution. Cell ids are "res:q:r" (axial coords). // At city/region scale this looks like H3/mapme.sh coverage without any deps. diff --git a/cmd/server/hidden_name_prefix_1181_endpoints_test.go b/cmd/server/hidden_name_prefix_1181_endpoints_test.go index 8b30bb020..dab2224b6 100644 --- a/cmd/server/hidden_name_prefix_1181_endpoints_test.go +++ b/cmd/server/hidden_name_prefix_1181_endpoints_test.go @@ -23,7 +23,7 @@ func TestHiddenNamePrefix_1181_NodeHealth(t *testing.T) { pk := "deadbeef00001184" if _, err := srv.db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, ?, ?, 0, 0, '2026-06-01T00:00:00Z', '2026-06-01T00:00:00Z', 1)`, + VALUES ($1, $2, $3, 0, 0, '2026-06-01T00:00:00Z', '2026-06-01T00:00:00Z', 1)`, pk, "🚫 health me", "companion"); err != nil { t.Fatalf("insert: %v", err) } @@ -57,7 +57,7 @@ func TestHiddenNamePrefix_1181_BulkHealth(t *testing.T) { pk := "deadbeef00001185" if _, err := srv.db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, ?, ?, 0, 0, '2026-06-01T00:00:00Z', '2026-06-01T00:00:00Z', 1)`, + VALUES ($1, $2, $3, 0, 0, '2026-06-01T00:00:00Z', '2026-06-01T00:00:00Z', 1)`, pk, "🚫 bulk me", "companion"); err != nil { t.Fatalf("insert: %v", err) } @@ -91,7 +91,7 @@ func TestHiddenNamePrefix_1181_Paths(t *testing.T) { pk := "deadbeef00001186" if _, err := srv.db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, ?, ?, 0, 0, '2026-06-01T00:00:00Z', '2026-06-01T00:00:00Z', 1)`, + VALUES ($1, $2, $3, 0, 0, '2026-06-01T00:00:00Z', '2026-06-01T00:00:00Z', 1)`, pk, "🚫 paths me", "companion"); err != nil { t.Fatalf("insert: %v", err) } @@ -113,7 +113,7 @@ func TestHiddenNamePrefix_1181_Analytics(t *testing.T) { pk := "deadbeef00001187" if _, err := srv.db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, ?, ?, 0, 0, '2026-06-01T00:00:00Z', '2026-06-01T00:00:00Z', 1)`, + VALUES ($1, $2, $3, 0, 0, '2026-06-01T00:00:00Z', '2026-06-01T00:00:00Z', 1)`, pk, "🚫 analytics me", "companion"); err != nil { t.Fatalf("insert: %v", err) } diff --git a/cmd/server/hidden_name_prefix_1181_test.go b/cmd/server/hidden_name_prefix_1181_test.go index d449fcae6..9dee2da4a 100644 --- a/cmd/server/hidden_name_prefix_1181_test.go +++ b/cmd/server/hidden_name_prefix_1181_test.go @@ -19,7 +19,7 @@ func TestHiddenNamePrefix_1181_NodesList(t *testing.T) { // Insert a node whose name starts with the configured 🚫 prefix. _, err := srv.db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, ?, ?, 0, 0, '2026-06-01T00:00:00Z', '2026-06-01T00:00:00Z', 1)`, + VALUES ($1, $2, $3, 0, 0, '2026-06-01T00:00:00Z', '2026-06-01T00:00:00Z', 1)`, "deadbeef00001181", "🚫 ban me", "companion") if err != nil { t.Fatalf("insert hidden node: %v", err) @@ -70,7 +70,7 @@ func TestHiddenNamePrefix_1181_Search(t *testing.T) { if _, err := srv.db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, ?, ?, 0, 0, '2026-06-01T00:00:00Z', '2026-06-01T00:00:00Z', 1)`, + VALUES ($1, $2, $3, 0, 0, '2026-06-01T00:00:00Z', '2026-06-01T00:00:00Z', 1)`, "deadbeef00001182", "🚫 search me", "companion"); err != nil { t.Fatalf("insert: %v", err) } @@ -105,7 +105,7 @@ func TestHiddenNamePrefix_1181_Detail(t *testing.T) { pk := "deadbeef00001183" if _, err := srv.db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, ?, ?, 0, 0, '2026-06-01T00:00:00Z', '2026-06-01T00:00:00Z', 1)`, + VALUES ($1, $2, $3, 0, 0, '2026-06-01T00:00:00Z', '2026-06-01T00:00:00Z', 1)`, pk, "🚫 detail me", "companion"); err != nil { t.Fatalf("insert: %v", err) } diff --git a/cmd/server/hop_disambig_e2e_test.go b/cmd/server/hop_disambig_e2e_test.go index 41ad1f127..4e993f9c3 100644 --- a/cmd/server/hop_disambig_e2e_test.go +++ b/cmd/server/hop_disambig_e2e_test.go @@ -76,7 +76,7 @@ func t1201InsertNode(t *testing.T, db *DB, n t1201Node) { // "Berlin would win tier-3" premise of this fixture weakens silently — // update both this insert and the candidate scoring assertions. _, err := db.conn.Exec( - `INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) VALUES (?, ?, 'repeater', ?, ?, ?, '2026-01-01T00:00:00Z', ?)`, + `INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) VALUES ($1, $2, 'repeater', $3, $4, $5, '2026-01-01T00:00:00Z', $6)`, n.pk, "node-"+n.pk[:4], n.lat, n.lon, "2026-05-01T00:00:00Z", n.obsCount, ) if err != nil { @@ -109,7 +109,7 @@ func TestTopHopsRespectsContextAcrossAllCallSites(t *testing.T) { // Insert observer row (referenced by observations via observer_idx). if _, err := db.conn.Exec( - `INSERT INTO observers (id, name, last_seen, first_seen, packet_count) VALUES (?, ?, ?, '2026-01-01T00:00:00Z', 100)`, + `INSERT INTO observers (id, name, last_seen, first_seen, packet_count) VALUES ($1, $2, $3, '2026-01-01T00:00:00Z', 100)`, t1201Observer, "obs-ca", "2026-05-01T00:00:00Z", ); err != nil { t.Fatal(err) @@ -128,17 +128,18 @@ func TestTopHopsRespectsContextAcrossAllCallSites(t *testing.T) { for i := 0; i < 50; i++ { ts := baseTime.Add(time.Duration(i) * time.Minute).Format(time.RFC3339) hash := fmt.Sprintf("hash1201_%03d", i) - res, err := tx.Exec( - `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) VALUES (?, ?, ?, 1, 1, ?)`, + var txID int64 + err = tx.QueryRow( + `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) VALUES ($1, $2, $3, 1, 1, $4) RETURNING id`, "AA", hash, ts, string(decoded), - ) + ).Scan(&txID) if err != nil { _ = tx.Rollback() t.Fatal(err) } - txID, _ := res.LastInsertId() + if _, err := tx.Exec( - `INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) VALUES (?, 1, 12.0, -90, ?, ?)`, + `INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) VALUES ($1, 1, 12.0, -90, $2, $3)`, txID, pathJSON, baseTime.Add(time.Duration(i)*time.Minute).Unix(), ); err != nil { _ = tx.Rollback() diff --git a/cmd/server/hot_startup_consistency_test.go b/cmd/server/hot_startup_consistency_test.go index 709333993..50929d6c5 100644 --- a/cmd/server/hot_startup_consistency_test.go +++ b/cmd/server/hot_startup_consistency_test.go @@ -26,7 +26,7 @@ func TestHotStartup_loadChunk_IndexSliceConsistency(t *testing.T) { // spans 3 batches, widening the inconsistency window for the reader. dbPath := createTestDBWithAgedPackets(t, 10, 1200) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } diff --git a/cmd/server/hot_startup_test.go b/cmd/server/hot_startup_test.go index d848da8e3..362344835 100644 --- a/cmd/server/hot_startup_test.go +++ b/cmd/server/hot_startup_test.go @@ -1,19 +1,16 @@ package main import ( - "database/sql" "encoding/json" "fmt" "net/http" "net/http/httptest" - "path/filepath" "sync" "sync/atomic" "testing" "time" "github.com/gorilla/mux" - _ "modernc.org/sqlite" ) // createTestDBMultiDay creates a test DB with packets spread across numDays days. @@ -21,10 +18,9 @@ import ( // Packets within each day are spaced 1 minute apart. func createTestDBMultiDay(t *testing.T, numDays, txPerDay int) string { t.Helper() - dir := t.TempDir() - dbPath := filepath.Join(dir, "test.db") + dbPath := postgresTestDSN(t) - conn, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL") + conn, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } @@ -35,33 +31,25 @@ func createTestDBMultiDay(t *testing.T, numDays, txPerDay int) string { t.Fatalf("createTestDBMultiDay setup: %v", err) } } - execOrFail(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT)`) - execOrFail(`CREATE TABLE observations (id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, direction TEXT, snr REAL, rssi REAL, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT)`) - execOrFail(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT)`) - execOrFail(`CREATE TABLE nodes (pubkey TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, frequency REAL)`) + execOrFail(testNativeSQL(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY AUTOINCREMENT, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT)`, `CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT)`)) + execOrFail(testNativeSQL(`CREATE TABLE observations (id INTEGER PRIMARY KEY AUTOINCREMENT, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT)`, `CREATE TABLE observations (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT)`)) + execOrFail(testNativeSQL(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY AUTOINCREMENT, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, `CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`)) + execOrFail(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, first_seen TEXT, frequency DOUBLE PRECISION)`) execOrFail(`CREATE TABLE schema_version (version INTEGER)`) execOrFail(`INSERT INTO schema_version (version) VALUES (1)`) execOrFail(`CREATE INDEX idx_tx_first_seen ON transmissions(first_seen)`) - id := 1 now := time.Now().UTC() - for day := numDays; day >= 1; day-- { - // Offset by +30 minutes so day boundaries don't coincide exactly with - // hotStartupHours/retentionHours cutoffs, preventing timing-boundary flakiness. - // E.g. for numDays=3: day3 starts at now-71.5h, day2 at now-47.5h, day1 at now-23.5h. - base := now.Add(-time.Duration(day)*24*time.Hour + 30*time.Minute) - for i := 0; i < txPerDay; i++ { - ts := base.Add(time.Duration(i) * time.Minute).Format(time.RFC3339) - hash := fmt.Sprintf("hash%06d", id) - if _, err := conn.Exec("INSERT INTO transmissions VALUES (?,?,?,?,0,4,1,?)", id, "aa", hash, ts, `{}`); err != nil { - t.Fatalf("createTestDBMultiDay insert tx: %v", err) - } - if _, err := conn.Exec("INSERT INTO observations VALUES (?,?,?,?,?,?,?,?,?,?,?)", id, id, "obs1", "Obs1", "RX", -10.0, -80.0, 5, `[]`, ts, ""); err != nil { - t.Fatalf("createTestDBMultiDay insert obs: %v", err) - } - id++ - } + stamp := func(n int) string { + day, offset := numDays-n/txPerDay, n%txPerDay + return now.Add(-time.Duration(day)*24*time.Hour + 30*time.Minute + time.Duration(offset)*time.Minute).Format(time.RFC3339) } + copyTestRows(t, conn, "transmissions", []string{"id", "raw_hex", "hash", "first_seen", "route_type", "payload_type", "payload_version", "decoded_json"}, numDays*txPerDay, func(n int) []any { + return []any{n + 1, "aa", fmt.Sprintf("hash%06d", n+1), stamp(n), 0, 4, 1, "{}"} + }) + copyTestRows(t, conn, "observations", []string{"id", "transmission_id", "observer_id", "observer_name", "direction", "snr", "rssi", "score", "path_json", "timestamp", "raw_hex"}, numDays*txPerDay, func(n int) []any { + return []any{n + 1, n + 1, "obs1", "Obs1", "RX", -10.0, -80.0, 5, "[]", stamp(n), ""} + }) return dbPath } @@ -81,7 +69,7 @@ func waitForBackgroundLoad(t *testing.T, store *PacketStore, timeout time.Durati func TestHotStartupConfig_Clamp(t *testing.T) { dbPath := createTestDB(t, 10) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } @@ -100,7 +88,7 @@ func TestHotStartupConfig_Clamp(t *testing.T) { func TestHotStartupConfig_ZeroIsDisabled(t *testing.T) { dbPath := createTestDB(t, 10) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } @@ -119,7 +107,7 @@ func TestHotStartup_LoadsOnlyHotWindow(t *testing.T) { // 50 old packets (48h ago), 10 recent (30min ago) dbPath := createTestDBWithAgedPackets(t, 10, 50) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } @@ -156,7 +144,7 @@ func TestHotStartup_DisabledWhenZero(t *testing.T) { // 50 old (48h ago), 10 recent (30min ago) — all within 72h retention dbPath := createTestDBWithAgedPackets(t, 10, 50) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } @@ -180,7 +168,7 @@ func TestHotStartup_loadChunk_AddsOlderData(t *testing.T) { // 50 old packets (48h ago), 10 recent (30min ago) dbPath := createTestDBWithAgedPackets(t, 10, 50) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } @@ -229,7 +217,7 @@ func TestHotStartup_BackgroundFillsToRetention(t *testing.T) { // 3 days × 50 tx/day = 150 total dbPath := createTestDBMultiDay(t, 3, 50) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } @@ -269,7 +257,7 @@ func TestHotStartup_BackgroundFillsToRetention(t *testing.T) { func TestHotStartup_ChunkErrorRecovery(t *testing.T) { dbPath := createTestDBWithAgedPackets(t, 10, 50) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } @@ -314,7 +302,7 @@ func TestHotStartup_SQLFallback_TriggeredForOldDate(t *testing.T) { // 50 old packets (48h ago), 10 recent (30min ago) dbPath := createTestDBWithAgedPackets(t, 10, 50) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } @@ -345,7 +333,7 @@ func TestHotStartup_SQLFallback_TriggeredForOldDate(t *testing.T) { func TestHotStartup_PerfStats(t *testing.T) { dbPath := createTestDBWithAgedPackets(t, 10, 50) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } @@ -376,7 +364,7 @@ func TestHotStartup_SQLFallback_NotTriggeredForRecentDate(t *testing.T) { // 50 old packets (48h ago), 10 recent (30min ago) dbPath := createTestDBWithAgedPackets(t, 10, 50) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } @@ -405,7 +393,7 @@ func TestHotStartup_SQLFallback_Until(t *testing.T) { // 50 old packets (48h ago), 10 recent (30min ago) dbPath := createTestDBWithAgedPackets(t, 10, 50) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } @@ -436,7 +424,7 @@ func TestHotStartup_SQLFallback_Until(t *testing.T) { func TestHotStartup_PerfStoreHTTP(t *testing.T) { dbPath := createTestDBWithAgedPackets(t, 10, 50) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } @@ -485,7 +473,7 @@ func TestHotStartup_ConcurrentQueryDuringBackgroundLoad(t *testing.T) { // large enough to give pollers >=1 query during the background fill. dbPath := createTestDBMultiDay(t, 5, 200) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } @@ -583,7 +571,7 @@ func (a *atomicSamples) get() int64 { func TestHotStartup_BackgroundLoadFailureSurfacesInPerf(t *testing.T) { dbPath := createTestDBMultiDay(t, 3, 50) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } diff --git a/cmd/server/index_ready_1008_test.go b/cmd/server/index_ready_1008_test.go index f7c6340f8..96f520c62 100644 --- a/cmd/server/index_ready_1008_test.go +++ b/cmd/server/index_ready_1008_test.go @@ -58,8 +58,12 @@ func TestIssue1008_HandlerReturns503WhileSubpathIndexLoading(t *testing.T) { if err := store.Load(); err != nil { t.Fatalf("Load() error: %v", err) } - // Don't wait for the background build — we want to observe the - // not-ready window. + // Finish the background build before simulating not-ready, so it cannot + // flip the flag back to true before the handler checks it. + if !store.WaitIndexesReady(5 * time.Second) { + t.Fatal("background builds never finished") + } + store.subpathReady.Store(false) cfg := &Config{} cfg.applyListLimitsDefaults() srv := &Server{store: store, cfg: cfg} diff --git a/cmd/server/issue1189_distinct_iatas_test.go b/cmd/server/issue1189_distinct_iatas_test.go index 3a096fe95..2bf397336 100644 --- a/cmd/server/issue1189_distinct_iatas_test.go +++ b/cmd/server/issue1189_distinct_iatas_test.go @@ -27,22 +27,22 @@ func TestQueryGroupedPacketsReturnsDistinctIATAs(t *testing.T) { // Observers: SJC + SFO + a third with no IATA (should be excluded). db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obsA', 'A', 'SJC', ?, '2026-01-01T00:00:00Z', 10)`, now.Format(time.RFC3339)) + VALUES ('obsA', 'A', 'SJC', $1, '2026-01-01T00:00:00Z', 10)`, now.Format(time.RFC3339)) db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obsB', 'B', 'SFO', ?, '2026-01-01T00:00:00Z', 10)`, now.Format(time.RFC3339)) + VALUES ('obsB', 'B', 'SFO', $1, '2026-01-01T00:00:00Z', 10)`, now.Format(time.RFC3339)) db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obsC', 'C', '', ?, '2026-01-01T00:00:00Z', 10)`, now.Format(time.RFC3339)) + VALUES ('obsC', 'C', '', $1, '2026-01-01T00:00:00Z', 10)`, now.Format(time.RFC3339)) // One transmission with 3 observations (SJC, SFO, no-IATA). db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('AABB', 'deadbeefcafef00d', ?, 1, 4, '{}')`, now.Format(time.RFC3339)) + VALUES ('AABB', 'deadbeefcafef00d', $1, 1, 4, '{}')`, now.Format(time.RFC3339)) // v3 schema: observer_idx = observers.rowid (auto-assigned 1,2,3 in insert order). db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 12.0, -80, '["aa"]', ?)`, recentEpoch) + VALUES (1, 1, 12.0, -80, '["aa"]', $1)`, recentEpoch) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 2, 8.0, -90, '["aa"]', ?)`, recentEpoch-30) + VALUES (1, 2, 8.0, -90, '["aa"]', $1)`, recentEpoch-30) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 3, 5.0, -95, '["aa"]', ?)`, recentEpoch-60) + VALUES (1, 3, 5.0, -95, '["aa"]', $1)`, recentEpoch-60) result, err := db.QueryGroupedPackets(PacketQuery{Limit: 50}) if err != nil { @@ -79,11 +79,11 @@ func TestQueryGroupedPacketsDistinctIATAsEmptyWhenNoIATA(t *testing.T) { recentEpoch := now.Add(-1 * time.Hour).Unix() db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obsX', 'X', '', ?, '2026-01-01T00:00:00Z', 1)`, now.Format(time.RFC3339)) + VALUES ('obsX', 'X', '', $1, '2026-01-01T00:00:00Z', 1)`, now.Format(time.RFC3339)) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('AA', '1111222233334444', ?, 1, 4, '{}')`, now.Format(time.RFC3339)) + VALUES ('AA', '1111222233334444', $1, 1, 4, '{}')`, now.Format(time.RFC3339)) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 10.0, -85, '[]', ?)`, recentEpoch) + VALUES (1, 1, 10.0, -85, '[]', $1)`, recentEpoch) result, err := db.QueryGroupedPackets(PacketQuery{Limit: 50}) if err != nil { diff --git a/cmd/server/issue1690_cold_load_test.go b/cmd/server/issue1690_cold_load_test.go index 2b1f85002..9fc02bacd 100644 --- a/cmd/server/issue1690_cold_load_test.go +++ b/cmd/server/issue1690_cold_load_test.go @@ -14,12 +14,8 @@ package main import ( "encoding/json" - "path/filepath" - "strings" "testing" "time" - - _ "modernc.org/sqlite" ) // createTestDBWithLastSeen seeds a DB with the post-fix schema (last_seen @@ -49,15 +45,14 @@ func createTestDBWithLastSeen(t *testing.T, dbPath string, numTx, obsPerTx int, // loads zero rows; the post-fix code (filtering on last_seen) must load // all 1000. func Test1690_ColdLoad_TimeAxis(t *testing.T) { - dir := t.TempDir() - dbPath := filepath.Join(dir, "test.db") + dbPath := postgresTestDSN(t) nowSec := time.Now().UTC().Unix() createTestDBWithLastSeen(t, dbPath, 1000, 1, nowSec, 30*24*time.Hour, // first_seen = 30d ago 30*time.Minute) // last_seen = 30min ago - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatalf("OpenDB: %v", err) } @@ -81,13 +76,10 @@ func Test1690_ColdLoad_TimeAxis(t *testing.T) { } } -// Test1690_BackgroundLoadHonesty seeds 1000 transmissions but caps the -// store's memory budget so it can only fit a fraction. After -// loadBackgroundChunks runs, backgroundLoadDone must be FALSE and -// backgroundLoadFailed must be TRUE because actual coverage is < 90%. +// A configured memory cap is successful bounded loading, not a failed startup. +// Keep reporting actual database coverage rather than pretending it is 100%. func Test1690_BackgroundLoadHonesty(t *testing.T) { - dir := t.TempDir() - dbPath := filepath.Join(dir, "test.db") + dbPath := postgresTestDSN(t) nowSec := time.Now().UTC().Unix() // 5000 rows; chunkSize=500 + maxMemoryMB=1 (→ maxPackets ≈ 1000) so @@ -96,7 +88,7 @@ func Test1690_BackgroundLoadHonesty(t *testing.T) { createTestDBWithLastSeen(t, dbPath, 5000, 1, nowSec, 30*time.Minute, 30*time.Minute) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatalf("OpenDB: %v", err) } @@ -112,34 +104,27 @@ func Test1690_BackgroundLoadHonesty(t *testing.T) { } store.loadBackgroundChunks() - if store.backgroundLoadDone.Load() { - t.Errorf("backgroundLoadDone=true with only %d/5000 packets loaded; "+ - "must be false until coverage ≥ 90%%", len(store.packets)) + if !store.backgroundLoadDone.Load() { + t.Error("memory-bounded background load must reach a terminal state") } - if !store.backgroundLoadFailed.Load() { - t.Errorf("backgroundLoadFailed=false despite under-coverage "+ - "(%d/5000 packets loaded); must be true with a reason", len(store.packets)) + if store.backgroundLoadFailed.Load() || store.BackgroundLoadError() != "" { + t.Errorf("configured memory budget treated as failure: %s", store.BackgroundLoadError()) } - // The error message must mention a percentage so operators can see - // the actual ratio surface in the perf endpoint. - errMsg := store.BackgroundLoadError() - if !strings.Contains(errMsg, "%") { - t.Errorf("backgroundLoadError=%q; expected human-readable ratio "+ - "(e.g. 'loaded X%% of Y rows')", errMsg) + if got, want := store.LoadCoverageRatio(), float64(len(store.packets))/5000; got != want || got >= 0.9 { + t.Errorf("coverage = %g, want actual bounded fraction %g", got, want) } } // Test1690_PerfStats_NewFields asserts the typed perf payload exposes the // retention/coverage fields needed for prod observability. func Test1690_PerfStats_NewFields(t *testing.T) { - dir := t.TempDir() - dbPath := filepath.Join(dir, "test.db") + dbPath := postgresTestDSN(t) nowSec := time.Now().UTC().Unix() createTestDBWithLastSeen(t, dbPath, 10, 1, nowSec, 30*time.Minute, 30*time.Minute) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatalf("OpenDB: %v", err) } diff --git a/cmd/server/issue1809_bg_load_race_test.go b/cmd/server/issue1809_bg_load_race_test.go index 14ba1850f..df799c824 100644 --- a/cmd/server/issue1809_bg_load_race_test.go +++ b/cmd/server/issue1809_bg_load_race_test.go @@ -21,9 +21,7 @@ package main // kept intact; we add the coverage assertions on top. import ( - "database/sql" "fmt" - "path/filepath" "testing" "time" ) @@ -34,8 +32,7 @@ import ( // produces backgroundLoadFailed=true deterministically because the bg // loader reads oldestLoaded="" and bails. func Test1809_StartupLoad_BgLoaderSeesOldestLoaded(t *testing.T) { - dir := t.TempDir() - dbPath := filepath.Join(dir, "test.db") + dbPath := postgresTestDSN(t) nowSec := time.Now().UTC().Unix() // Seed 100 rows spread over 14 days with last_seen also spread, so @@ -47,7 +44,7 @@ func Test1809_StartupLoad_BgLoaderSeesOldestLoaded(t *testing.T) { const spanDays = 14 createTestDBSpreadOverDays(t, dbPath, totalRows, spanDays, nowSec) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatalf("OpenDB: %v", err) } @@ -138,7 +135,7 @@ func createTestDBSpreadOverDays(t *testing.T, dbPath string, numTx, spanDays int // (adv #11: prior code carried a duplicated annotation on every line)." func seedTestDBRows(t *testing.T, dbPath string, numTx, obsPerTx int, rowTimes func(i int) (string, int64)) { t.Helper() - conn, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL") + conn, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } @@ -150,23 +147,33 @@ func seedTestDBRows(t *testing.T, dbPath string, numTx, obsPerTx int, rowTimes f } } // PREFLIGHT: async=true reason="unit-test fixture seeder; in-memory ephemeral SQLite; all CREATE/INSERT below covered by this block annotation (pr-preflight 25-line lookback window)" - execOrFail(`CREATE TABLE transmissions ( - id INTEGER PRIMARY KEY, + execOrFail(testNativeSQL(`CREATE TABLE transmissions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT, last_seen INTEGER NOT NULL DEFAULT 0 - )`) + )`, `CREATE TABLE transmissions ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + raw_hex TEXT, hash TEXT, first_seen TEXT, + route_type INTEGER, payload_type INTEGER, + payload_version INTEGER, decoded_json TEXT, + last_seen INTEGER NOT NULL DEFAULT 0 + )`)) // PREFLIGHT: async=true reason="unit-test fixture seeder" - execOrFail(`CREATE TABLE observations ( - id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, - direction TEXT, snr REAL, rssi REAL, score INTEGER, + execOrFail(testNativeSQL(`CREATE TABLE observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT - )`) + )`, `CREATE TABLE observations ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, + path_json TEXT, timestamp TEXT, raw_hex TEXT + )`)) // PREFLIGHT: async=true reason="unit-test fixture seeder" - execOrFail(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT)`) + execOrFail(testNativeSQL(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY AUTOINCREMENT, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, `CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`)) // PREFLIGHT: async=true reason="unit-test fixture seeder" - execOrFail(`CREATE TABLE nodes (pubkey TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, frequency REAL)`) + execOrFail(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, first_seen TEXT, frequency DOUBLE PRECISION)`) // PREFLIGHT: async=true reason="unit-test fixture seeder" execOrFail(`CREATE TABLE schema_version (version INTEGER)`) execOrFail(`INSERT INTO schema_version (version) VALUES (1)`) @@ -175,30 +182,15 @@ func seedTestDBRows(t *testing.T, dbPath string, numTx, obsPerTx int, rowTimes f // PREFLIGHT: async=true reason="unit-test fixture seeder" execOrFail(`CREATE INDEX idx_tx_last_seen ON transmissions(last_seen)`) - txStmt, err := conn.Prepare("INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json, last_seen) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)") - if err != nil { - t.Fatalf("prepare tx: %v", err) - } - defer txStmt.Close() - obsStmt, err := conn.Prepare("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") - if err != nil { - t.Fatalf("prepare obs: %v", err) - } - defer obsStmt.Close() - - obsID := 1 - for i := 1; i <= numTx; i++ { - firstSeenStr, lastSeenUnix := rowTimes(i) - hash := fmt.Sprintf("h%06d", i) - if _, err := txStmt.Exec(i, "aabb", hash, firstSeenStr, 0, 4, 1, "{}", lastSeenUnix); err != nil { - t.Fatalf("insert tx %d: %v", i, err) - } - for j := 0; j < obsPerTx; j++ { - obsTs := time.Unix(lastSeenUnix, 0).UTC().Add(-time.Duration(j) * time.Minute).Format(time.RFC3339) - if _, err := obsStmt.Exec(obsID, i, "obs1", "Obs1", "RX", -10.0, -80.0, 5, "[]", obsTs); err != nil { - t.Fatalf("insert obs: %v", err) - } - obsID++ - } - } + copyTestRows(t, conn, "transmissions", []string{"id", "raw_hex", "hash", "first_seen", "route_type", "payload_type", "payload_version", "decoded_json", "last_seen"}, numTx, func(n int) []any { + i := n + 1 + firstSeen, lastSeen := rowTimes(i) + return []any{i, "aabb", fmt.Sprintf("h%06d", i), firstSeen, 0, 4, 1, "{}", lastSeen} + }) + copyTestRows(t, conn, "observations", []string{"id", "transmission_id", "observer_id", "observer_name", "direction", "snr", "rssi", "score", "path_json", "timestamp"}, numTx*obsPerTx, func(n int) []any { + i, j := n/obsPerTx+1, n%obsPerTx + _, lastSeen := rowTimes(i) + stamp := time.Unix(lastSeen, 0).UTC().Add(-time.Duration(j) * time.Minute).Format(time.RFC3339) + return []any{n + 1, i, "obs1", "Obs1", "RX", -10.0, -80.0, 5, "[]", stamp} + }) } diff --git a/cmd/server/issue673_test.go b/cmd/server/issue673_test.go index b5ac55173..0295270b8 100644 --- a/cmd/server/issue673_test.go +++ b/cmd/server/issue673_test.go @@ -4,8 +4,6 @@ import ( "encoding/json" "testing" "time" - - _ "modernc.org/sqlite" ) const issue673NodePK = "7502f19f44cad6d7b626e1d811c00a914af452636182ccded3fd019803395ec9" @@ -18,7 +16,7 @@ func setupIssue673Store(t *testing.T) (*PacketStore, *DB) { db := setupTestDB(t) _, err := db.conn.Exec( - "INSERT INTO nodes (public_key, name, role) VALUES (?, ?, ?)", + "INSERT INTO nodes (public_key, name, role) VALUES ($1, $2, $3)", issue673NodePK, "Quail Hollow Park", "repeater", ) if err != nil { diff --git a/cmd/server/issue804_repeater_region_test.go b/cmd/server/issue804_repeater_region_test.go index 05e7a7a5c..a6bd86600 100644 --- a/cmd/server/issue804_repeater_region_test.go +++ b/cmd/server/issue804_repeater_region_test.go @@ -34,19 +34,19 @@ func TestIssue804_AnalyticsAttributesByRepeaterRegion(t *testing.T) { // Observers: one in PDX, one in SJC db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obs-pdx', 'Obs PDX', 'PDX', ?, '2026-01-01T00:00:00Z', 100)`, recent) + VALUES ('obs-pdx', 'Obs PDX', 'PDX', $1, '2026-01-01T00:00:00Z', 100)`, recent) db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obs-sjc', 'Obs SJC', 'SJC', ?, '2026-01-01T00:00:00Z', 100)`, recent) + VALUES ('obs-sjc', 'Obs SJC', 'SJC', $1, '2026-01-01T00:00:00Z', 100)`, recent) // PDX-Repeater node (lives in Portland) pdxPK := "pdx0000000000001" db.conn.Exec(`INSERT INTO nodes (public_key, name, role) - VALUES (?, 'PDX-Repeater', 'repeater')`, pdxPK) + VALUES ($1, 'PDX-Repeater', 'repeater')`, pdxPK) // SJC-Repeater node (lives in San Jose) — sanity baseline sjcPK := "sjc0000000000001" db.conn.Exec(`INSERT INTO nodes (public_key, name, role) - VALUES (?, 'SJC-Repeater', 'repeater')`, sjcPK) + VALUES ($1, 'SJC-Repeater', 'repeater')`, sjcPK) pdxDecoded := `{"pubKey":"` + pdxPK + `","name":"PDX-Repeater","type":"ADVERT","flags":{"isRepeater":true}}` sjcDecoded := `{"pubKey":"` + sjcPK + `","name":"SJC-Repeater","type":"ADVERT","flags":{"isRepeater":true}}` @@ -56,31 +56,31 @@ func TestIssue804_AnalyticsAttributesByRepeaterRegion(t *testing.T) { // raw_hex header 0x12 = route_type 2 (direct), payload_type 4 // pathByte 0x40 (hashSize bits=01 → 2, hop_count=0) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('1240aabbccdd', 'pdx_zh_direct', ?, 2, 4, ?)`, recent, pdxDecoded) + VALUES ('1240aabbccdd', 'pdx_zh_direct', $1, 2, 4, $2)`, recent, pdxDecoded) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 12.0, -85, '[]', ?)`, recentEpoch) + VALUES (1, 1, 12.0, -85, '[]', $1)`, recentEpoch) // 2) PDX-Repeater FLOOD advert with hashSize=2 (reliable). // Heard ONLY by obs-SJC via a relay path (this is the polluting case). // raw_hex header 0x11 = route_type 1 (flood), payload_type 4 // pathByte 0x41 (hashSize bits=01 → 2, hop_count=1) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('1141aabbccdd', 'pdx_flood', ?, 1, 4, ?)`, recent, pdxDecoded) + VALUES ('1141aabbccdd', 'pdx_flood', $1, 1, 4, $2)`, recent, pdxDecoded) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (2, 2, 8.0, -95, '["aa11"]', ?)`, recentEpoch) + VALUES (2, 2, 8.0, -95, '["aa11"]', $1)`, recentEpoch) // 3) SJC-Repeater zero-hop DIRECT advert heard only by obs-SJC. // Establishes SJC as the repeater's home region. db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('1240ccddeeff', 'sjc_zh_direct', ?, 2, 4, ?)`, recent, sjcDecoded) + VALUES ('1240ccddeeff', 'sjc_zh_direct', $1, 2, 4, $2)`, recent, sjcDecoded) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (3, 2, 14.0, -82, '[]', ?)`, recentEpoch) + VALUES (3, 2, 14.0, -82, '[]', $1)`, recentEpoch) // 4) SJC-Repeater FLOOD advert with hashSize=2, heard by obs-SJC. db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('1141ccddeeff', 'sjc_flood', ?, 1, 4, ?)`, recent, sjcDecoded) + VALUES ('1141ccddeeff', 'sjc_flood', $1, 1, 4, $2)`, recent, sjcDecoded) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (4, 2, 11.0, -88, '["cc22"]', ?)`, recentEpoch) + VALUES (4, 2, 11.0, -88, '["cc22"]', $1)`, recentEpoch) store := NewPacketStore(db, nil) store.Load() diff --git a/cmd/server/issue810_repro_test.go b/cmd/server/issue810_repro_test.go index c5e6e9a85..d24b3d118 100644 --- a/cmd/server/issue810_repro_test.go +++ b/cmd/server/issue810_repro_test.go @@ -10,7 +10,7 @@ import ( ) // TestRepro810 reproduces #810: when the longest-path observation has NULL -// resolved_path but a shorter-path observation has one, fetchResolvedPathForTxBest +// resolved_path but a shorter-path observation has one, bestResolvedPath // returns nil → /api/nodes/{pk}/health.recentPackets[].resolved_path is missing // while /api/packets shows it. func TestRepro810(t *testing.T) { @@ -18,14 +18,14 @@ func TestRepro810(t *testing.T) { now := time.Now().UTC() recent := now.Add(-1 * time.Hour).Format(time.RFC3339) recentEpoch := now.Add(-1 * time.Hour).Unix() - db.conn.Exec(`INSERT INTO observers (id, name, last_seen, first_seen, packet_count) VALUES ('obs1','O1',?, '2026-01-01T00:00:00Z', 100)`, recent) - db.conn.Exec(`INSERT INTO observers (id, name, last_seen, first_seen, packet_count) VALUES ('obs2','O2',?, '2026-01-01T00:00:00Z', 100)`, recent) - db.conn.Exec(`INSERT INTO nodes (public_key, name, role, last_seen, first_seen, advert_count) VALUES ('aabbccdd11223344','R','repeater',?, '2026-01-01T00:00:00Z', 1)`, recent) - db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) VALUES ('AABB','testhash00000001',?,1,4,'{"pubKey":"aabbccdd11223344","type":"ADVERT"}')`, recent) + db.conn.Exec(`INSERT INTO observers (id, name, last_seen, first_seen, packet_count) VALUES ('obs1','O1',$1, '2026-01-01T00:00:00Z', 100)`, recent) + db.conn.Exec(`INSERT INTO observers (id, name, last_seen, first_seen, packet_count) VALUES ('obs2','O2',$1, '2026-01-01T00:00:00Z', 100)`, recent) + db.conn.Exec(`INSERT INTO nodes (public_key, name, role, last_seen, first_seen, advert_count) VALUES ('aabbccdd11223344','R','repeater',$1, '2026-01-01T00:00:00Z', 1)`, recent) + db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) VALUES ('AABB','testhash00000001',$1,1,4,'{"pubKey":"aabbccdd11223344","type":"ADVERT"}')`, recent) // Longest-path obs WITHOUT resolved_path - db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) VALUES (1,1,12.5,-90,'["aa","bb","cc"]',?)`, recentEpoch) + db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) VALUES (1,1,12.5,-90,'["aa","bb","cc"]',$1)`, recentEpoch) // Shorter-path obs WITH resolved_path - db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp, resolved_path) VALUES (1,2,8.0,-95,'["aa","bb"]',?,'["aabbccdd11223344","eeff00112233aabb"]')`, recentEpoch-100) + db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp, resolved_path) VALUES (1,2,8.0,-95,'["aa","bb"]',$1,'["aabbccdd11223344","eeff00112233aabb"]')`, recentEpoch-100) cfg := &Config{Port: 3000} hub := NewHub() diff --git a/cmd/server/issue871_test.go b/cmd/server/issue871_test.go index 43f2768aa..2539df728 100644 --- a/cmd/server/issue871_test.go +++ b/cmd/server/issue871_test.go @@ -19,7 +19,7 @@ func TestIssue871_NoNullHashOrTimestamp(t *testing.T) { // Insert bad legacy data: packet with empty hash now := time.Now().UTC().Add(-30 * time.Minute).Format(time.RFC3339) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('DEAD', '', ?, 1, 4, '{}')`, now) + VALUES ('DEAD', '', $1, 1, 4, '{}')`, now) // Insert bad legacy data: packet with NULL first_seen (timestamp) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) VALUES ('BEEF', 'aa11bb22cc33dd44', NULL, 1, 4, '{}')`) diff --git a/cmd/server/load_graph_invariant_test.go b/cmd/server/load_graph_invariant_test.go index e128756b7..32cd88320 100644 --- a/cmd/server/load_graph_invariant_test.go +++ b/cmd/server/load_graph_invariant_test.go @@ -1,12 +1,8 @@ package main import ( - "database/sql" - "path/filepath" "testing" "time" - - _ "modernc.org/sqlite" ) // TestLoad_PanicsWhenGraphNotLoadedAndEdgesExist pins the startup-ordering @@ -19,10 +15,9 @@ import ( // Load() must panic when neighbor_edges has rows but s.graph.Load() returns // nil. Fast-fail at startup beats silently-wrong attribution. func TestLoad_PanicsWhenGraphNotLoadedAndEdgesExist(t *testing.T) { - dir := t.TempDir() - dbPath := filepath.Join(dir, "test.db") + dbPath := postgresTestDSN(t) - rw, err := sql.Open("sqlite", "file:"+dbPath+"?_journal_mode=WAL") + rw, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } @@ -35,24 +30,34 @@ func TestLoad_PanicsWhenGraphNotLoadedAndEdgesExist(t *testing.T) { } // Minimal CoreScope schema. PREFLIGHT: async=true reason="test fixture, in-memory tmpdir DB" - exec(`CREATE TABLE transmissions ( - id INTEGER PRIMARY KEY, + exec(testNativeSQL(`CREATE TABLE transmissions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT - )`) + )`, `CREATE TABLE transmissions ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + raw_hex TEXT, hash TEXT, first_seen TEXT, + route_type INTEGER, payload_type INTEGER, payload_version INTEGER, + decoded_json TEXT + )`)) // PREFLIGHT: async=true reason="test fixture, in-memory tmpdir DB" - exec(`CREATE TABLE observations ( - id INTEGER PRIMARY KEY, transmission_id INTEGER, + exec(testNativeSQL(`CREATE TABLE observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, - direction TEXT, snr REAL, rssi REAL, score INTEGER, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT, resolved_path TEXT - )`) + )`, `CREATE TABLE observations ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, + observer_id TEXT, observer_name TEXT, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, + path_json TEXT, timestamp TEXT, raw_hex TEXT, resolved_path TEXT + )`)) // PREFLIGHT: async=true reason="test fixture, in-memory tmpdir DB" - exec(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT)`) + exec(testNativeSQL(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY AUTOINCREMENT, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, `CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`)) // PREFLIGHT: async=true reason="test fixture, in-memory tmpdir DB" exec(`CREATE TABLE nodes ( - public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, + public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, first_seen TEXT, advert_count INTEGER DEFAULT 0 )`) // PREFLIGHT: async=true reason="test fixture, in-memory tmpdir DB" @@ -67,10 +72,10 @@ func TestLoad_PanicsWhenGraphNotLoadedAndEdgesExist(t *testing.T) { PRIMARY KEY (node_a, node_b) )`) now := time.Now().UTC().Format(time.RFC3339) - exec(`INSERT INTO neighbor_edges (node_a, node_b, count, last_seen) VALUES (?, ?, ?, ?)`, + exec(`INSERT INTO neighbor_edges (node_a, node_b, count, last_seen) VALUES ($1, $2, $3, $4)`, "aaa", "bbb", 5, now) - d, err := OpenDB(dbPath) + d, err := openFixtureReader(t, dbPath) if err != nil { t.Fatalf("OpenDB: %v", err) } diff --git a/cmd/server/loadchunk_ambiguous_prefix_test.go b/cmd/server/loadchunk_ambiguous_prefix_test.go index 9a0d4265c..c4ab913cf 100644 --- a/cmd/server/loadchunk_ambiguous_prefix_test.go +++ b/cmd/server/loadchunk_ambiguous_prefix_test.go @@ -1,13 +1,9 @@ package main import ( - "database/sql" "fmt" - "path/filepath" "testing" "time" - - _ "modernc.org/sqlite" ) // createTestDBAmbiguousPrefix builds a fixture where TWO repeaters share the @@ -21,10 +17,9 @@ import ( // picks today's tier-4 winner against ~7-day-old observations. func createTestDBAmbiguousPrefix(t *testing.T, relayA, relayB, hop, firstSeen string) string { t.Helper() - dir := t.TempDir() - dbPath := filepath.Join(dir, "test.db") + dbPath := postgresTestDSN(t) - conn, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL") + conn, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } @@ -37,27 +32,40 @@ func createTestDBAmbiguousPrefix(t *testing.T, relayA, relayB, hop, firstSeen st } // PREFLIGHT: async=true reason="test fixture: in-memory t.TempDir SQLite, never touches a real DB." - exec(`CREATE TABLE transmissions ( - id INTEGER PRIMARY KEY, + exec(testNativeSQL(`CREATE TABLE transmissions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT - )`) + )`, `CREATE TABLE transmissions ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + raw_hex TEXT, hash TEXT, first_seen TEXT, + route_type INTEGER, payload_type INTEGER, payload_version INTEGER, + decoded_json TEXT + )`)) // PREFLIGHT: async=true reason="test fixture, in-memory tmpdir DB" - exec(`CREATE TABLE observations ( - id INTEGER PRIMARY KEY, + exec(testNativeSQL(`CREATE TABLE observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, - direction TEXT, snr REAL, rssi REAL, score INTEGER, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT, resolved_path TEXT - )`) + )`, `CREATE TABLE observations ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + transmission_id INTEGER, + observer_id TEXT, observer_name TEXT, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, + path_json TEXT, timestamp TEXT, + raw_hex TEXT, + resolved_path TEXT + )`)) // PREFLIGHT: async=true reason="test fixture, in-memory tmpdir DB" - exec(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT)`) + exec(testNativeSQL(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY AUTOINCREMENT, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, `CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`)) // PREFLIGHT: async=true reason="test fixture, in-memory tmpdir DB" exec(`CREATE TABLE nodes ( - public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, + public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, first_seen TEXT, advert_count INTEGER DEFAULT 0 )`) // PREFLIGHT: async=true reason="test fixture, in-memory tmpdir DB" @@ -69,15 +77,15 @@ func createTestDBAmbiguousPrefix(t *testing.T, relayA, relayB, hop, firstSeen st // Two repeaters sharing the same 2-char prefix `hop`. // Different advert_counts so tier-4 tiebreak deterministically picks one // (proving the bug: it over-attributes to the higher-count node). - exec(`INSERT INTO nodes (public_key, name, role, advert_count) VALUES (?,?,?,?)`, + exec(`INSERT INTO nodes (public_key, name, role, advert_count) VALUES ($1,$2,$3,$4)`, relayA, "Relay A", "repeater", 50) - exec(`INSERT INTO nodes (public_key, name, role, advert_count) VALUES (?,?,?,?)`, + exec(`INSERT INTO nodes (public_key, name, role, advert_count) VALUES ($1,$2,$3,$4)`, relayB, "Relay B", "repeater", 10) // Aged 48h so it lands in the background window (loadChunk path). - exec("INSERT INTO transmissions VALUES (?,?,?,?,0,4,1,?)", + exec("INSERT INTO transmissions VALUES ($1,$2,$3,$4,0,4,1,$5)", 1, "aa", "hashamb_1", firstSeen, `{}`) - exec("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp, raw_hex, resolved_path) VALUES (?,?,?,?,?,?,?,?,?,?,?,NULL)", + exec("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp, raw_hex, resolved_path) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,NULL)", 1, 1, "obs1", "Obs1", "RX", -10.0, -80.0, 5, fmt.Sprintf(`[%q]`, hop), firstSeen, "") return dbPath @@ -97,7 +105,7 @@ func TestLoadChunk_AmbiguousPrefix_SkipsAttribution(t *testing.T) { aged := time.Now().UTC().Add(-48 * time.Hour).Format(time.RFC3339) dbPath := createTestDBAmbiguousPrefix(t, relayA, relayB, hop, aged) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } @@ -148,7 +156,7 @@ func TestLoad_AmbiguousPrefix_SkipsAttribution(t *testing.T) { ts := time.Now().UTC().Format(time.RFC3339) dbPath := createTestDBAmbiguousPrefix(t, relayA, relayB, hop, ts) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } diff --git a/cmd/server/loadchunk_pathjson_fallback_test.go b/cmd/server/loadchunk_pathjson_fallback_test.go index 518fe1611..70757cdd8 100644 --- a/cmd/server/loadchunk_pathjson_fallback_test.go +++ b/cmd/server/loadchunk_pathjson_fallback_test.go @@ -1,13 +1,9 @@ package main import ( - "database/sql" "fmt" - "path/filepath" "testing" "time" - - _ "modernc.org/sqlite" ) // createTestDBPathJSONNoResolvedPath builds a fixture that mirrors the LIVE @@ -22,10 +18,9 @@ import ( // s.byNode is via path_json → resolvePathForObs relay-hop resolution. func createTestDBPathJSONNoResolvedPath(t *testing.T, relayPubkey, hopPrefix, firstSeen string) string { t.Helper() - dir := t.TempDir() - dbPath := filepath.Join(dir, "test.db") + dbPath := postgresTestDSN(t) - conn, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL") + conn, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } @@ -38,30 +33,43 @@ func createTestDBPathJSONNoResolvedPath(t *testing.T, relayPubkey, hopPrefix, fi } // PREFLIGHT: async=true reason="test fixture: in-memory t.TempDir SQLite, never touches a real DB. Tables are CREATE-from-empty in a one-shot OpenDB call, not a schema migration over existing data." - exec(`CREATE TABLE transmissions ( - id INTEGER PRIMARY KEY, + exec(testNativeSQL(`CREATE TABLE transmissions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT - )`) + )`, `CREATE TABLE transmissions ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + raw_hex TEXT, hash TEXT, first_seen TEXT, + route_type INTEGER, payload_type INTEGER, payload_version INTEGER, + decoded_json TEXT + )`)) // resolved_path column present (matches live schema) but left NULL. // PREFLIGHT: async=true reason="test fixture, in-memory tmpdir DB" - exec(`CREATE TABLE observations ( - id INTEGER PRIMARY KEY, + exec(testNativeSQL(`CREATE TABLE observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, - direction TEXT, snr REAL, rssi REAL, score INTEGER, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT, resolved_path TEXT - )`) + )`, `CREATE TABLE observations ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + transmission_id INTEGER, + observer_id TEXT, observer_name TEXT, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, + path_json TEXT, timestamp TEXT, + raw_hex TEXT, + resolved_path TEXT + )`)) // PREFLIGHT: async=true reason="test fixture, in-memory tmpdir DB" - exec(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT)`) + exec(testNativeSQL(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY AUTOINCREMENT, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, `CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`)) // Production nodes schema uses public_key (not pubkey) — getAllNodes / // buildPrefixMap reads public_key, role, advert_count, first_seen. // PREFLIGHT: async=true reason="test fixture, in-memory tmpdir DB" exec(`CREATE TABLE nodes ( - public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, + public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, first_seen TEXT, advert_count INTEGER DEFAULT 0 )`) // PREFLIGHT: async=true reason="test fixture, in-memory tmpdir DB" @@ -71,13 +79,13 @@ func createTestDBPathJSONNoResolvedPath(t *testing.T, relayPubkey, hopPrefix, fi exec(`CREATE INDEX idx_tx_first_seen ON transmissions(first_seen)`) // Repeater node so canAppearInPath() admits it to the prefix map. - exec(`INSERT INTO nodes (public_key, name, role, advert_count) VALUES (?,?,?,?)`, + exec(`INSERT INTO nodes (public_key, name, role, advert_count) VALUES ($1,$2,$3,$4)`, relayPubkey, "Relay One", "repeater", 10) - exec("INSERT INTO transmissions VALUES (?,?,?,?,0,4,1,?)", + exec("INSERT INTO transmissions VALUES ($1,$2,$3,$4,0,4,1,$5)", 1, "aa", "hashpjf_1", firstSeen, `{}`) // resolved_path explicitly NULL; path_json carries the relay hop prefix. - exec("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp, raw_hex, resolved_path) VALUES (?,?,?,?,?,?,?,?,?,?,?,NULL)", + exec("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp, raw_hex, resolved_path) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,NULL)", 1, 1, "obs1", "Obs1", "RX", -10.0, -80.0, 5, fmt.Sprintf(`[%q]`, hopPrefix), firstSeen, "") return dbPath @@ -103,7 +111,7 @@ func TestLoadChunked_ResolvesRelayHopsFromPathJSON_WhenResolvedPathEmpty(t *test ts := time.Now().UTC().Format(time.RFC3339) dbPath := createTestDBPathJSONNoResolvedPath(t, relayPK, hop, ts) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } @@ -146,7 +154,7 @@ func TestLoadChunk_ResolvesRelayHopsFromPathJSON_WhenResolvedPathEmpty(t *testin aged := time.Now().UTC().Add(-48 * time.Hour).Format(time.RFC3339) dbPath := createTestDBPathJSONNoResolvedPath(t, relayPK, hop, aged) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } diff --git a/cmd/server/loadchunk_resolved_path_1558_test.go b/cmd/server/loadchunk_resolved_path_1558_test.go index b9ede3c4f..81373a899 100644 --- a/cmd/server/loadchunk_resolved_path_1558_test.go +++ b/cmd/server/loadchunk_resolved_path_1558_test.go @@ -1,13 +1,9 @@ package main import ( - "database/sql" "fmt" - "path/filepath" "testing" "time" - - _ "modernc.org/sqlite" ) // createTestDBWithResolvedPath creates a fixture DB containing numTx old @@ -17,10 +13,9 @@ import ( // column so loadChunk's hasResolvedPath branch is exercised. func createTestDBWithResolvedPath(t *testing.T, numTx int, relayPubkeys []string) string { t.Helper() - dir := t.TempDir() - dbPath := filepath.Join(dir, "test.db") + dbPath := postgresTestDSN(t) - conn, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL") + conn, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } @@ -32,23 +27,36 @@ func createTestDBWithResolvedPath(t *testing.T, numTx int, relayPubkeys []string } } - exec(`CREATE TABLE transmissions ( - id INTEGER PRIMARY KEY, + exec(testNativeSQL(`CREATE TABLE transmissions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT - )`) - exec(`CREATE TABLE observations ( - id INTEGER PRIMARY KEY, + )`, `CREATE TABLE transmissions ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + raw_hex TEXT, hash TEXT, first_seen TEXT, + route_type INTEGER, payload_type INTEGER, payload_version INTEGER, + decoded_json TEXT + )`)) + exec(testNativeSQL(`CREATE TABLE observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + transmission_id INTEGER, + observer_id TEXT, observer_name TEXT, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, + path_json TEXT, timestamp TEXT, + raw_hex TEXT, + resolved_path TEXT + )`, `CREATE TABLE observations ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, - direction TEXT, snr REAL, rssi REAL, score INTEGER, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT, resolved_path TEXT - )`) - exec(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT)`) - exec(`CREATE TABLE nodes (pubkey TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, frequency REAL)`) + )`)) + exec(testNativeSQL(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY AUTOINCREMENT, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, `CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`)) + exec(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, first_seen TEXT, frequency DOUBLE PRECISION)`) exec(`CREATE TABLE schema_version (version INTEGER)`) exec(`INSERT INTO schema_version (version) VALUES (1)`) exec(`CREATE INDEX idx_tx_first_seen ON transmissions(first_seen)`) @@ -67,9 +75,9 @@ func createTestDBWithResolvedPath(t *testing.T, numTx int, relayPubkeys []string for i := 0; i < numTx; i++ { ts := now.Add(-48 * time.Hour).Add(time.Duration(i) * time.Second).Format(time.RFC3339) hash := fmt.Sprintf("hash1558_%d", i) - exec("INSERT INTO transmissions VALUES (?,?,?,?,0,4,1,?)", + exec("INSERT INTO transmissions VALUES ($1,$2,$3,$4,0,4,1,$5)", i+1, "aa", hash, ts, `{}`) - exec("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp, raw_hex, resolved_path) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)", + exec("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp, raw_hex, resolved_path) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12)", i+1, i+1, "obs1", "Obs1", "RX", -10.0, -80.0, 5, `[]`, ts, "", rpJSON) } return dbPath @@ -107,7 +115,7 @@ func TestLoadChunk_IndexesResolvedPathPubkeys_Issue1558(t *testing.T) { dbPath := createTestDBWithResolvedPath(t, 3, []string{relayPK1, relayPK2}) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } diff --git a/cmd/server/mail_webhook.go b/cmd/server/mail_webhook.go new file mode 100644 index 000000000..6f7cedc36 --- /dev/null +++ b/cmd/server/mail_webhook.go @@ -0,0 +1,39 @@ +package main + +import ( + "io" + "log" + "net/http" + + "github.com/meshcore-analyzer/mailer" +) + +// handleBrevoWebhook ingests Brevo transactional events. Brevo is configured +// with auth {"type":"bearer","token":}, so it sends +// "Authorization: Bearer ". Unknown message ids get 200 so Brevo +// does not retry forever. +func (s *Server) handleBrevoWebhook(w http.ResponseWriter, r *http.Request) { + a := s.auth + if !a.allow(w, r, a.hook) { + return + } + if !constantTimeEqual(r.Header.Get("Authorization"), "Bearer "+a.set.webhookSecret) { + writeError(w, http.StatusUnauthorized, "unauthorized") + return + } + body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, 256<<10)) + if err != nil { + writeError(w, http.StatusBadRequest, "body too large") + return + } + evs, err := mailer.ParseBrevoWebhook(body) + if err != nil { + // Authenticated but unusable: answer 200 so Brevo does not retry + // forever; log for the operator. + log.Printf("[users] brevo webhook: ignored payload: %s", redactAddrs(err)) + writeJSON(w, okResponse{OK: true}) + return + } + a.ingestMailEvents(evs) + writeJSON(w, okResponse{OK: true}) +} diff --git a/cmd/server/mail_webhook_test.go b/cmd/server/mail_webhook_test.go new file mode 100644 index 000000000..de6a3c011 --- /dev/null +++ b/cmd/server/mail_webhook_test.go @@ -0,0 +1,69 @@ +package main + +import ( + "bytes" + "fmt" + "net/http/httptest" + "testing" + + "github.com/gorilla/mux" +) + +func TestBrevoWebhookAuthAndIngest(t *testing.T) { + f, _, uma := adminFixture(t) + mails, _ := f.st.MailForUser(uma.me.ID, 10) + m := mails[0] + body := fmt.Sprintf(`{"event":"hard_bounce","email":"uma@example.org","message-id":%q,"ts_event":%d,"reason":"mailbox unavailable"}`, + m.ProviderMessageID, m.SentAt.Unix()+30) + post := func(auth, b string) int { + req := httptest.NewRequest("POST", "/api/mail/brevo/webhook", bytes.NewBufferString(b)) + req.RemoteAddr = "203.0.113.50:443" + if auth != "" { + req.Header.Set("Authorization", auth) + } + w := httptest.NewRecorder() + f.router.ServeHTTP(w, req) + return w.Code + } + if c := post("", body); c != 401 { + t.Fatalf("no auth = %d", c) + } + if c := post("Bearer wrong-secret-xxxxxxxx", body); c != 401 { + t.Fatalf("wrong auth = %d", c) + } + if c := post("Bearer "+testHook, "garbage"); c != 200 { // authenticated junk: 200 so Brevo stops retrying + t.Fatalf("garbage = %d", c) + } + if c := post("Bearer "+testHook, body); c != 200 { + t.Fatalf("valid = %d", c) + } + rec, _ := f.st.MailByID(m.ID) + if rec.LastEvent != "hard_bounce" || rec.LastReason != "mailbox unavailable" { + t.Fatalf("mail record = %+v", rec) + } + if u, _ := f.st.GetByID(uma.me.ID); !u.EmailBouncing { + t.Fatal("bounce flag not set") + } + unknown := `{"event":"delivered","message-id":"","ts_event":1}` + if c := post("Bearer "+testHook, unknown); c != 200 { + t.Fatalf("unknown id = %d (must be 200 so Brevo stops retrying)", c) + } +} + +func TestBrevoWebhookAbsentWithoutSecret(t *testing.T) { + f, _, _ := adminFixture(t) + f.srv.auth.set.webhookSecret = "" + r := newAuthFixtureRouterOnly(f) + req := httptest.NewRequest("POST", "/api/mail/brevo/webhook", bytes.NewBufferString("{}")) + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + if w.Code != 404 { + t.Fatalf("no secret configured = %d, want 404", w.Code) + } +} + +func newAuthFixtureRouterOnly(f *authFixture) *mux.Router { + r := mux.NewRouter() + f.srv.registerAuthRoutes(r) + return r +} diff --git a/cmd/server/main.go b/cmd/server/main.go index c4166d1e6..b9a8cdca5 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -2,7 +2,6 @@ package main import ( "context" - "database/sql" "flag" "fmt" "log" @@ -18,7 +17,7 @@ import ( "time" "github.com/gorilla/mux" - "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/dbconfig" ) // Set via -ldflags at build time @@ -86,24 +85,30 @@ func main() { } var ( - configDir string - port int - dbPath string - publicDir string - pollMs int + configDir string + port int + legacyDB string + backend string + databaseURL string + stateDir string + publicDir string + pollMs int ) flag.StringVar(&configDir, "config-dir", ".", "Directory containing config.json") flag.IntVar(&port, "port", 0, "HTTP port (overrides config)") - flag.StringVar(&dbPath, "db", "", "SQLite database path (overrides config/env)") + flag.StringVar(&legacyDB, "db", "", "SQLite telemetry path (bootstrap override)") + flag.StringVar(&backend, "backend", "", "Storage backend: sqlite or postgres (bootstrap choice)") + flag.StringVar(&databaseURL, "database-url", "", "PostgreSQL telemetry reader URL (overrides config/env)") + flag.StringVar(&stateDir, "state-dir", "", "Local directory for queues and sidecars") flag.StringVar(&publicDir, "public", "public", "Directory to serve static files from") - flag.IntVar(&pollMs, "poll-ms", 1000, "SQLite poll interval for WebSocket broadcast (ms)") + flag.IntVar(&pollMs, "poll-ms", 1000, "Database poll interval for WebSocket broadcast (ms)") flag.Parse() // Load config cfg, err := LoadConfig(configDir) if err != nil { - log.Printf("[config] warning: %v (using defaults)", err) + log.Fatalf("[config] %v", err) } // CLI flags override config @@ -113,9 +118,7 @@ func main() { if cfg.Port == 0 { cfg.Port = 3000 } - if dbPath != "" { - cfg.DBPath = dbPath - } + if cfg.APIKey == "" { log.Printf("[security] WARNING: no apiKey configured — write endpoints are BLOCKED (set apiKey in config.json to enable them)") } else if IsWeakAPIKey(cfg.APIKey) { @@ -157,9 +160,18 @@ func main() { warnIfMemlimitUnderprovisioned(limit) } - // Resolve DB path - resolvedDB := cfg.ResolveDBPath(configDir) - log.Printf("[config] port=%d db=%s public=%s", cfg.Port, resolvedDB, publicDir) + rawStorage, err := cfg.storageInputs(configDir, storageFlags{Backend: dbconfig.Backend(backend), DBPath: legacyDB, DatabaseURL: databaseURL, StateDir: stateDir}, os.Getenv) + if err != nil { + log.Fatalf("[config] %v", err) + } + storage, selectionLease, err := resolveRuntimeStorage(rawStorage) + if err != nil { + log.Fatalf("[storage] %v", err) + } + defer selectionLease.Close() + runtimeStateDir = storage.StateDir + log.Printf("[config] port=%d database=%s public=%s", cfg.Port, storage.Backend, publicDir) + if len(cfg.NodeBlacklist) > 0 { log.Printf("[config] nodeBlacklist: %d node(s) will be hidden from API", len(cfg.NodeBlacklist)) for _, pk := range cfg.NodeBlacklist { @@ -170,10 +182,11 @@ func main() { } // Open database - database, err := OpenDB(resolvedDB) + database, err := OpenStorage(storage) if err != nil { - log.Fatalf("[db] failed to open %s: %v", resolvedDB, err) + log.Fatalf("[db] failed to open selected database: %v", err) } + database.stateDir = runtimeStateDir var dbCloseOnce sync.Once dbClose := func() error { var err error @@ -182,13 +195,6 @@ func main() { } defer dbClose() - // Verify DB has expected tables - var tableName string - err = database.conn.QueryRow("SELECT name FROM sqlite_master WHERE type='table' AND name='transmissions'").Scan(&tableName) - if err == sql.ErrNoRows { - log.Fatalf("[db] table 'transmissions' not found — is this a CoreScope database?") - } - stats, err := database.GetStats() if err != nil { log.Printf("[db] warning: could not read stats: %v", err) @@ -204,8 +210,8 @@ func main() { // (#1287). The server NEVER migrates — it only reads. If a required // column/index/table is missing, the operator must restart the // ingestor (which owns dbschema.Apply) before this server can start. - if err := dbschema.AssertReady(database.conn); err != nil { - log.Fatalf("[db] schema not ready (ingestor must run migrations first): %v", err) + if err := database.AssertReady(); err != nil { + log.Fatalf("[db] schema not ready (run the selected backend setup/writer first): %v", err) } // In-memory packet store @@ -285,7 +291,6 @@ func main() { // loaded above, before the packet load. Per #1287 schema migrations // all live in the ingestor; the server only reads the snapshot and // then refreshes it via the recompNeighborGraph slot every 60s. - dbPath = database.path database.hasResolvedPath = true // dbschema.AssertReady above already verified observations.resolved_path exists // WaitGroup for background init steps that gate /api/healthz readiness. @@ -339,6 +344,7 @@ func main() { store.mu.Lock() for j := i; j < end && j < len(store.packets); j++ { pickBestObservation(store.packets[j]) + store.trackedBytes += rechargeTx(store.packets[j]) } store.mu.Unlock() if end < totalPackets { @@ -358,12 +364,19 @@ func main() { // WebSocket hub hub := NewHub() hub.SetAllowedOrigins(cfg.CORSAllowedOrigins) + hub.ConfigureLimits(cfg.WSMaxConnsPerIP(), cfg.WSUpgradesPerMinPerIP(), + cfg.WSTrustedProxies(), cfg.WSDeny()) // #1794 hub.upgrader.EnableCompression = cfg.WSCompressionEnabled() // HTTP server srv := NewServer(database, cfg, hub) srv.configDir = configDir srv.store = store + // Optional user management (off by default). Fails startup on a bad + // config rather than running with registration that cannot work. + if err := srv.initUserManagementStorage(storage); err != nil { + log.Fatalf("[users] %v", err) + } router := mux.NewRouter() srv.RegisterRoutes(router) @@ -384,7 +397,7 @@ func main() { }) } - // Start SQLite poller for WebSocket broadcast + // Start PostgreSQL poller for WebSocket broadcast poller := NewPoller(database, hub, time.Duration(pollMs)*time.Millisecond) poller.store = store go poller.Start() @@ -526,7 +539,7 @@ func main() { // 1c. Stop steady-state analytics recomputers (issue #1240). // Must happen before dbClose so any in-flight compute that - // reaches into SQLite has finished. + // queries PostgreSQL has finished. if stopAnalyticsRecomp != nil { stopAnalyticsRecomp() } @@ -541,7 +554,10 @@ func main() { // 3. Close WebSocket hub hub.Close() - // 4. Close database (release SQLite WAL lock) + // 3b. Close users.db (user management, opt-in; no-op when off). + srv.closeUserManagement() + + // 4. Close the database connection pool if err := dbClose(); err != nil { log.Printf("[server] DB close error: %v", err) } @@ -555,8 +571,8 @@ func main() { // process. The server reads the results via the periodic // recompNeighborGraph / fetchResolvedPathForObs paths. - // Migrate old content hashes in background (one-time, idempotent). - go migrateContentHashesAsync(store, 5000, 100*time.Millisecond) + // Verify loaded content hashes without issuing telemetry writes. + go verifyContentHashesAsync(store, 5000, 100*time.Millisecond) if err := httpServer.ListenAndServe(); err != http.ErrServerClosed { log.Fatalf("[server] %v", err) diff --git a/cmd/server/memlimit.go b/cmd/server/memlimit.go index 4ff590600..4cbfa1d3b 100644 --- a/cmd/server/memlimit.go +++ b/cmd/server/memlimit.go @@ -36,6 +36,10 @@ func applyMemoryLimit(maxMemoryMB int, envSet bool) (int64, string) { // 1.5x headroom over the steady-state packet store budget covers // transient peaks (cold-load row-scan / decode pipeline, Go's NextGC // trigger at ~2x live heap). See issue #836 heap profile. + // + // GOMEMLIMIT covers Go-managed memory, not an RSS ceiling. PostgreSQL + // runs in another process and needs its own memory budget; its buffers + // and work memory are not represented by this packet-store limit. limit := int64(maxMemoryMB) * 1024 * 1024 * 3 / 2 debug.SetMemoryLimit(limit) return limit, "derived" diff --git a/cmd/server/memory.go b/cmd/server/memory.go index 2ffe4d7ce..2b43a67c0 100644 --- a/cmd/server/memory.go +++ b/cmd/server/memory.go @@ -32,9 +32,10 @@ import ( // eviction; both can shrink when packets age out. goHeapInuseMB and goSysMB // fluctuate with GC. // -// cgoBytesMB intentionally absent: this build uses the pure-Go -// modernc.org/sqlite driver, so there is no cgo allocator to measure. -// Reintroduce only if we ever switch back to mattn/go-sqlite3. +// These are server-process measurements. PostgreSQL buffers and query work +// memory live in the database process and must be measured separately. +// processRSSMB - goSysMB is not an allocation counter: goSysMB includes address +// space reserved but not resident, so that subtraction can be negative. type MemorySnapshot struct { ProcessRSSMB float64 `json:"processRSSMB"` GoHeapInuseMB float64 `json:"goHeapInuseMB"` diff --git a/cmd/server/multi_node_cap_test.go b/cmd/server/multi_node_cap_test.go new file mode 100644 index 000000000..cacf3e99f --- /dev/null +++ b/cmd/server/multi_node_cap_test.go @@ -0,0 +1,38 @@ +package main + +import ( + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +// GET /api/packets?nodes=... resolves every entry with a SQLite lookup while +// holding the packet store's read lock, so the list must be capped. +func TestMultiNodePacketsListCap(t *testing.T) { + _, router := setupTestServerWithAPIKey(t, "") + + get := func(n int) *httptest.ResponseRecorder { + keys := make([]string, n) + for i := range keys { + keys[i] = fmt.Sprintf("%064x", i+1) + } + req := httptest.NewRequest("GET", "/api/packets?nodes="+strings.Join(keys, ",")+"&limit=1", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + return w + } + + t.Run("at the cap is accepted", func(t *testing.T) { + if w := get(maxMultiNodePubkeys); w.Code != http.StatusOK { + t.Fatalf("expected 200 for %d nodes, got %d (body: %s)", maxMultiNodePubkeys, w.Code, w.Body.String()) + } + }) + + t.Run("one over the cap is rejected with 400", func(t *testing.T) { + if w := get(maxMultiNodePubkeys + 1); w.Code != http.StatusBadRequest { + t.Fatalf("expected 400 for %d nodes, got %d (body: %s)", maxMultiNodePubkeys+1, w.Code, w.Body.String()) + } + }) +} diff --git a/cmd/server/multibyte_capability_test.go b/cmd/server/multibyte_capability_test.go index 1325510a4..11327d8f0 100644 --- a/cmd/server/multibyte_capability_test.go +++ b/cmd/server/multibyte_capability_test.go @@ -1,14 +1,11 @@ package main import ( - "database/sql" "encoding/json" "fmt" "strings" "testing" "time" - - _ "modernc.org/sqlite" ) // recentTS returns a timestamp string N hours ago, ensuring test data @@ -20,23 +17,27 @@ func recentTS(hoursAgo int) string { // setupCapabilityTestDB creates a minimal in-memory DB with nodes table. func setupCapabilityTestDB(t *testing.T) *DB { t.Helper() - conn, err := sql.Open("sqlite", ":memory:") + conn, err := openPostgresTestDB(t) if err != nil { t.Fatal(err) } conn.SetMaxOpenConns(1) conn.Exec(`CREATE TABLE nodes ( public_key TEXT PRIMARY KEY, name TEXT, role TEXT, - lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, - advert_count INTEGER DEFAULT 0, battery_mv INTEGER, temperature_c REAL + lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, first_seen TEXT, + advert_count INTEGER DEFAULT 0, battery_mv INTEGER, temperature_c DOUBLE PRECISION )`) - conn.Exec(`CREATE TABLE observers ( + conn.Exec(testNativeSQL(`CREATE TABLE observers (id TEXT PRIMARY KEY, name TEXT, iata TEXT, last_seen TEXT, + first_seen TEXT, packet_count INTEGER DEFAULT 0, model TEXT, + firmware TEXT, client_version TEXT, radio TEXT, battery_mv INTEGER, + uptime_secs INTEGER + )`, `CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY UNIQUE, id TEXT PRIMARY KEY, name TEXT, iata TEXT, last_seen TEXT, first_seen TEXT, packet_count INTEGER DEFAULT 0, model TEXT, firmware TEXT, client_version TEXT, radio TEXT, battery_mv INTEGER, uptime_secs INTEGER - )`) - return &DB{conn: conn} + )`)) + return &DB{backend: testBackendValue(), conn: conn} } // addTestPacket adds a StoreTx to the store's internal structures including @@ -86,7 +87,7 @@ func TestMultiByteCapability_Confirmed(t *testing.T) { db := setupCapabilityTestDB(t) defer db.conn.Close() - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ($1, $2, $3, $4)", "aabbccdd11223344", "RepA", "repeater", recentTS(24)) store := NewPacketStore(db, nil) @@ -113,7 +114,7 @@ func TestMultiByteCapability_Suspected(t *testing.T) { db := setupCapabilityTestDB(t) defer db.conn.Close() - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ($1, $2, $3, $4)", "aabbccdd11223344", "RepB", "repeater", recentTS(48)) store := NewPacketStore(db, nil) @@ -151,7 +152,7 @@ func TestMultiByteCapability_Unknown(t *testing.T) { db := setupCapabilityTestDB(t) defer db.conn.Close() - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ($1, $2, $3, $4)", "aabbccdd11223344", "RepC", "repeater", recentTS(72)) store := NewPacketStore(db, nil) @@ -179,9 +180,9 @@ func TestMultiByteCapability_PrefixCollision(t *testing.T) { defer db.conn.Close() // Two repeaters sharing 1-byte prefix "aa" - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ($1, $2, $3, $4)", "aabb000000000001", "RepConfirmed", "repeater", recentTS(24)) - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ($1, $2, $3, $4)", "aacc000000000002", "RepOther", "repeater", recentTS(24)) store := NewPacketStore(db, nil) @@ -227,7 +228,7 @@ func TestMultiByteCapability_TraceExcluded(t *testing.T) { db := setupCapabilityTestDB(t) defer db.conn.Close() - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ($1, $2, $3, $4)", "aabbccdd11223344", "RepTrace", "repeater", recentTS(48)) store := NewPacketStore(db, nil) @@ -259,7 +260,7 @@ func TestMultiByteCapability_NonTraceStillSuspected(t *testing.T) { db := setupCapabilityTestDB(t) defer db.conn.Close() - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ($1, $2, $3, $4)", "aabbccdd11223344", "RepNonTrace", "repeater", recentTS(48)) store := NewPacketStore(db, nil) @@ -291,7 +292,7 @@ func TestMultiByteCapability_ConfirmedUnaffectedByTraceExclusion(t *testing.T) { db := setupCapabilityTestDB(t) defer db.conn.Close() - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ($1, $2, $3, $4)", "aabbccdd11223344", "RepConfirmedTrace", "repeater", recentTS(24)) store := NewPacketStore(db, nil) @@ -326,7 +327,7 @@ func TestMultiByteCapability_CompanionConfirmed(t *testing.T) { db := setupCapabilityTestDB(t) defer db.conn.Close() - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ($1, $2, $3, $4)", "aabbccdd11223344", "CompA", "companion", recentTS(24)) store := NewPacketStore(db, nil) @@ -353,11 +354,11 @@ func TestMultiByteCapability_RoleColumnPopulated(t *testing.T) { db := setupCapabilityTestDB(t) defer db.conn.Close() - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ($1, $2, $3, $4)", "aabb000000000001", "Rep1", "repeater", recentTS(24)) - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ($1, $2, $3, $4)", "ccdd000000000002", "Comp1", "companion", recentTS(24)) - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ($1, $2, $3, $4)", "eeff000000000003", "Room1", "room_server", recentTS(24)) store := NewPacketStore(db, nil) @@ -392,7 +393,7 @@ func TestMultiByteCapability_AdopterEvidenceTakesPrecedence(t *testing.T) { db := setupCapabilityTestDB(t) defer db.conn.Close() - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ($1, $2, $3, $4)", "aabbccdd11223344", "RepAdopter", "repeater", recentTS(24)) store := NewPacketStore(db, nil) @@ -446,26 +447,26 @@ func TestMultiByteCapability_AdopterEvidenceTakesPrecedence(t *testing.T) { // setupPersistTestDB creates an in-memory DB with multibyte_sup/multibyte_evidence columns. func setupPersistTestDB(t *testing.T) *DB { t.Helper() - conn, err := sql.Open("sqlite", ":memory:") + conn, err := openPostgresTestDB(t) if err != nil { t.Fatal(err) } conn.SetMaxOpenConns(1) conn.Exec(`CREATE TABLE nodes ( public_key TEXT PRIMARY KEY, name TEXT, role TEXT, - lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, - advert_count INTEGER DEFAULT 0, battery_mv INTEGER, temperature_c REAL, + lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, first_seen TEXT, + advert_count INTEGER DEFAULT 0, battery_mv INTEGER, temperature_c DOUBLE PRECISION, foreign_advert INTEGER DEFAULT 0, default_scope TEXT, multibyte_sup INTEGER NOT NULL DEFAULT 0, multibyte_evidence TEXT )`) conn.Exec(`CREATE TABLE inactive_nodes ( public_key TEXT PRIMARY KEY, name TEXT, role TEXT, - lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, - advert_count INTEGER DEFAULT 0, battery_mv INTEGER, temperature_c REAL, + lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, first_seen TEXT, + advert_count INTEGER DEFAULT 0, battery_mv INTEGER, temperature_c DOUBLE PRECISION, foreign_advert INTEGER DEFAULT 0, default_scope TEXT, multibyte_sup INTEGER NOT NULL DEFAULT 0, multibyte_evidence TEXT )`) - return &DB{conn: conn, hasMultibyteSupCols: true} + return &DB{backend: testBackendValue(), conn: conn, hasMultibyteSupCols: true} } // TestMultibyteCapGetMultibyteCapForO1 verifies that GetMultibyteCapFor returns diff --git a/cmd/server/multibyte_region_filter_test.go b/cmd/server/multibyte_region_filter_test.go index a66fedc13..6fe8d6719 100644 --- a/cmd/server/multibyte_region_filter_test.go +++ b/cmd/server/multibyte_region_filter_test.go @@ -30,9 +30,9 @@ func TestMultiByteCapability_RegionFiltered_PreservesConfirmedStatus(t *testing. // Two observers in different regions. db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obs-sjc', 'Obs SJC', 'SJC', ?, '2026-01-01T00:00:00Z', 100)`, recent) + VALUES ('obs-sjc', 'Obs SJC', 'SJC', $1, '2026-01-01T00:00:00Z', 100)`, recent) db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obs-jkg', 'Obs JKG', 'JKG', ?, '2026-01-01T00:00:00Z', 100)`, recent) + VALUES ('obs-jkg', 'Obs JKG', 'JKG', $1, '2026-01-01T00:00:00Z', 100)`, recent) // Node A: a JKG-region repeater that advertises multi-byte (hash_size=2). // Its zero-hop direct advert is only heard by obs-SJC (e.g. an out-of-region @@ -42,7 +42,7 @@ func TestMultiByteCapability_RegionFiltered_PreservesConfirmedStatus(t *testing. // to "confirmed" via the global advert evidence. pkA := "aaa0000000000001" db.conn.Exec(`INSERT INTO nodes (public_key, name, role) - VALUES (?, 'Node-A', 'repeater')`, pkA) + VALUES ($1, 'Node-A', 'repeater')`, pkA) decodedA := `{"pubKey":"` + pkA + `","name":"Node-A","type":"ADVERT","flags":{"isRepeater":true}}` @@ -50,17 +50,17 @@ func TestMultiByteCapability_RegionFiltered_PreservesConfirmedStatus(t *testing. // pathByte 0x40 → hash_size bits 01 → 2 bytes. // Heard by obs-SJC ONLY. db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('1240aabbccdd', 'a_zh_direct', ?, 2, 4, ?)`, recent, decodedA) + VALUES ('1240aabbccdd', 'a_zh_direct', $1, 2, 4, $2)`, recent, decodedA) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 12.0, -85, '[]', ?)`, recentEpoch) + VALUES (1, 1, 12.0, -85, '[]', $1)`, recentEpoch) // Node A also appears as a path hop in a JKG-observed packet, so it // shows up in the JKG region's node list. // route_type=1 (flood), payload_type=4, pathByte 0x41 (hs=2, hops=1) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('1141aabbccdd', 'a_jkg_relay', ?, 1, 4, ?)`, recent, decodedA) + VALUES ('1141aabbccdd', 'a_jkg_relay', $1, 1, 4, $2)`, recent, decodedA) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (2, 2, 8.0, -95, '["aa"]', ?)`, recentEpoch) + VALUES (2, 2, 8.0, -95, '["aa"]', $1)`, recentEpoch) store := NewPacketStore(db, nil) store.Load() diff --git a/cmd/server/neighbor_api_test.go b/cmd/server/neighbor_api_test.go index 4a146eaf7..78e1b5740 100644 --- a/cmd/server/neighbor_api_test.go +++ b/cmd/server/neighbor_api_test.go @@ -1,7 +1,6 @@ package main import ( - "database/sql" "encoding/json" "net/http" "net/http/httptest" @@ -9,7 +8,6 @@ import ( "time" "github.com/gorilla/mux" - _ "modernc.org/sqlite" ) // ─── Helpers ─────────────────────────────────────────────────────────────────── @@ -464,10 +462,9 @@ func TestNeighborGraphAPI_ResponseShape(t *testing.T) { func TestBuildNodeInfoMap_ObserverEnrichment(t *testing.T) { // Create a temp SQLite DB with nodes and observers tables. - tmpDir := t.TempDir() - dbPath := tmpDir + "/test.db" + dbPath := postgresTestDSN(t) - conn, err := sql.Open("sqlite", dbPath) + conn, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } @@ -475,20 +472,21 @@ func TestBuildNodeInfoMap_ObserverEnrichment(t *testing.T) { // Create tables for _, stmt := range []string{ - "CREATE TABLE nodes (public_key TEXT, name TEXT, role TEXT, lat REAL, lon REAL)", - "CREATE TABLE observers (id TEXT, name TEXT, iata TEXT)", + "CREATE TABLE nodes (public_key TEXT, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION)", + testNativeSQL(`CREATE TABLE observers (id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, "CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY UNIQUE, id TEXT, name TEXT, iata TEXT, inactive INTEGER)"), "INSERT INTO nodes VALUES ('AAAA1111', 'Repeater-1', 'repeater', 0, 0)", - "INSERT INTO observers VALUES ('BBBB2222', 'Observer-Alpha', '')", - "INSERT INTO observers VALUES ('AAAA1111', 'Obs-also-repeater', '')", + "INSERT INTO observers (id, name, iata) VALUES ('BBBB2222', 'Observer-Alpha', '')", + "INSERT INTO observers (id, name, iata) VALUES ('AAAA1111', 'Obs-also-repeater', '')", } { if _, err := conn.Exec(stmt); err != nil { t.Fatalf("exec %q: %v", stmt, err) } } + ensurePreparable(t, conn) conn.Close() // Open via our DB wrapper - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } @@ -538,26 +536,26 @@ func TestBuildNodeInfoMap_ObserverEnrichment(t *testing.T) { // existing 30s node cache, both calls return the original value — same as // every other nodeInfo field that comes from getAllNodes(). func TestBuildNodeInfoMap_FirstSeenIsCached(t *testing.T) { - tmpDir := t.TempDir() - dbPath := tmpDir + "/test.db" + dbPath := postgresTestDSN(t) // Seed via rw connection. - rw, err := sql.Open("sqlite", dbPath) + rw, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } defer rw.Close() for _, stmt := range []string{ - "CREATE TABLE nodes (public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, advert_count INTEGER)", - "CREATE TABLE observers (id TEXT, name TEXT, iata TEXT)", + "CREATE TABLE nodes (public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, first_seen TEXT, advert_count INTEGER)", + testNativeSQL(`CREATE TABLE observers (id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, "CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY UNIQUE, id TEXT, name TEXT, iata TEXT, inactive INTEGER)"), "INSERT INTO nodes VALUES ('AAAA1111', 'Repeater-1', 'repeater', 0, 0, '', '2024-01-01T00:00:00Z', 0)", } { if _, err := rw.Exec(stmt); err != nil { t.Fatalf("seed exec %q: %v", stmt, err) } } + ensurePreparable(t, rw) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } @@ -604,26 +602,26 @@ func TestBuildNodeInfoMap_FirstSeenIsCached(t *testing.T) { // values must have empty FirstSeen with no panic. Regression coverage for the // existing fallback branch (#1632 review loop 1). func TestGetAllNodes_FirstSeenSchemaFallback(t *testing.T) { - tmpDir := t.TempDir() - dbPath := tmpDir + "/test.db" + dbPath := postgresTestDSN(t) // Seed a nodes table WITHOUT first_seen (advert_count + last_seen present). - rw, err := sql.Open("sqlite", dbPath) + rw, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } defer rw.Close() for _, stmt := range []string{ - "CREATE TABLE nodes (public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, advert_count INTEGER)", - "CREATE TABLE observers (id TEXT, name TEXT, iata TEXT)", + "CREATE TABLE nodes (public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, advert_count INTEGER)", + testNativeSQL(`CREATE TABLE observers (id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, "CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY UNIQUE, id TEXT, name TEXT, iata TEXT, inactive INTEGER)"), "INSERT INTO nodes VALUES ('BBBB2222', 'Repeater-2', 'repeater', 0, 0, '2024-02-02T00:00:00Z', 3)", } { if _, err := rw.Exec(stmt); err != nil { t.Fatalf("seed exec %q: %v", stmt, err) } } + ensurePreparable(t, rw) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } diff --git a/cmd/server/neighbor_persist.go b/cmd/server/neighbor_persist.go index 9deea6285..d73b972cf 100644 --- a/cmd/server/neighbor_persist.go +++ b/cmd/server/neighbor_persist.go @@ -183,18 +183,7 @@ func resolvePathForObsColdLoad(pathJSON, observerID string, tx *StoreTx, pm *pre // observation_count_fallback would still pick a winner for // ambiguous prefixes, which is exactly what we must NOT do. // Hence the explicit candidate-count check here. - h := strings.ToLower(hop) - candidates := pm.m[h] - if len(pm.nonRelay) > 0 && len(candidates) > 0 { - filtered := candidates[:0:0] - for j := range candidates { - if _, isListener := pm.nonRelay[strings.ToLower(candidates[j].PublicKey)]; isListener { - continue - } - filtered = append(filtered, candidates[j]) - } - candidates = filtered - } + candidates := pm.relayCandidates(hop) if len(candidates) == 1 { pk := strings.ToLower(candidates[0].PublicKey) resolved[i] = &pk diff --git a/cmd/server/neighbor_persist_legacy_test.go b/cmd/server/neighbor_persist_legacy_test.go index efb8d6dd1..051c7ba64 100644 --- a/cmd/server/neighbor_persist_legacy_test.go +++ b/cmd/server/neighbor_persist_legacy_test.go @@ -1,12 +1,8 @@ package main import ( - "database/sql" - "path/filepath" "testing" "time" - - _ "modernc.org/sqlite" ) // TestNeighborPersist_LegacyEdgeInvariant (#1638 adv-#1): edges loaded from @@ -16,9 +12,8 @@ import ( // pre-existing observations land in bucket 0 (legacy/unknown, conservative // weight in the JS confidence indicator). func TestNeighborPersist_LegacyEdgeInvariant(t *testing.T) { - dir := t.TempDir() - dbPath := filepath.Join(dir, "neighbor_legacy.db") - rw, err := sql.Open("sqlite", "file:"+dbPath+"?_journal_mode=WAL") + dbPath := postgresTestDSN(t) + rw, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } @@ -34,7 +29,7 @@ func TestNeighborPersist_LegacyEdgeInvariant(t *testing.T) { } now := time.Now().UTC().Format(time.RFC3339) if _, err := rw.Exec( - `INSERT INTO neighbor_edges (node_a, node_b, count, last_seen) VALUES (?, ?, ?, ?)`, + `INSERT INTO neighbor_edges (node_a, node_b, count, last_seen) VALUES ($1, $2, $3, $4)`, "aaaa", "bbbb", 7, now, ); err != nil { t.Fatal(err) @@ -77,9 +72,8 @@ func TestNeighborPersist_LegacyEdgeInvariant(t *testing.T) { // in the snapshot" path (loader's else-branch): subsequent counts must // accumulate into bucket 0 too, preserving the invariant. func TestNeighborPersist_LegacyEdgeMergeOnReload(t *testing.T) { - dir := t.TempDir() - dbPath := filepath.Join(dir, "neighbor_legacy_merge.db") - rw, err := sql.Open("sqlite", "file:"+dbPath+"?_journal_mode=WAL") + dbPath := postgresTestDSN(t) + rw, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } @@ -97,7 +91,7 @@ func TestNeighborPersist_LegacyEdgeMergeOnReload(t *testing.T) { now := time.Now().UTC().Format(time.RFC3339) for _, cnt := range []int{3, 4} { if _, err := rw.Exec( - `INSERT INTO neighbor_edges (node_a, node_b, count, last_seen) VALUES (?, ?, ?, ?)`, + `INSERT INTO neighbor_edges (node_a, node_b, count, last_seen) VALUES ($1, $2, $3, $4)`, "aaaa", "bbbb", cnt, now, ); err != nil { t.Fatal(err) diff --git a/cmd/server/neighbor_recomputer_test.go b/cmd/server/neighbor_recomputer_test.go index f78770a43..80a986883 100644 --- a/cmd/server/neighbor_recomputer_test.go +++ b/cmd/server/neighbor_recomputer_test.go @@ -1,13 +1,10 @@ package main import ( - "database/sql" - "path/filepath" "testing" "time" "github.com/meshcore-analyzer/dbschema" - _ "modernc.org/sqlite" ) // TestNeighborGraphRecomputerLoadsSnapshot enforces #1287 Option 4: @@ -15,11 +12,10 @@ import ( // snapshot the ingestor writes. After a write to neighbor_edges (here // done synthetically), the recomputer's atomic-swap must reflect it. func TestNeighborGraphRecomputerLoadsSnapshot(t *testing.T) { - dir := t.TempDir() - dbPath := filepath.Join(dir, "neighbor_recomp.db") + dbPath := postgresTestDSN(t) // Bootstrap a WAL DB with the neighbor_edges table. - rw, err := sql.Open("sqlite", "file:"+dbPath+"?_journal_mode=WAL") + rw, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } @@ -33,18 +29,19 @@ func TestNeighborGraphRecomputerLoadsSnapshot(t *testing.T) { )`); err != nil { t.Fatal(err) } + ensurePreparable(t, rw) // Stage one edge. now := time.Now().UTC().Format(time.RFC3339) if _, err := rw.Exec( - `INSERT INTO neighbor_edges (node_a, node_b, count, last_seen) VALUES (?, ?, ?, ?)`, + `INSERT INTO neighbor_edges (node_a, node_b, count, last_seen) VALUES ($1, $2, $3, $4)`, "aaa", "bbb", 5, now, ); err != nil { t.Fatal(err) } // Server opens read-only and refreshes via the recomputer. - d, err := OpenDB(dbPath) + d, err := openFixtureReader(t, dbPath) if err != nil { t.Fatalf("OpenDB: %v", err) } @@ -63,7 +60,7 @@ func TestNeighborGraphRecomputerLoadsSnapshot(t *testing.T) { // Add another row, refresh, assert the new total. if _, err := rw.Exec( - `INSERT INTO neighbor_edges (node_a, node_b, count, last_seen) VALUES (?, ?, ?, ?)`, + `INSERT INTO neighbor_edges (node_a, node_b, count, last_seen) VALUES ($1, $2, $3, $4)`, "ccc", "ddd", 2, now, ); err != nil { t.Fatal(err) @@ -80,22 +77,27 @@ func TestNeighborGraphRecomputerLoadsSnapshot(t *testing.T) { // AssertReady on a DB missing the required columns returns an error // listing every missing surface; main.go then calls log.Fatalf. func TestServerStartupRequiresMigratedSchema(t *testing.T) { - dir := t.TempDir() - dbPath := filepath.Join(dir, "unmigrated.db") + dbPath := postgresTestDSN(t) // Bootstrap with ONLY transmissions/observations (the things // server tries to read) but WITHOUT the columns dbschema asserts - // (resolved_path, inactive, last_packet_at, iata, foreign_advert, - // from_pubkey, neighbor_edges). - rw, err := sql.Open("sqlite", "file:"+dbPath+"?_journal_mode=WAL") + // (resolved_path, last_packet_at, iata, foreign_advert, from_pubkey, + // neighbor_edges). + // + // observers.inactive used to be in that list. It no longer is: OpenDB + // prepares statements eagerly under github.com/mattn/go-sqlite3 and one of + // them selects on inactive, so a fixture without it cannot reach + // AssertReady at all. The other missing surfaces above still make + // AssertReady fail, which is what this test asserts. + rw, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } defer rw.Close() for _, s := range []string{ - `CREATE TABLE transmissions (id INTEGER PRIMARY KEY, hash TEXT, payload_type INTEGER)`, - `CREATE TABLE observations (id INTEGER PRIMARY KEY, transmission_id INTEGER)`, - `CREATE TABLE observers (id TEXT PRIMARY KEY, name TEXT)`, + testNativeSQL(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY AUTOINCREMENT, hash TEXT, payload_type INTEGER)`, `CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, hash TEXT, payload_type INTEGER)`), + testNativeSQL(`CREATE TABLE observations (id INTEGER PRIMARY KEY AUTOINCREMENT, transmission_id INTEGER)`, `CREATE TABLE observations (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER)`), + testNativeSQL(`CREATE TABLE observers (id TEXT PRIMARY KEY, name TEXT, inactive INTEGER)`, `CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY UNIQUE, id TEXT PRIMARY KEY, name TEXT, inactive INTEGER)`), `CREATE TABLE nodes (public_key TEXT PRIMARY KEY)`, `CREATE TABLE inactive_nodes (public_key TEXT PRIMARY KEY)`, } { @@ -104,9 +106,11 @@ func TestServerStartupRequiresMigratedSchema(t *testing.T) { } } + ensurePreparable(t, rw) + // Open the read-only server handle and call AssertReady directly // (production path: main.go does this before any business logic). - d, err := OpenDB(dbPath) + d, err := openFixtureReader(t, dbPath) if err != nil { t.Fatalf("OpenDB: %v", err) } diff --git a/cmd/server/node_battery.go b/cmd/server/node_battery.go index 7e66ac579..9c1ca3943 100644 --- a/cmd/server/node_battery.go +++ b/cmd/server/node_battery.go @@ -53,9 +53,9 @@ func (db *DB) GetNodeBatteryHistory(pubkey, since string) ([]NodeBatterySample, rows, err := db.conn.Query(` SELECT timestamp, battery_mv FROM observer_metrics - WHERE LOWER(observer_id) = ? + WHERE LOWER(observer_id) = `+db.parameter(1)+` AND battery_mv IS NOT NULL - AND timestamp >= ? + AND timestamp >= `+db.parameter(2)+` ORDER BY timestamp ASC`, pk, since) if err != nil { return nil, err diff --git a/cmd/server/node_battery_test.go b/cmd/server/node_battery_test.go index 247bd963a..55edf96d5 100644 --- a/cmd/server/node_battery_test.go +++ b/cmd/server/node_battery_test.go @@ -20,19 +20,19 @@ func TestGetNodeBatteryHistory_FromObserverMetrics(t *testing.T) { // node + observer with matching pubkey (cases differ on purpose) pkLower := "deadbeefcafef00d11223344" idUpper := strings.ToUpper(pkLower) - db.conn.Exec(`INSERT INTO nodes (public_key, name, role, last_seen, first_seen) VALUES (?, 'BatNode', 'repeater', ?, ?)`, + db.conn.Exec(`INSERT INTO nodes (public_key, name, role, last_seen, first_seen) VALUES ($1, 'BatNode', 'repeater', $2, $3)`, pkLower, now.Format(time.RFC3339), now.Add(-72*time.Hour).Format(time.RFC3339)) - db.conn.Exec(`INSERT INTO observers (id, name, last_seen, first_seen) VALUES (?, 'BatNode', ?, ?)`, + db.conn.Exec(`INSERT INTO observers (id, name, last_seen, first_seen) VALUES ($1, 'BatNode', $2, $3)`, idUpper, now.Format(time.RFC3339), now.Add(-72*time.Hour).Format(time.RFC3339)) // 3 metrics samples: 3700, 3500, 3200 mV for i, mv := range []int{3700, 3500, 3200} { ts := now.Add(time.Duration(-2+i) * time.Hour).Format(time.RFC3339) - db.conn.Exec(`INSERT INTO observer_metrics (observer_id, timestamp, battery_mv) VALUES (?, ?, ?)`, + db.conn.Exec(`INSERT INTO observer_metrics (observer_id, timestamp, battery_mv) VALUES ($1, $2, $3)`, idUpper, ts, mv) } // One sample with NULL battery should be skipped - db.conn.Exec(`INSERT INTO observer_metrics (observer_id, timestamp) VALUES (?, ?)`, + db.conn.Exec(`INSERT INTO observer_metrics (observer_id, timestamp) VALUES ($1, $2)`, idUpper, now.Add(-3*time.Hour).Format(time.RFC3339)) since := now.Add(-24 * time.Hour).Format(time.RFC3339) @@ -57,11 +57,11 @@ func TestNodeBatteryEndpoint(t *testing.T) { now := time.Now().UTC() pkLower := "aabbccdd11223344" idUpper := strings.ToUpper(pkLower) - db.conn.Exec(`INSERT INTO observers (id, name, last_seen, first_seen) VALUES (?, 'TestRepeater', ?, ?)`, + db.conn.Exec(`INSERT INTO observers (id, name, last_seen, first_seen) VALUES ($1, 'TestRepeater', $2, $3)`, idUpper, now.Format(time.RFC3339), now.Add(-72*time.Hour).Format(time.RFC3339)) for i, mv := range []int{3800, 3600, 3200} { ts := now.Add(time.Duration(-2+i) * time.Hour).Format(time.RFC3339) - db.conn.Exec(`INSERT INTO observer_metrics (observer_id, timestamp, battery_mv) VALUES (?, ?, ?)`, + db.conn.Exec(`INSERT INTO observer_metrics (observer_id, timestamp, battery_mv) VALUES ($1, $2, $3)`, idUpper, ts, mv) } diff --git a/cmd/server/node_blacklist_test.go b/cmd/server/node_blacklist_test.go index a6d877693..b22d7f0a8 100644 --- a/cmd/server/node_blacklist_test.go +++ b/cmd/server/node_blacklist_test.go @@ -71,8 +71,8 @@ func TestConfigBlacklistEmptyEntries(t *testing.T) { func TestBlacklistFiltersHandleNodes(t *testing.T) { db := setupTestDB(t) - db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role, last_seen) VALUES ('goodnode', 'GoodNode', 'companion', datetime('now'))") - db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role, last_seen) VALUES ('badnode', 'BadNode', 'companion', datetime('now'))") + db.conn.Exec(testNativeSQL(`INSERT INTO nodes (public_key, name, role, last_seen) VALUES ('goodnode', 'GoodNode', 'companion', strftime('%Y-%m-%dT%H:%M:%SZ','now')) ON CONFLICT DO NOTHING`, "INSERT INTO nodes (public_key, name, role, last_seen) VALUES ('goodnode', 'GoodNode', 'companion', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')) ON CONFLICT DO NOTHING")) + db.conn.Exec(testNativeSQL(`INSERT INTO nodes (public_key, name, role, last_seen) VALUES ('badnode', 'BadNode', 'companion', strftime('%Y-%m-%dT%H:%M:%SZ','now')) ON CONFLICT DO NOTHING`, "INSERT INTO nodes (public_key, name, role, last_seen) VALUES ('badnode', 'BadNode', 'companion', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')) ON CONFLICT DO NOTHING")) cfg := &Config{ NodeBlacklist: []string{"badnode"}, @@ -105,7 +105,7 @@ func TestBlacklistFiltersHandleNodes(t *testing.T) { func TestBlacklistFiltersNodeDetail(t *testing.T) { db := setupTestDB(t) - db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role, last_seen) VALUES ('badnode', 'BadNode', 'companion', datetime('now'))") + db.conn.Exec(testNativeSQL(`INSERT INTO nodes (public_key, name, role, last_seen) VALUES ('badnode', 'BadNode', 'companion', strftime('%Y-%m-%dT%H:%M:%SZ','now')) ON CONFLICT DO NOTHING`, "INSERT INTO nodes (public_key, name, role, last_seen) VALUES ('badnode', 'BadNode', 'companion', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')) ON CONFLICT DO NOTHING")) cfg := &Config{ NodeBlacklist: []string{"badnode"}, @@ -124,8 +124,8 @@ func TestBlacklistFiltersNodeDetail(t *testing.T) { func TestBlacklistFiltersNodeSearch(t *testing.T) { db := setupTestDB(t) - db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role, last_seen) VALUES ('badnode', 'TrollNode', 'companion', datetime('now'))") - db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role, last_seen) VALUES ('goodnode', 'GoodNode', 'companion', datetime('now'))") + db.conn.Exec(testNativeSQL(`INSERT INTO nodes (public_key, name, role, last_seen) VALUES ('badnode', 'TrollNode', 'companion', strftime('%Y-%m-%dT%H:%M:%SZ','now')) ON CONFLICT DO NOTHING`, "INSERT INTO nodes (public_key, name, role, last_seen) VALUES ('badnode', 'TrollNode', 'companion', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')) ON CONFLICT DO NOTHING")) + db.conn.Exec(testNativeSQL(`INSERT INTO nodes (public_key, name, role, last_seen) VALUES ('goodnode', 'GoodNode', 'companion', strftime('%Y-%m-%dT%H:%M:%SZ','now')) ON CONFLICT DO NOTHING`, "INSERT INTO nodes (public_key, name, role, last_seen) VALUES ('goodnode', 'GoodNode', 'companion', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')) ON CONFLICT DO NOTHING")) cfg := &Config{ NodeBlacklist: []string{"badnode"}, @@ -155,7 +155,7 @@ func TestBlacklistFiltersNodeSearch(t *testing.T) { func TestNoBlacklistPassesAll(t *testing.T) { db := setupTestDB(t) - db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role, last_seen) VALUES ('somenode', 'SomeNode', 'companion', datetime('now'))") + db.conn.Exec(testNativeSQL(`INSERT INTO nodes (public_key, name, role, last_seen) VALUES ('somenode', 'SomeNode', 'companion', strftime('%Y-%m-%dT%H:%M:%SZ','now')) ON CONFLICT DO NOTHING`, "INSERT INTO nodes (public_key, name, role, last_seen) VALUES ('somenode', 'SomeNode', 'companion', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')) ON CONFLICT DO NOTHING")) cfg := &Config{} srv := NewServer(db, cfg, NewHub()) @@ -234,7 +234,7 @@ func TestBlacklistFiltersNeighborGraph(t *testing.T) { func TestBlacklistFiltersResolveHops(t *testing.T) { db := setupTestDB(t) - db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role, last_seen) VALUES ('badnode', 'BadNode', 'companion', datetime('now'))") + db.conn.Exec(testNativeSQL(`INSERT INTO nodes (public_key, name, role, last_seen) VALUES ('badnode', 'BadNode', 'companion', strftime('%Y-%m-%dT%H:%M:%SZ','now')) ON CONFLICT DO NOTHING`, "INSERT INTO nodes (public_key, name, role, last_seen) VALUES ('badnode', 'BadNode', 'companion', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')) ON CONFLICT DO NOTHING")) cfg := &Config{ NodeBlacklist: []string{"badnode"}, diff --git a/cmd/server/node_declared_regions_api_test.go b/cmd/server/node_declared_regions_api_test.go new file mode 100644 index 000000000..3c79ed471 --- /dev/null +++ b/cmd/server/node_declared_regions_api_test.go @@ -0,0 +1,293 @@ +package main + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "reflect" + "strings" + "testing" +) + +// declaredRegionsOf reads the declared_regions field back as a []string, and +// reports whether the field was present at all. Absent and present-but-empty +// are different answers (never asked vs answered with no named region), so +// the helper keeps them apart instead of collapsing both to nil. +func declaredRegionsOf(t *testing.T, node map[string]interface{}) ([]string, bool) { + t.Helper() + raw, present := node["declared_regions"] + if !present { + return nil, false + } + list, ok := raw.([]interface{}) + if !ok { + t.Fatalf("declared_regions is %T (%v), want a JSON array", raw, raw) + } + out := make([]string, 0, len(list)) + for _, v := range list { + s, ok := v.(string) + if !ok { + t.Fatalf("declared_regions entry is %T (%v), want a string", v, v) + } + out = append(out, s) + } + return out, true +} + +// TestHandleNodesExposesDeclaredRegions pins the field the map's region +// filter reads (#1862): the named regions a repeater declared, spelled the way +// the Scope Audit spells them ('#' stripped, the wildcard left out), so the +// map can answer "which repeaters handle #be" from the one bulk fetch it +// already makes. +func TestHandleNodesExposesDeclaredRegions(t *testing.T) { + srv, router := setupScopeConfigStateServer(t) + + if _, err := srv.db.conn.Exec(`INSERT INTO nodes + (public_key, name, role, lat, lon, last_seen, first_seen, advert_count, configured_scope, configured_scope_at) + VALUES + ('PK_NAMED', 'named-rp', 'repeater', 51.0, 4.0, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z', 1, '#be,*,be-bru', '2026-01-01T00:00:00Z'), + ('PK_STARONLY', 'star-rp', 'repeater', 51.1, 4.1, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z', 1, '*', '2026-01-01T00:00:00Z'), + ('PK_NEVER', 'never-rp', 'repeater', 51.2, 4.2, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z', 1, NULL, NULL), + ('PK_COMPANION', 'phone', 'companion', 51.3, 4.3, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z', 1, 'be,*', '2026-01-01T00:00:00Z')`, + ); err != nil { + t.Fatal(err) + } + + nodes := nodesByPubkey(t, router, "?limit=200") + + got, present := declaredRegionsOf(t, nodes["PK_NAMED"]) + if !present { + t.Fatalf("declared repeater has no declared_regions field: %v", nodes["PK_NAMED"]) + } + if want := []string{"be", "be-bru"}; !reflect.DeepEqual(got, want) { + t.Errorf("declared_regions = %v, want %v ('#' stripped, '*' left out, list order kept)", got, want) + } + + // Answered with only the wildcard: the answer exists and names no region. + // That is an empty list, not an absent field. + got, present = declaredRegionsOf(t, nodes["PK_STARONLY"]) + if !present { + t.Errorf("wildcard-only repeater has no declared_regions field, want an empty list") + } else if len(got) != 0 { + t.Errorf("wildcard-only repeater declared_regions = %v, want []", got) + } + + // Never asked: no answer to report. + if v, present := nodes["PK_NEVER"]["declared_regions"]; present { + t.Errorf("never-asked repeater carries declared_regions = %v, want the field absent", v) + } + + // A companion does not forward, so a declared list on it is not something + // the region filter should read, same rule as scope_config_state. + if v, present := nodes["PK_COMPANION"]["declared_regions"]; present { + t.Errorf("companion carries declared_regions = %v, want the field absent", v) + } +} + +// TestHandleNodeDetailExposesDeclaredRegions keeps the node endpoint and the +// list endpoint saying the same thing about one repeater. +func TestHandleNodeDetailExposesDeclaredRegions(t *testing.T) { + srv, router := setupScopeConfigStateServer(t) + + if _, err := srv.db.conn.Exec(`INSERT INTO nodes + (public_key, name, role, lat, lon, last_seen, first_seen, advert_count, configured_scope, configured_scope_at) + VALUES ('PK_DETAIL_RGN', 'detail-rp', 'repeater', 51.3, 4.3, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z', 1, 'nl,#be', '2026-01-01T00:00:00Z')`, + ); err != nil { + t.Fatal(err) + } + + req := httptest.NewRequest("GET", "/api/nodes/PK_DETAIL_RGN", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", w.Code, w.Body.String()) + } + var resp struct { + Node map[string]interface{} `json:"node"` + } + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatalf("decode: %v", err) + } + got, present := declaredRegionsOf(t, resp.Node) + if !present { + t.Fatalf("detail endpoint has no declared_regions field: %v", resp.Node) + } + if want := []string{"nl", "be"}; !reflect.DeepEqual(got, want) { + t.Errorf("declared_regions = %v, want %v", got, want) + } +} + +// TestHandleNodesOmitsDeclaredRegionsWithoutASource: on a database that cannot +// hold a declared answer, an empty list would claim "answered, names nothing". +func TestHandleNodesOmitsDeclaredRegionsWithoutASource(t *testing.T) { + srv, router := setupTestServer(t) + if srv.db.hasConfiguredScope || srv.db.hasDeclaredRegionsTable { + t.Fatal("fixture has a declared-regions source: this test would prove nothing") + } + if _, err := srv.db.conn.Exec(`INSERT INTO nodes + (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) + VALUES ('PK_NOSOURCE_RGN', 'rp', 'repeater', 51.0, 4.0, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z', 1)`, + ); err != nil { + t.Fatal(err) + } + + nodes := nodesByPubkey(t, router, "?limit=200") + + if v, present := nodes["PK_NOSOURCE_RGN"]["declared_regions"]; present { + t.Errorf("repeater carries declared_regions = %v on a schema with no declared-regions source, want the field absent", v) + } +} + +// TestScopeAuditAndNodesAgreeOnDeclaredRegions drives both real handlers over +// the same stored answers. The region filter on the map and the region search +// on the Scope Audit page must list a repeater under the same declared names, +// or searching "be" on one page and picking #be on the other give different +// repeaters. +func TestScopeAuditAndNodesAgreeOnDeclaredRegions(t *testing.T) { + srv, router := setupScopeConfigStateServer(t) + if _, err := srv.db.conn.Exec(`ALTER TABLE transmissions ADD COLUMN scope_name TEXT`); err != nil { + t.Fatal(err) + } + if err := srv.db.detectSchema(context.Background(), srv.db.conn); err != nil { + t.Fatal(err) + } + + cases := map[string]string{ + "PK_RGN_HASHSTAR": "#be,#*", + "PK_RGN_BARESTAR": "be,*,nl-li", + "PK_RGN_STARONLY": "#*", + "PK_RGN_NAMEDONLY": "#be, de", + "PK_RGN_EMPTY": "", + } + for pk, csv := range cases { + if _, err := srv.db.conn.Exec(`INSERT INTO nodes + (public_key, name, role, lat, lon, last_seen, first_seen, advert_count, configured_scope, configured_scope_at) + VALUES ($1, $2, 'repeater', 51.0, 4.0, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z', 1, $3, '2026-01-01T00:00:00Z')`, + pk, "rp-"+pk, csv, + ); err != nil { + t.Fatal(err) + } + } + + audit := getScopeAudit(t, router, "?window=24h") + auditRegions := map[string][]string{} + for _, row := range audit.Repeaters { + auditRegions[strings.ToLower(row.PublicKey)] = row.DeclaredRegions + } + nodes := nodesByPubkey(t, router, "?limit=200") + + for pk, csv := range cases { + want, ok := auditRegions[strings.ToLower(pk)] + if !ok { + t.Errorf("%s (%q): the scope audit does not list the repeater at all", pk, csv) + continue + } + got, present := declaredRegionsOf(t, nodes[pk]) + if !present { + t.Errorf("regions_csv %q: the scope audit lists %v, the map has no declared_regions field", csv, want) + continue + } + if !reflect.DeepEqual(got, want) { + t.Errorf("regions_csv %q: the scope audit lists %v, the map lists %v", csv, want, got) + } + } +} + +// TestOpenAPINodeSchemaDocumentsDeclaredRegions keeps the served spec in step +// with the wire. +func TestOpenAPINodeSchemaDocumentsDeclaredRegions(t *testing.T) { + spec := fetchSpec(t) + components := asMap(t, spec["components"], "components") + schemas := asMap(t, components["schemas"], "components.schemas") + props := asMap(t, asMap(t, schemas["Node"], "Node")["properties"], "Node.properties") + + p, ok := props["declared_regions"] + if !ok { + t.Fatal("Node.properties.declared_regions missing") + } + pm := asMap(t, p, "declared_regions") + if pm["type"] != "array" { + t.Errorf("declared_regions: want type array, got %v", pm["type"]) + } + items := asMap(t, pm["items"], "declared_regions.items") + if items["type"] != "string" { + t.Errorf("declared_regions.items: want type string, got %v", items["type"]) + } +} + +// TestNodesCarryDeclaredRegionsTruncated: a repeater whose declared list was +// cut off returns a partial list, and the Scope Audit flags it as such. The +// node row must carry the same caveat, or the map popup shows a partial list +// (or "no named region") as the whole answer (#2022 review). +func TestNodesCarryDeclaredRegionsTruncated(t *testing.T) { + srv, router := setupScopeConfigStateServer(t) + createDeclaredRegionsTable(t, srv) + + if _, err := srv.db.conn.Exec(`INSERT INTO nodes + (public_key, name, role, lat, lon, last_seen, first_seen, advert_count, configured_scope, configured_scope_at) + VALUES + ('pk_trunc', 'trunc-rp', 'repeater', 51.0, 4.0, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z', 1, NULL, NULL), + ('pk_trunc_empty', 'trunc-empty-rp', 'room', 51.1, 4.1, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z', 1, NULL, NULL), + ('pk_complete', 'complete-rp', 'repeater', 51.2, 4.2, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z', 1, NULL, NULL), + ('pk_cfg_only', 'cfg-rp', 'repeater', 51.3, 4.3, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z', 1, 'be', '2026-01-01T00:00:00Z'), + ('pk_newer_cfg', 'newer-rp', 'repeater', 51.4, 4.4, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z', 1, 'be,nl', '2026-03-01T00:00:00Z'), + ('pk_companion', 'phone', 'companion', 51.5, 4.5, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z', 1, NULL, NULL)`, + ); err != nil { + t.Fatal(err) + } + seedSecondSource(t, srv, "pk_trunc", "2026-02-01T00:00:00Z", "#be", 1) + seedSecondSource(t, srv, "pk_trunc_empty", "2026-02-01T00:00:00Z", "", 1) + seedSecondSource(t, srv, "pk_complete", "2026-02-01T00:00:00Z", "#be", 0) + // Older truncated answer, newer answer from the other source: the newest + // answer decides, caveat included. + seedSecondSource(t, srv, "pk_newer_cfg", "2026-02-01T00:00:00Z", "#be", 1) + seedSecondSource(t, srv, "pk_companion", "2026-02-01T00:00:00Z", "#be", 1) + + nodes := nodesByPubkey(t, router, "?limit=200") + + for _, pk := range []string{"pk_trunc", "pk_trunc_empty"} { + if _, present := declaredRegionsOf(t, nodes[pk]); !present { + t.Errorf("%s: no declared_regions field", pk) + } + if v, present := nodes[pk]["declared_regions_truncated"]; !present || v != true { + t.Errorf("%s: declared_regions_truncated = %v (present %v), want true", pk, v, present) + } + } + for _, pk := range []string{"pk_complete", "pk_cfg_only", "pk_newer_cfg", "pk_companion"} { + if v, present := nodes[pk]["declared_regions_truncated"]; present { + t.Errorf("%s: declared_regions_truncated = %v, want the field absent", pk, v) + } + } + + req := httptest.NewRequest("GET", "/api/nodes/pk_trunc", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("detail status = %d, want 200: %s", w.Code, w.Body.String()) + } + var resp struct { + Node map[string]interface{} `json:"node"` + } + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatalf("decode: %v", err) + } + if v := resp.Node["declared_regions_truncated"]; v != true { + t.Errorf("detail endpoint declared_regions_truncated = %v, want true", v) + } +} + +func TestOpenAPINodeSchemaDocumentsDeclaredRegionsTruncated(t *testing.T) { + spec := fetchSpec(t) + components := asMap(t, spec["components"], "components") + schemas := asMap(t, components["schemas"], "components.schemas") + props := asMap(t, asMap(t, schemas["Node"], "Node")["properties"], "Node.properties") + + p, ok := props["declared_regions_truncated"] + if !ok { + t.Fatal("Node.properties.declared_regions_truncated missing") + } + if pm := asMap(t, p, "declared_regions_truncated"); pm["type"] != "boolean" { + t.Errorf("declared_regions_truncated: want type boolean, got %v", pm["type"]) + } +} diff --git a/cmd/server/node_health_can_relay_case_1290_test.go b/cmd/server/node_health_can_relay_case_1290_test.go index d2785ba5d..0a3159dd6 100644 --- a/cmd/server/node_health_can_relay_case_1290_test.go +++ b/cmd/server/node_health_can_relay_case_1290_test.go @@ -36,27 +36,44 @@ func TestNodeHealth_CanRelayCaseInsensitive_Issue1290(t *testing.T) { } if _, err := srv.store.db.conn.Exec( `INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count, can_relay, can_relay_seen) - VALUES (?, 'ListenerOnly', 'SJC', ?, '2026-01-01T00:00:00Z', 1, 0, 1)`, + VALUES ($1, 'ListenerOnly', 'SJC', $2, '2026-01-01T00:00:00Z', 1, 0, 1)`, obsIDLower, now); err != nil { t.Fatalf("seed observer: %v", err) } // In-memory packet with the MIXED-case observer id so the badge resolver // must lower-case both sides to match against the lower-cased pubkey set. + // The packet is a flood ADVERT heard with an empty path, which is what + // makes the observer a DIRECT receiver of nodePubkey — only direct rows + // carry the badge (see direct_heard.go). snr := 7.0 - srv.store.mu.Lock() - if srv.store.byNode == nil { - srv.store.byNode = make(map[string][]*StoreTx) - } - srv.store.byNode[nodePubkey] = append(srv.store.byNode[nodePubkey], &StoreTx{ + routeFlood := RouteFlood + payloadAdvert := PayloadADVERT + tx := &StoreTx{ Hash: "1290casebadge00", FirstSeen: now, + RouteType: &routeFlood, + PayloadType: &payloadAdvert, + DecodedJSON: `{"type":"ADVERT","pubKey":"` + nodePubkey + `"}`, SNR: &snr, ObservationCount: 1, ObserverID: obsIDMixed, ObserverName: "ListenerOnly", - }) + Observations: []*StoreObs{{ + ObserverID: obsIDMixed, + ObserverName: "ListenerOnly", + PathJSON: "[]", + SNR: &snr, + }}, + } + srv.store.mu.Lock() + if srv.store.byNode == nil { + srv.store.byNode = make(map[string][]*StoreTx) + } + srv.store.byNode[nodePubkey] = append(srv.store.byNode[nodePubkey], tx) + srv.store.packets = append(srv.store.packets, tx) srv.store.mu.Unlock() + srv.store.publishDirectHeard(srv.store.computeDirectHeard()) req := httptest.NewRequest(http.MethodGet, "/api/nodes/"+nodePubkey+"/health", nil) w := httptest.NewRecorder() diff --git a/cmd/server/node_health_direct_rf_test.go b/cmd/server/node_health_direct_rf_test.go new file mode 100644 index 000000000..5084df7a4 --- /dev/null +++ b/cmd/server/node_health_direct_rf_test.go @@ -0,0 +1,134 @@ +package main + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" +) + +// End-to-end guard for the reported bug: a 433 MHz repeater listed 13 +// observers on its detail page, twelve of which run on 868 MHz only and +// cannot physically have heard it. They were credited because the node's +// one-byte pubkey prefix collides with ten other repeaters and +// resolveWithContext picks a winner instead of abstaining. +// +// /api/nodes/{pk}/health must now report only observers that received the +// node's own transmission off the air, and count the rest separately. +func TestNodeHealth_RelayedObserverIsNotHeardBy(t *testing.T) { + srv, router := setupTestServer(t) + const nodePubkey = "aabbccdd11223344" // seeded by seedTestData + now := time.Now().UTC().Format(time.RFC3339) + + routeFlood := RouteFlood + payload := PayloadTXT_MSG + snr := 3.0 + rssi := -110.0 + + // A long flood path that merely passes through the node. The observer + // at the far end saw the packet; it never heard this node. + relayed := &StoreTx{ + Hash: "directrf-relayed", + FirstSeen: now, + RouteType: &routeFlood, + PayloadType: &payload, + SNR: &snr, + RSSI: &rssi, + ObservationCount: 1, + ObserverID: "farawayobserver", + ObserverName: "FarAway", + Observations: []*StoreObs{{ + ObserverID: "farawayobserver", + ObserverName: "FarAway", + PathJSON: `["AABB","1234","5678"]`, + SNR: &snr, + RSSI: &rssi, + }}, + } + + srv.store.mu.Lock() + if srv.store.byNode == nil { + srv.store.byNode = make(map[string][]*StoreTx) + } + srv.store.byNode[nodePubkey] = append(srv.store.byNode[nodePubkey], relayed) + srv.store.packets = append(srv.store.packets, relayed) + srv.store.mu.Unlock() + srv.store.publishDirectHeard(srv.store.computeDirectHeard()) + + req := httptest.NewRequest(http.MethodGet, "/api/nodes/"+nodePubkey+"/health", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("expected 200, got %d (body: %s)", w.Code, w.Body.String()) + } + + var body map[string]interface{} + if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil { + t.Fatalf("json: %v", err) + } + + obs, _ := body["observers"].([]interface{}) + for _, raw := range obs { + row, ok := raw.(map[string]interface{}) + if ok && row["observer_id"] == "farawayobserver" { + t.Fatalf("an observer that only saw relayed traffic was listed as having heard the node: %v", row) + } + } + + relayCount, ok := body["relayObserverCount"].(float64) + if !ok { + t.Fatalf("relayObserverCount missing or not a number: %T %v", + body["relayObserverCount"], body["relayObserverCount"]) + } + if relayCount < 1 { + t.Fatalf("relayObserverCount = %v, want at least 1 (the relayed observer must still be counted)", relayCount) + } +} + +// The walk runs over every observation in the store on each recompute pass. +// The reference deployment holds 2.9M of them; this pins that a full pass +// stays well inside one recompute interval. +func BenchmarkBuildDirectHeardIndex(b *testing.B) { + const ( + txCount = 60000 + perTx = 50 // 3M observations total + hopCount = 8 + ) + nodes := make([]nodeInfo, 0, 64) + for i := 0; i < 64; i++ { + nodes = append(nodes, nodeInfo{ + Role: "repeater", + PublicKey: string([]byte{hexDigit(i / 16), hexDigit(i % 16)}) + "00112233445566778899aabbccddeeff00112233445566778899aabbccddee", + }) + } + pm := buildPrefixMap(nodes) + + routeFlood := RouteFlood + payload := PayloadTXT_MSG + path := `["AABB","1234","5678","9ABC","DEF0","0011","2233","4455"]` + packets := make([]*StoreTx, 0, txCount) + for i := 0; i < txCount; i++ { + obsList := make([]*StoreObs, 0, perTx) + for j := 0; j < perTx; j++ { + obsList = append(obsList, &StoreObs{ObserverID: "obs", PathJSON: path}) + } + packets = append(packets, &StoreTx{ + RouteType: &routeFlood, + PayloadType: &payload, + Observations: obsList, + }) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + _ = buildDirectHeardIndex(packets, pm) + } + _ = hopCount +} + +func hexDigit(v int) byte { + if v < 10 { + return byte('0' + v) + } + return byte('a' + v - 10) +} diff --git a/cmd/server/node_hop_analytics.go b/cmd/server/node_hop_analytics.go new file mode 100644 index 000000000..9ae8b407d --- /dev/null +++ b/cmd/server/node_hop_analytics.go @@ -0,0 +1,331 @@ +package main + +import ( + "strings" + "time" +) + +// Per-node hop count statistics (issue #1812). +// +// A repeater decides whether to forward a flood with +// isFloodHopLimitExceeded (firmware src/helpers/RoutingPolicy.h:15-21), +// comparing getPathHashCount() (src/Packet.h:80, path_len & 63) against +// flood.max, flood.max.unscoped (ROUTE_TYPE_FLOOD only) and flood.max.advert +// (PAYLOAD_TYPE_ADVERT only). Mesh::routeRecvPacket (src/Mesh.cpp:344-350) +// runs that check with n hashes in the path and then writes its own hash at +// index n. So the node's zero-based index in an observed flood path is the hop +// count its flood.max check saw. Firmware refs: meshcore-dev/MeshCore 0679dbef. +// +// Hash rules the attribution relies on: +// - A node forwards a given flood once (wasSeen/markSeen before +// routeRecvPacket, e.g. src/Mesh.cpp:265-285 for adverts), so it sits at +// one index in every observation that contains it. A prefix match at two +// different indices means another node shares the prefix. +// - All hops of one packet share one hash size (src/Packet.h:79), so every +// match in a packet is the same prefix string. +// - An originator marks its own flood seen before sending +// (Mesh::sendFlood, src/Mesh.cpp:651 and :680), so it never appears in +// that path. +// - DIRECT paths are the remaining route and shrink at every hop +// (removeSelfFromPath, src/Mesh.cpp:334-341); the flood.max limits only +// apply to floods (examples/simple_repeater/MyMesh.cpp:436-437). DIRECT +// packets carry no hop count for this purpose and are skipped. + +var ( + hopTagsUnscoped = []string{"flood", "unscoped"} + hopTagsScoped = []string{"flood", "scoped"} + hopTagsUnscopedAdvert = []string{"flood", "unscoped", "advert"} + hopTagsScopedAdvert = []string{"flood", "scoped", "advert"} +) + +// nextPathHop returns the hop string starting after offset pos of a path_json +// array and the offset after it, or ok=false when no hop is left. path_json +// is a JSON array of hex hop strings, which carry no escapes, so hops are read +// between quotes without unmarshalling and without allocating: the scan runs +// under s.mu.RLock for every observation in the window. +func nextPathHop(p string, pos int) (hop string, next int, ok bool) { + open := strings.IndexByte(p[pos:], '"') + if open < 0 { + return "", pos, false + } + start := pos + open + 1 + n := strings.IndexByte(p[start:], '"') + if n < 0 { + return "", pos, false + } + return p[start : start+n], start + n + 1, true +} + +// hopAttributor decides whether a path hop is a given node without using the +// server resolver's tiebreaks (affinity score, GPS distance, advert count, +// pubkey order), which always name a winner among colliding prefixes. +// Candidates are prefixMap.relayCandidates; adjacency is the neighbor graph, +// which the server loads from the ingestor's neighbor_edges, the same table +// the ingestor's resolver reads. Caches live for one request. +type hopAttributor struct { + pm *prefixMap + graph *NeighborGraph + cands map[string][]string + adj map[string]map[string]struct{} + seen []string +} + +func newHopAttributor(pm *prefixMap, graph *NeighborGraph) *hopAttributor { + return &hopAttributor{pm: pm, graph: graph, cands: map[string][]string{}, adj: map[string]map[string]struct{}{}} +} + +// candidates returns the lowercase pubkeys of the relay candidates for a hop. +// The cache is keyed by the hop as given, so wire-case hops are looked up +// without lowercasing a copy. +func (a *hopAttributor) candidates(hop string) []string { + c, ok := a.cands[hop] + if !ok { + for _, n := range a.pm.relayCandidates(hop) { + c = append(c, strings.ToLower(n.PublicKey)) + } + a.cands[hop] = c + } + return c +} + +func (a *hopAttributor) adjacent(anchor, pk string) bool { + nbrs, ok := a.adj[anchor] + if !ok { + nbrs = map[string]struct{}{} + if a.graph != nil { + for _, e := range a.graph.Neighbors(anchor) { + if e.Ambiguous || e.NodeA == "" || e.NodeB == "" { + continue + } + other := e.NodeA + if other == anchor { + other = e.NodeB + } + nbrs[other] = struct{}{} + } + } + a.adj[anchor] = nbrs + } + _, ok = nbrs[pk] + return ok +} + +// strictHopAt resolves the hop at index idx of path p the way the ingestor +// does (cmd/ingestor/path_resolver.go resolvePathWithContext): walking from +// hop 0, a hop with one candidate resolves to it, a hop with several resolves +// only when exactly one of them is a graph neighbor of the previous resolved +// hop (the advert originator for hop 0), and nodes already on the path are +// excluded. An unresolved hop breaks the chain for the next one. +// +// ok is false when p has no hop idx or that hop is not prefix. Otherwise pk is +// the lowercase pubkey the hop resolves to, or "" when it does not resolve, +// and end is the offset in p after hop idx. +func (a *hopAttributor) strictHopAt(p string, idx int, prefix, origin string) (pk string, end int, ok bool) { + anchor := origin + a.seen = a.seen[:0] + if anchor != "" { + a.seen = append(a.seen, anchor) + } + pos := 0 + for i := 0; i <= idx; i++ { + hop, next, found := nextPathHop(p, pos) + if !found || (i == idx && !strings.EqualFold(hop, prefix)) { + return "", 0, false + } + pos = next + cands := a.candidates(hop) + match, survivors := "", 0 + switch { + case len(cands) == 1: + if c := cands[0]; !a.onPath(c) { + match, survivors = c, 1 + } + case len(cands) > 1 && anchor != "" && a.graph != nil: + for _, c := range cands { + if !a.onPath(c) && a.adjacent(anchor, c) { + match = c + survivors++ + } + } + } + if survivors != 1 { + match = "" + } + if i == idx { + return match, pos, true + } + anchor = match + if match != "" { + a.seen = append(a.seen, match) + } + } + return "", 0, false +} + +func (a *hopAttributor) onPath(pk string) bool { + for _, s := range a.seen { + if s == pk { + return true + } + } + return false +} + +// computeNodeHopPackets returns one entry per flood packet in txs, first seen +// after fromISO, that the node forwarded, and how many packets carry the node's +// prefix at one index without being attributable to it. +// +// A packet counts when the node's prefix sits at exactly one index across all +// its observations, and that hop is the node independently of the server +// resolver's pick: either the node is the only relay candidate for the prefix, +// or, for a colliding prefix, at least one observation resolves that hop to the +// node under the ingestor's strict neighbor rule (hopAttributor.strictHopAt) +// and none resolves it to another node. Everything else with the node's prefix +// is counted as ambiguous and left out. The answer depends only on the +// observed paths, the prefix map and the neighbor graph, so it is the same +// after a restart as after live ingest. +// +// Every observation is read, not an index: byNode holds the resolver's pick at +// ingest and other picks after a cold load, and byPathHop only indexes each +// packet's longest path, which for a busy relay often runs through another +// branch of the flood (on a live 7-day sample it held 9,995 of the 23,081 +// attributable packets of one repeater, skewed towards higher hop counts). +// +// txs must be deduplicated. Cost is linear in the observations of the flood +// packets in the window: a substring test per observation, the hop scan only +// for observations that contain the node's first prefix byte, and the strict +// walk only for colliding prefixes. +func computeNodeHopPackets(pubkey string, txs []*StoreTx, fromISO string, pm *prefixMap, graph *NeighborGraph) ([]NodeHopPacket, int) { + lowerPK := strings.ToLower(pubkey) + // Every hop hash of the node, whatever its size, starts with its first + // pubkey byte, and path hops are written in upper case + // (internal/packetpath/path.go:50; none of 1,495,712 live observations of + // 7 days had a lower-case hex digit), so an observation without that byte + // after an opening quote holds no hop of the node. + quotedFirstByte := `"` + strings.ToUpper(lowerPK[:2]) + packets := make([]NodeHopPacket, 0) + ambiguous := 0 + attr := newHopAttributor(pm, graph) + onlyCandidate := map[string]bool{} + + for _, tx := range txs { + if tx.RouteType == nil || (*tx.RouteType != RouteFlood && *tx.RouteType != RouteTransportFlood) || tx.FirstSeen <= fromISO { + continue + } + + idx, prefix, conflict := -1, "", false + for _, obs := range tx.Observations { + p := obs.PathJSON + if !strings.Contains(p, quotedFirstByte) { + continue + } + for i, pos := 0, 0; ; i++ { + hop, next, ok := nextPathHop(p, pos) + if !ok { + break + } + pos = next + if len(hop) == 0 || len(hop) > len(lowerPK) || !strings.EqualFold(hop, lowerPK[:len(hop)]) { + continue + } + if idx < 0 { + idx, prefix = i, lowerPK[:len(hop)] + } else if i != idx { + conflict = true + } + } + } + if idx < 0 { + continue + } + origin := "" + if tx.DecodedJSON != "" && strings.Contains(tx.DecodedJSON, "ubKey") { + origin = strings.ToLower(extractFromNode(tx)) + if origin == lowerPK { + continue + } + } + if conflict { + ambiguous++ + continue + } + + advert := tx.PayloadType != nil && *tx.PayloadType == PayloadADVERT + only, known := onlyCandidate[prefix] + if !known { + cands := attr.candidates(prefix) + only = len(cands) == 1 && cands[0] == lowerPK + onlyCandidate[prefix] = only + } + if !only && !attributeByNeighbor(attr, tx, idx, prefix, advert, origin, lowerPK) { + ambiguous++ + continue + } + + tags := hopTagsUnscoped + switch { + case *tx.RouteType == RouteTransportFlood && advert: + tags = hopTagsScopedAdvert + case *tx.RouteType == RouteTransportFlood: + tags = hopTagsScoped + case advert: + tags = hopTagsUnscopedAdvert + } + packets = append(packets, NodeHopPacket{Hash: tx.Hash, Timestamp: tx.FirstSeen, Hops: idx, Tags: tags}) + } + return packets, ambiguous +} + +// attributeByNeighbor reports whether the hop at idx, which carries prefix, is +// the node under the strict neighbor rule in at least one observation and +// another node in none. The ingestor anchors hop 0 on the originator only for +// adverts (cmd/ingestor/db.go FromPubkey), and so does this. An observation +// whose path text equals the previous walked one up to hop idx reuses its +// result. +func attributeByNeighbor(attr *hopAttributor, tx *StoreTx, idx int, prefix string, advert bool, origin, lowerPK string) bool { + if !advert { + origin = "" + } + self := false + walked, pk := "", "" + for _, obs := range tx.Observations { + if walked == "" || !strings.HasPrefix(obs.PathJSON, walked) { + r, end, ok := attr.strictHopAt(obs.PathJSON, idx, prefix, origin) + if !ok { + continue + } + walked, pk = obs.PathJSON[:end], r + } + switch pk { + case "": + case lowerPK: + self = true + default: + return false + } + } + return self +} + +// GetNodeHopAnalytics returns the hop count at this node for every flood packet +// it forwarded in the last days. Returns nil for an unknown node. +func (s *PacketStore) GetNodeHopAnalytics(pubkey string, days int) (*NodeHopAnalyticsResponse, error) { + node, err := s.db.GetNodeByPubkey(pubkey) + if err != nil || node == nil { + return nil, err + } + + now := time.Now() + fromISO := now.Add(-time.Duration(days) * 24 * time.Hour).Format(time.RFC3339) + + s.mu.RLock() + _, pm := s.getCachedNodesAndPM() + packets, ambiguous := computeNodeHopPackets(pubkey, s.packets, fromISO, pm, s.graph.Load()) + s.mu.RUnlock() + + return &NodeHopAnalyticsResponse{ + TimeRange: TimeRangeResp{From: fromISO, To: now.Format(time.RFC3339), Days: days}, + Packets: packets, + Ambiguous: ambiguous, + }, nil +} diff --git a/cmd/server/node_hop_analytics_test.go b/cmd/server/node_hop_analytics_test.go new file mode 100644 index 000000000..a0a41606f --- /dev/null +++ b/cmd/server/node_hop_analytics_test.go @@ -0,0 +1,504 @@ +package main + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "reflect" + "sort" + "testing" + "time" + + "github.com/gorilla/mux" +) + +// Issue #1812: per-node hop count, as the repeater's own flood.max check sees +// it. The firmware compares getPathHashCount() (the number of hashes already in +// the path when the packet arrives) against the limits before appending its own +// hash, so the target's zero-based index in an observed flood path is exactly +// that count. + +const ( + hopTarget = "ab12cd34ef567890" + hopOther = "77aa88bb99cc0011" + hopCollider = "ab99ffee00112233" // shares the 1-byte prefix "ab" with hopTarget + hopListener = "ab12ffff00000000" // shares the 2-byte prefix "ab12", listener only + hopBridge = "88cc000000000000" // neighbor of hopCollider only + hopBoth = "55dd000000000000" // neighbor of hopTarget and hopCollider + hopTwinA = "99aa000000000000" // shares the 1-byte prefix "99" with hopTwinB + hopTwinB = "99bb000000000000" + hopTimestamp = "2026-09-10T10:00:00Z" +) + +func hopTx(id int, hash string, routeType, payloadType int, paths ...string) *StoreTx { + rt, pt := routeType, payloadType + tx := &StoreTx{ID: id, Hash: hash, FirstSeen: hopTimestamp, RouteType: &rt, PayloadType: &pt} + for i, p := range paths { + tx.Observations = append(tx.Observations, &StoreObs{ID: id*10 + i, TransmissionID: id, PathJSON: p}) + } + return tx +} + +func hopPM() *prefixMap { + return buildPrefixMap([]nodeInfo{ + {PublicKey: hopTarget, Role: "repeater"}, + {PublicKey: hopOther, Role: "repeater"}, + {PublicKey: hopCollider, Role: "repeater"}, + {PublicKey: hopBridge, Role: "repeater"}, + {PublicKey: hopBoth, Role: "repeater"}, + {PublicKey: hopTwinA, Role: "repeater"}, + {PublicKey: hopTwinB, Role: "repeater"}, + }) +} + +// hopGraph: hopOther and hopTwinB neighbor hopTarget, hopBridge neighbors +// hopCollider, hopBoth neighbors both, and hopOther and hopBoth neighbor both +// twins. +func hopGraph() *NeighborGraph { + g := NewNeighborGraph() + now := time.Now() + for _, e := range [][2]string{ + {hopOther, hopTarget}, + {hopTwinB, hopTarget}, + {hopBridge, hopCollider}, + {hopBoth, hopTarget}, + {hopBoth, hopCollider}, + {hopOther, hopTwinA}, + {hopOther, hopTwinB}, + {hopBoth, hopTwinA}, + {hopBoth, hopTwinB}, + } { + g.upsertEdge(e[0], e[1], "", "obs", nil, now) + } + return g +} + +func hopsOf(packets []NodeHopPacket) []int { + out := make([]int, 0, len(packets)) + for _, p := range packets { + out = append(out, p.Hops) + } + return out +} + +func hashesOf(packets []NodeHopPacket) []string { + out := make([]string, 0, len(packets)) + for _, p := range packets { + out = append(out, p.Hash) + } + return out +} + +func TestNodeHopPackets_IndexIsHopCountAtDifferentPositions(t *testing.T) { + txs := []*StoreTx{ + hopTx(1, "h1", RouteFlood, PayloadGRP_TXT, `["AB12"]`), + hopTx(2, "h2", RouteFlood, PayloadGRP_TXT, `["77AA","AB12"]`), + hopTx(3, "h3", RouteTransportFlood, PayloadGRP_TXT, `["77AA","77AA","77AA","AB12","77AA"]`), + } + packets, ambiguous := computeNodeHopPackets(hopTarget, txs, "", hopPM(), nil) + if got, want := hopsOf(packets), []int{0, 1, 3}; !reflect.DeepEqual(got, want) { + t.Fatalf("hops = %v, want %v (zero-based index of the node in the path, no +1)", got, want) + } + if ambiguous != 0 { + t.Errorf("ambiguous = %d, want 0", ambiguous) + } + if packets[1].Hash != "h2" || packets[1].Timestamp != hopTimestamp { + t.Errorf("packet[1] = %+v, want hash h2 and timestamp %s", packets[1], hopTimestamp) + } +} + +func TestNodeHopPackets_Tags(t *testing.T) { + txs := []*StoreTx{ + hopTx(1, "unscoped", RouteFlood, PayloadGRP_TXT, `["AB12"]`), + hopTx(2, "scoped", RouteTransportFlood, PayloadGRP_TXT, `["AB12"]`), + hopTx(3, "advert", RouteFlood, PayloadADVERT, `["AB12"]`), + hopTx(4, "scoped-advert", RouteTransportFlood, PayloadADVERT, `["AB12"]`), + } + packets, _ := computeNodeHopPackets(hopTarget, txs, "", hopPM(), nil) + want := map[string][]string{ + "unscoped": {"flood", "unscoped"}, + "scoped": {"flood", "scoped"}, + "advert": {"flood", "unscoped", "advert"}, + "scoped-advert": {"flood", "scoped", "advert"}, + } + if len(packets) != len(want) { + t.Fatalf("got %d packets, want %d", len(packets), len(want)) + } + for _, p := range packets { + if !reflect.DeepEqual(p.Tags, want[p.Hash]) { + t.Errorf("%s tags = %v, want %v", p.Hash, p.Tags, want[p.Hash]) + } + } +} + +func TestNodeHopPackets_DirectRoutesExcluded(t *testing.T) { + txs := []*StoreTx{ + hopTx(1, "direct", RouteDirect, PayloadTXT_MSG, `["77AA","AB12"]`), + hopTx(2, "tdirect", RouteTransportDirect, PayloadTXT_MSG, `["AB12"]`), + } + packets, ambiguous := computeNodeHopPackets(hopTarget, txs, "", hopPM(), nil) + if len(packets) != 0 || ambiguous != 0 { + t.Fatalf("direct packets must be skipped: packets=%v ambiguous=%d", packets, ambiguous) + } +} + +func TestNodeHopPackets_OneEventPerPacketAcrossObservations(t *testing.T) { + txs := []*StoreTx{ + hopTx(1, "h1", RouteFlood, PayloadGRP_TXT, `["77AA","AB12"]`, `["77AA","AB12","77AA"]`, `["77AA"]`), + } + packets, ambiguous := computeNodeHopPackets(hopTarget, txs, "", hopPM(), nil) + if got := hopsOf(packets); !reflect.DeepEqual(got, []int{1}) { + t.Fatalf("hops = %v, want [1] (one event per packet hash)", got) + } + if ambiguous != 0 { + t.Errorf("ambiguous = %d, want 0", ambiguous) + } +} + +func TestNodeHopPackets_NotInPathIsNeitherCountedNorAmbiguous(t *testing.T) { + txs := []*StoreTx{hopTx(1, "h1", RouteFlood, PayloadGRP_TXT, `["77AA"]`, `[]`)} + packets, ambiguous := computeNodeHopPackets(hopTarget, txs, "", hopPM(), nil) + if len(packets) != 0 || ambiguous != 0 { + t.Fatalf("packets=%v ambiguous=%d, want none", packets, ambiguous) + } +} + +func TestNodeHopPackets_OriginatorNeverForwardsOwnFlood(t *testing.T) { + tx := hopTx(1, "own-advert", RouteFlood, PayloadADVERT, `["AB12"]`) + tx.DecodedJSON = `{"pubKey":"` + hopTarget + `"}` + packets, ambiguous := computeNodeHopPackets(hopTarget, []*StoreTx{tx}, "", hopPM(), hopGraph()) + if len(packets) != 0 || ambiguous != 0 { + t.Fatalf("own advert: packets=%v ambiguous=%d, want none", packets, ambiguous) + } +} + +// A colliding prefix counts for the node only under the ingestor's strict rule +// (cmd/ingestor/path_resolver.go resolvePathWithContext): the previous hop is +// itself identified without a tiebreak, and exactly one candidate is its +// graph neighbor. The server resolver's pick (affinity, GPS, advert count) +// plays no part. +func TestNodeHopPackets_CollisionNeedsStrictNeighborAttribution(t *testing.T) { + advert := func(id int, hash, origin string, path string) *StoreTx { + tx := hopTx(id, hash, RouteFlood, PayloadADVERT, path) + tx.DecodedJSON = `{"pubKey":"` + origin + `"}` + return tx + } + txs := []*StoreTx{ + hopTx(1, "after-neighbor", RouteFlood, PayloadGRP_TXT, `["77","AB"]`), + hopTx(2, "no-previous-hop", RouteFlood, PayloadGRP_TXT, `["AB","77"]`), + hopTx(3, "after-collider-neighbor", RouteFlood, PayloadGRP_TXT, `["88","AB"]`), + hopTx(4, "after-common-neighbor", RouteFlood, PayloadGRP_TXT, `["55","AB"]`), + hopTx(5, "previous-hop-ambiguous", RouteFlood, PayloadGRP_TXT, `["99","AB"]`), + hopTx(6, "previous-hop-repeats", RouteFlood, PayloadGRP_TXT, `["77","88","77","AB"]`), + advert(7, "advert-from-neighbor", hopOther, `["AB"]`), + advert(8, "advert-from-stranger", hopBridge, `["AB"]`), + hopTx(9, "observations-disagree", RouteFlood, PayloadGRP_TXT, `["77","AB"]`, `["88","AB"]`), + hopTx(10, "one-observation-unresolved", RouteFlood, PayloadGRP_TXT, `["77","AB"]`, `["99","AB"]`), + hopTx(11, "observations-without-node", RouteFlood, PayloadGRP_TXT, `["77","88"]`, `["77"]`, `["77","AB"]`), + hopTx(12, "chain-broken", RouteFlood, PayloadGRP_TXT, `["77","99","AB"]`), + hopTx(13, "origin-of-non-advert", RouteFlood, PayloadTXT_MSG, `["AB"]`), + advert(14, "originator-excluded-later", hopTwinA, `["55","99","AB"]`), + } + txs[12].DecodedJSON = `{"pubKey":"` + hopOther + `"}` + packets, ambiguous := computeNodeHopPackets(hopTarget, txs, "", hopPM(), hopGraph()) + got := map[string]int{} + for _, p := range packets { + got[p.Hash] = p.Hops + } + want := map[string]int{"after-neighbor": 1, "advert-from-neighbor": 0, "one-observation-unresolved": 1, + "observations-without-node": 1, "originator-excluded-later": 2} + if !reflect.DeepEqual(got, want) { + t.Fatalf("attributed = %v, want %v", got, want) + } + if ambiguous != 9 { + t.Errorf("ambiguous = %d, want 9", ambiguous) + } +} + +func TestNodeHopPackets_CollisionWithoutGraphIsAmbiguous(t *testing.T) { + txs := []*StoreTx{hopTx(1, "h1", RouteFlood, PayloadGRP_TXT, `["77","AB"]`)} + packets, ambiguous := computeNodeHopPackets(hopTarget, txs, "", hopPM(), nil) + if len(packets) != 0 || ambiguous != 1 { + t.Fatalf("packets=%v ambiguous=%d, want none and 1", packets, ambiguous) + } +} + +func TestNodeHopPackets_PrefixAtSeveralPositionsIsAmbiguous(t *testing.T) { + txs := []*StoreTx{ + hopTx(1, "same-path", RouteFlood, PayloadGRP_TXT, `["AB","77","AB"]`), + hopTx(2, "across-obs", RouteFlood, PayloadGRP_TXT, `["AB"]`, `["77","AB"]`), + } + packets, ambiguous := computeNodeHopPackets(hopTarget, txs, "", hopPM(), hopGraph()) + if len(packets) != 0 { + t.Fatalf("packets = %+v, want none: a node forwards a flood once, so two positions mean a collision", packets) + } + if ambiguous != 2 { + t.Errorf("ambiguous = %d, want 2", ambiguous) + } +} + +func TestNodeHopPackets_ListenerDoesNotMakePrefixAmbiguous(t *testing.T) { + pm := buildPrefixMap([]nodeInfo{ + {PublicKey: hopTarget, Role: "repeater"}, + {PublicKey: hopListener, Role: "repeater"}, + }) + pm.markNonRelay([]string{hopListener}) + txs := []*StoreTx{hopTx(1, "h1", RouteFlood, PayloadGRP_TXT, `["AB12","77AA"]`)} + packets, ambiguous := computeNodeHopPackets(hopTarget, txs, "", pm, nil) + if got := hopsOf(packets); !reflect.DeepEqual(got, []int{0}) || ambiguous != 0 { + t.Fatalf("hops=%v ambiguous=%d, want [0] and 0", got, ambiguous) + } +} + +// When the node itself is a listener, the only relay candidate for its prefix +// is another node: nothing may be attributed to it. +func TestNodeHopPackets_OnlyRelayCandidateIsAnotherNode(t *testing.T) { + pm := buildPrefixMap([]nodeInfo{ + {PublicKey: hopTarget, Role: "repeater"}, + {PublicKey: hopCollider, Role: "repeater"}, + }) + pm.markNonRelay([]string{hopTarget}) + txs := []*StoreTx{hopTx(1, "h1", RouteFlood, PayloadGRP_TXT, `["AB"]`)} + packets, ambiguous := computeNodeHopPackets(hopTarget, txs, "", pm, nil) + if len(packets) != 0 || ambiguous != 1 { + t.Fatalf("packets=%v ambiguous=%d, want none and 1", packets, ambiguous) + } +} + +// Uniqueness is per prefix: "ab" collides with hopCollider, "ab12" does not. +func TestNodeHopPackets_UniquenessIsPerPrefixLength(t *testing.T) { + txs := []*StoreTx{ + hopTx(1, "two-byte", RouteFlood, PayloadGRP_TXT, `["AB12"]`), + hopTx(2, "one-byte", RouteFlood, PayloadGRP_TXT, `["AB"]`), + hopTx(3, "two-byte-again", RouteFlood, PayloadGRP_TXT, `["AB12"]`), + hopTx(4, "one-byte-again", RouteFlood, PayloadGRP_TXT, `["AB"]`), + } + packets, ambiguous := computeNodeHopPackets(hopTarget, txs, "", hopPM(), nil) + if got, want := hashesOf(packets), []string{"two-byte", "two-byte-again"}; !reflect.DeepEqual(got, want) { + t.Fatalf("attributed = %v, want %v", got, want) + } + if ambiguous != 2 { + t.Errorf("ambiguous = %d, want 2", ambiguous) + } +} + +// ─── Store level: live ingest and cold load ──────────────────────────────── + +func hopInsertNode(t *testing.T, db *DB, pk, name string, adverts int) { + t.Helper() + mustExec(t, db, `INSERT INTO nodes (public_key, name, role, last_seen, first_seen, advert_count) + VALUES ($1, $2, 'repeater', $3, '2026-01-01', $4)`, pk, name, time.Now().UTC().Format(time.RFC3339), adverts) +} + +func hopInsertTx(t *testing.T, db *DB, id int, hash string, routeType, payloadType int, paths ...string) { + t.Helper() + ts := time.Now().Add(-1 * time.Hour) + mustExec(t, db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type) VALUES ($1, 'AA', $2, $3, $4, $5)`, + id, hash, ts.UTC().Format(time.RFC3339), routeType, payloadType) + for _, p := range paths { + mustExec(t, db, `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp, resolved_path) VALUES ($1, NULL, $2, $3, NULL)`, + id, p, ts.Unix()) + } +} + +func hopLoadedStore(t *testing.T, db *DB) *PacketStore { + t.Helper() + store := NewPacketStore(db, nil) + store.graph.Store(loadNeighborEdgesFromDB(db.conn)) + if err := store.Load(); err != nil { + t.Fatalf("store.Load: %v", err) + } + if !store.WaitIndexesReady(10 * time.Second) { + t.Fatal("indexes not ready") + } + return store +} + +func hopAnalytics(t *testing.T, store *PacketStore, pk string) *NodeHopAnalyticsResponse { + t.Helper() + resp, err := store.GetNodeHopAnalytics(pk, 7) + if err != nil || resp == nil { + t.Fatalf("GetNodeHopAnalytics(%s): resp=%v err=%v", pk, resp, err) + } + sort.Slice(resp.Packets, func(i, j int) bool { return resp.Packets[i].Hash < resp.Packets[j].Hash }) + return resp +} + +// Review repro: live ingest resolves a colliding hop with a tiebreak (here the +// advert count). That pick must not become a hop count. +func TestNodeHopAnalytics_LiveIngestDoesNotTrustResolverPick(t *testing.T) { + db := setupTestDB(t) + hopInsertNode(t, db, hopTarget, "Target", 500) + hopInsertNode(t, db, hopCollider, "Collider", 1) + store := hopLoadedStore(t, db) + + hopInsertTx(t, db, 1, "collides", RouteFlood, PayloadGRP_TXT, `["AB"]`) + if _, maxID := store.IngestNewFromDB(0, 100); maxID != 1 { + t.Fatalf("IngestNewFromDB maxID = %d, want 1", maxID) + } + + for _, pk := range []string{hopTarget, hopCollider} { + resp := hopAnalytics(t, store, pk) + if len(resp.Packets) != 0 || resp.Ambiguous != 1 { + t.Errorf("%s: packets=%+v ambiguous=%d, want none and 1", pk, resp.Packets, resp.Ambiguous) + } + } +} + +// Live ingest and a cold load of the same DB must give the same answer: hops +// are read from every observation's raw path and attributed from the prefix +// map and neighbor graph, not from what each load path indexed. +func TestNodeHopAnalytics_SameResultAfterRestart(t *testing.T) { + db := setupTestDB(t) + mustExec(t, db, `CREATE TABLE neighbor_edges (node_a TEXT NOT NULL, node_b TEXT NOT NULL, + count INTEGER DEFAULT 1, last_seen TEXT, PRIMARY KEY (node_a, node_b))`) + mustExec(t, db, `INSERT INTO neighbor_edges (node_a, node_b, count, last_seen) VALUES ($1, $2, 50, $3)`, + hopOther, hopTarget, time.Now().UTC().Format(time.RFC3339)) + hopInsertNode(t, db, hopTarget, "Target", 1) + hopInsertNode(t, db, hopCollider, "Collider", 500) + hopInsertNode(t, db, hopOther, "Other", 1) + live := hopLoadedStore(t, db) + + hopInsertTx(t, db, 1, "after-neighbor", RouteFlood, PayloadGRP_TXT, `["77","AB"]`, `["77","AB","99"]`) + hopInsertTx(t, db, 2, "no-previous-hop", RouteFlood, PayloadGRP_TXT, `["AB"]`) + hopInsertTx(t, db, 3, "two-byte", RouteTransportFlood, PayloadGRP_TXT, `["77AA","AB12"]`) + hopInsertTx(t, db, 4, "three-byte", RouteFlood, PayloadGRP_TXT, `["77AA88","77AA88","AB12CD"]`) + // The longest observation runs through another branch of the flood. + hopInsertTx(t, db, 5, "not-on-longest-path", RouteFlood, PayloadGRP_TXT, `["77AA","77AA","77AA"]`, `["77AA","AB12"]`) + live.IngestNewFromDB(0, 100) + fromLive := hopAnalytics(t, live, hopTarget) + + restarted := hopAnalytics(t, hopLoadedStore(t, db), hopTarget) + + want := []NodeHopPacket{ + {Hash: "after-neighbor", Hops: 1, Tags: hopTagsUnscoped}, + {Hash: "not-on-longest-path", Hops: 1, Tags: hopTagsUnscoped}, + {Hash: "three-byte", Hops: 2, Tags: hopTagsUnscoped}, + {Hash: "two-byte", Hops: 1, Tags: hopTagsScoped}, + } + for name, resp := range map[string]*NodeHopAnalyticsResponse{"live ingest": fromLive, "cold load": restarted} { + got := make([]NodeHopPacket, len(resp.Packets)) + for i, p := range resp.Packets { + got[i] = NodeHopPacket{Hash: p.Hash, Hops: p.Hops, Tags: p.Tags} + } + if !reflect.DeepEqual(got, want) || resp.Ambiguous != 1 { + t.Errorf("%s: packets=%+v ambiguous=%d, want %+v and 1", name, got, resp.Ambiguous, want) + } + } + if !reflect.DeepEqual(fromLive.Packets, restarted.Packets) || fromLive.Ambiguous != restarted.Ambiguous { + t.Errorf("live ingest %+v/%d differs from cold load %+v/%d", + fromLive.Packets, fromLive.Ambiguous, restarted.Packets, restarted.Ambiguous) + } +} + +// End to end through the route: days window, response shape, 404. +func TestHandleNodeHopAnalytics(t *testing.T) { + db := setupTestDB(t) + recent := time.Now().Add(-1 * time.Hour).Format(time.RFC3339) + recentEpoch := time.Now().Add(-1 * time.Hour).Unix() + old := time.Now().Add(-3 * 24 * time.Hour).Format(time.RFC3339) + oldEpoch := time.Now().Add(-3 * 24 * time.Hour).Unix() + + hopInsertNode(t, db, hopTarget, "Target", 1) + hopInsertNode(t, db, hopOther, "Other", 1) + hopInsertNode(t, db, hopCollider, "Collider", 1) + + mustExec(t, db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type) VALUES (1, 'AA', 'hop_recent_2', $1, 1, 5)`, recent) + mustExec(t, db, `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp) VALUES (1, NULL, '["77AA","77AA","AB12"]', $1)`, recentEpoch) + mustExec(t, db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type) VALUES (2, 'BB', 'hop_recent_1byte', $1, 0, 4)`, recent) + mustExec(t, db, `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp) VALUES (2, NULL, '["AB"]', $1)`, recentEpoch) + mustExec(t, db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type) VALUES (3, 'CC', 'hop_scoped_advert', $1, 0, 4)`, recent) + mustExec(t, db, `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp) VALUES (3, NULL, '["AB12"]', $1)`, recentEpoch) + mustExec(t, db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type) VALUES (4, 'DD', 'hop_old', $1, 1, 5)`, old) + mustExec(t, db, `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp) VALUES (4, NULL, '["AB12"]', $1)`, oldEpoch) + + srv := NewServer(db, &Config{Port: 3000}, NewHub()) + router := mux.NewRouter() + srv.RegisterRoutes(router) + get := func(url string) *httptest.ResponseRecorder { + w := httptest.NewRecorder() + router.ServeHTTP(w, httptest.NewRequest("GET", url, nil)) + return w + } + + srv.store = hopLoadedStore(t, db) + + w := get("/api/nodes/" + hopTarget + "/hop_analytics?days=1") + if w.Code != http.StatusOK { + t.Fatalf("code=%d body=%s", w.Code, w.Body.String()) + } + var resp NodeHopAnalyticsResponse + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatalf("unmarshal: %v", err) + } + byHash := map[string]NodeHopPacket{} + for _, p := range resp.Packets { + byHash[p.Hash] = p + } + if len(resp.Packets) != 2 { + t.Fatalf("packets = %+v, want the two attributable packets inside the 1-day window", resp.Packets) + } + if p := byHash["hop_recent_2"]; p.Hops != 2 || p.Timestamp != recent || !reflect.DeepEqual(p.Tags, []string{"flood", "unscoped"}) { + t.Errorf("hop_recent_2 = %+v, want hops 2, timestamp %s, tags [flood unscoped]", p, recent) + } + if p := byHash["hop_scoped_advert"]; p.Hops != 0 || !reflect.DeepEqual(p.Tags, []string{"flood", "scoped", "advert"}) { + t.Errorf("hop_scoped_advert = %+v, want hops 0 tags [flood scoped advert]", p) + } + if resp.TimeRange.Days != 1 || resp.Ambiguous != 1 { + t.Errorf("timeRange.days=%d ambiguous=%d, want 1 and 1 (colliding 1-byte prefix, no neighbor graph)", resp.TimeRange.Days, resp.Ambiguous) + } + + if w := get("/api/nodes/" + hopTarget + "/hop_analytics?days=7"); w.Code != http.StatusOK || !json.Valid(w.Body.Bytes()) { + t.Fatalf("days=7: code=%d", w.Code) + } else { + var wide NodeHopAnalyticsResponse + _ = json.Unmarshal(w.Body.Bytes(), &wide) + if len(wide.Packets) != 3 { + t.Errorf("days=7 packets = %d, want 3", len(wide.Packets)) + } + } + + if w := get("/api/nodes/ffffffffffffffff/hop_analytics"); w.Code != http.StatusNotFound { + t.Errorf("unknown node: code=%d, want 404", w.Code) + } +} + +// BenchmarkNodeHopPackets sizes one request for a busy repeater over 7 days, +// as counted in a 1,669-node mesh DB on 2026-09-13: 73,782 flood packets +// with 1,430,280 observations (19 per packet). A third of the packets pass the +// node under a unique 2-byte prefix, a sixth under a 1-byte prefix shared with +// another relay (strict neighbor walk), the rest do not pass it; in each, half +// of the observations contain the node. Observation slices are shared between +// packets to keep the fixture small; the scan only reads them. +func BenchmarkNodeHopPackets(b *testing.B) { + obsOf := func(self, other string) []*StoreObs { + through := `["` + other + `","` + self + `","` + other + `","` + other + `","` + other + `","` + other + `"]` + elsewhere := `["` + other + `","` + other + `","` + other + `","` + other + `","` + other + `","` + other + `"]` + obs := make([]*StoreObs, 19) + for j := range obs { + obs[j] = &StoreObs{PathJSON: elsewhere} + if j%2 == 0 { + obs[j].PathJSON = through + } + } + return obs + } + unique, colliding, elsewhere := obsOf("AB12", "77AA"), obsOf("AB", "77"), obsOf("88CC", "77AA") + rt, pt := RouteFlood, PayloadGRP_TXT + txs := make([]*StoreTx, 73782) + for i := range txs { + txs[i] = &StoreTx{ID: i, Hash: "h", FirstSeen: hopTimestamp, RouteType: &rt, PayloadType: &pt, Observations: elsewhere} + switch i % 6 { + case 0, 1: + txs[i].Observations = unique + case 2: + txs[i].Observations = colliding + } + } + pm, graph := hopPM(), hopGraph() + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + computeNodeHopPackets(hopTarget, txs, "", pm, graph) + } +} diff --git a/cmd/server/node_reach.go b/cmd/server/node_reach.go index 8cb1d9283..20798e44c 100644 --- a/cmd/server/node_reach.go +++ b/cmd/server/node_reach.go @@ -4,6 +4,8 @@ import ( "context" "database/sql" "encoding/json" + "errors" + "fmt" "log" "net/http" "sort" @@ -187,6 +189,48 @@ func newResolver(pm *prefixMap) func(string) string { } } +// reachNodeScopes returns the node's inferred default_scope and, when an +// observer /neighbors report has confirmed it, the configured_scope and the +// instant that confirmation was reported (#1865). +// +// Both columns are optional: they are added by migration and an older database +// simply lacks them, so each is gated on the schema flag the handle probed at +// open time. Every return value is best-effort — a missing row, a missing +// column or a scan error yields empty strings, because a reach report must +// still render for a node we know nothing else about. +func (s *Server) reachNodeScopes(ctx context.Context, pubkey string) (string, string, string) { + if s.db == nil || s.db.conn == nil { + return "", "", "" + } + cols := make([]string, 0, 3) + if s.db.hasDefaultScope { + cols = append(cols, "default_scope") + } + if s.db.hasConfiguredScope { + cols = append(cols, "configured_scope", "configured_scope_at") + } + if len(cols) == 0 { + return "", "", "" + } + vals := make([]sql.NullString, len(cols)) + args := make([]interface{}, len(cols)) + for i := range vals { + args[i] = &vals[i] + } + row := s.db.conn.QueryRowContext(ctx, + "SELECT "+strings.Join(cols, ", ")+" FROM nodes WHERE public_key = "+s.db.parameter(1), pubkey) + if err := row.Scan(args...); err != nil { + return "", "", "" + } + out := map[string]string{} + for i, c := range cols { + if vals[i].Valid { + out[c] = vals[i].String + } + } + return out["default_scope"], out["configured_scope"], out["configured_scope_at"] +} + type NodeReachInfo struct { Pubkey string `json:"pubkey"` Name string `json:"name"` @@ -194,6 +238,15 @@ type NodeReachInfo struct { Lat *float64 `json:"lat"` Lon *float64 `json:"lon"` FirstSeen string `json:"first_seen"` + // #1865: region scopes, so a printed reach report says which region the + // node actually serves. DefaultScope is INFERRED from the transport scope + // of observed adverts; ConfiguredScope is CONFIRMED, read back off the node + // itself via an observer /neighbors report. Both omitempty: an instance + // whose schema predates either column, or a node that has never been + // confirmed, sends neither field and the UI shows nothing. + DefaultScope string `json:"default_scope,omitempty"` + ConfiguredScope string `json:"configured_scope,omitempty"` + ConfiguredScopeAt string `json:"configured_scope_at,omitempty"` } type NodeReachWindow struct { Days int `json:"days"` @@ -280,6 +333,12 @@ type reachCacheEntry struct { type reachState struct { cacheMu sync.RWMutex cache map[string]reachCacheEntry + // buildSem bounds how many cold-cache reach scans run at the same time. + // singleflight only collapses identical keys; distinct (pubkey, days) + // keys each start a scan, and the endpoint is unauthenticated, so + // without a cap a handful of requests can hold every database reader. + buildSemOnce sync.Once + buildSem chan struct{} // sf dedups concurrent cold-cache requests for the same key so N // simultaneous callers run the scan + attribution once, not N times. sf singleflight.Group @@ -441,6 +500,11 @@ func (s *Server) handleNodeReach(w http.ResponseWriter, r *http.Request) { if raw, ok := s.reachCacheGet(cacheKey); ok { return raw, nil } + release, ok := s.reachAcquireBuildSlot() + if !ok { + return nil, errReachBusy + } + defer release() resp, ok, cErr := s.computeNodeReach(r.Context(), pubkey, days) if cErr != nil { // Real backend failure (e.g. DB scan exploded) — propagate so the @@ -459,6 +523,11 @@ func (s *Server) handleNodeReach(w http.ResponseWriter, r *http.Request) { s.reachCachePut(cacheKey, raw) return raw, nil }) + if errors.Is(err, errReachBusy) { + w.Header().Set("Retry-After", strconv.Itoa(reachBusyRetryAfterSeconds)) + writeError(w, http.StatusTooManyRequests, "too many reach computations in progress, retry shortly") + return + } if err != nil { writeError(w, 500, "reach computation failed") return @@ -512,6 +581,10 @@ func (s *Server) computeNodeReach(ctx context.Context, pubkey string, days int) // observer-only or pre-first_seen-schema). firstSeen := self.FirstSeen + // #1865: scopes for the report header. Single-row lookup rather than widening + // buildNodeInfoMap's bulk SELECT, which every other consumer would pay for. + defScope, confScope, confAt := s.reachNodeScopes(ctx, pubkey) + // assemble links links := make([]NodeReachLink, 0, len(d.we)+len(d.they)) bidir := 0 @@ -576,7 +649,8 @@ func (s *Server) computeNodeReach(ctx context.Context, pubkey string, days int) selfLat, selfLon := gpsPtrs(self) return NodeReachResponse{ Node: NodeReachInfo{Pubkey: pubkey, Name: self.Name, Role: self.Role, - Lat: selfLat, Lon: selfLon, FirstSeen: firstSeen}, + Lat: selfLat, Lon: selfLon, FirstSeen: firstSeen, + DefaultScope: defScope, ConfiguredScope: confScope, ConfiguredScopeAt: confAt}, Window: NodeReachWindow{Days: days, Since: since.Format(time.RFC3339)}, ReliableTokens: toks, Importance: NodeReachImportance{ @@ -662,6 +736,23 @@ func (s *Server) getDegreeSnapshot(ctx context.Context) *degreeSnapshot { return snap } +// reachScanSQL builds the windowed reach scan. Arguments are the window +// start, one LIKE pattern per token, and the hard row limit. +// SQLite CROSS JOIN keeps the timestamp window as the outer loop (#2155). +// PostgreSQL retains its ordinary JOIN syntax and native planner. +func (db *DB) reachScanSQL(nTokens int) string { + likes := make([]string, nTokens) + for i := range likes { + likes[i] = fmt.Sprintf(db.nativeSQL("o.path_json LIKE %s", "o.path_json ILIKE %s ESCAPE ''"), db.parameter(i+2)) + } + return `SELECT LOWER(COALESCE(obs.id,'')), LOWER(COALESCE(t.from_pubkey,'')), COALESCE(t.payload_type,0), o.path_json, o.snr + FROM observations o + ` + db.nativeSQL("CROSS JOIN", "JOIN") + ` transmissions t ON t.id = o.transmission_id + LEFT JOIN observers obs ON obs.rowid = o.observer_idx + WHERE o.timestamp >= ` + db.parameter(1) + ` AND (` + strings.Join(likes, " OR ") + `) + LIMIT ` + db.parameter(nTokens+2) +} + // scanReachRows reads windowed observations whose path contains any reliable // token, with the originator + observer + snr needed for attribution. Observer // id and originator pubkey are lowercased in SQL (not per row), the path slice @@ -676,7 +767,6 @@ func (s *Server) scanReachRows(ctx context.Context, tokens map[string]bool, sinc if len(tokens) == 0 { return nil, nil // defensive: an empty LIKE chain would render `AND ()` (SQL error) } - likes := make([]string, 0, len(tokens)) args := []interface{}{sinceEpoch} // Sort tokens so the generated SQL text is byte-stable across requests // with the same token set — preserves the driver's prepared-statement @@ -687,17 +777,10 @@ func (s *Server) scanReachRows(ctx context.Context, tokens map[string]bool, sinc } sort.Strings(toks) for _, tok := range toks { - likes = append(likes, "o.path_json LIKE ?") args = append(args, "%\""+tok+"\"%") } - q := `SELECT LOWER(COALESCE(obs.id,'')), LOWER(COALESCE(t.from_pubkey,'')), COALESCE(t.payload_type,0), o.path_json, o.snr - FROM observations o - JOIN transmissions t ON t.id = o.transmission_id - LEFT JOIN observers obs ON obs.rowid = o.observer_idx - WHERE o.timestamp >= ? AND (` + strings.Join(likes, " OR ") + `) - LIMIT ?` args = append(args, reachScanRowLimit) - rows, err := s.db.conn.QueryContext(ctx, q, args...) + rows, err := s.db.conn.QueryContext(ctx, s.db.reachScanSQL(len(toks)), args...) if err != nil { log.Printf("[reach] scan query failed: %v", err) return nil, err @@ -736,3 +819,26 @@ func (s *Server) scanReachRows(ctx context.Context, tokens map[string]bool, sinc } return out, nil } + +// reachMaxConcurrentBuilds is the number of cold-cache reach scans allowed at +// once. Two keeps half of the default 4-connection PostgreSQL pool free for +// every other handler while a scan runs. +const reachMaxConcurrentBuilds = 2 + +// reachBusyRetryAfterSeconds is the Retry-After we send with a 429. +const reachBusyRetryAfterSeconds = 5 + +// errReachBusy is returned inside the singleflight when no build slot is free. +var errReachBusy = errors.New("reach: too many concurrent builds") + +// reachAcquireBuildSlot takes a build slot without blocking. It returns a +// release func and true, or nil and false when all slots are busy. +func (s *Server) reachAcquireBuildSlot() (func(), bool) { + s.reach.buildSemOnce.Do(func() { s.reach.buildSem = make(chan struct{}, reachMaxConcurrentBuilds) }) + select { + case s.reach.buildSem <- struct{}{}: + return func() { <-s.reach.buildSem }, true + default: + return nil, false + } +} diff --git a/cmd/server/node_reach_bench_test.go b/cmd/server/node_reach_bench_test.go index 4159c80ed..12db04bbb 100644 --- a/cmd/server/node_reach_bench_test.go +++ b/cmd/server/node_reach_bench_test.go @@ -2,11 +2,8 @@ package main import ( "context" - "database/sql" "fmt" "testing" - - _ "modernc.org/sqlite" ) // benchReachDB builds an in-memory DB with nObs observations. matchEvery @@ -24,14 +21,14 @@ func benchReachDB(b *testing.B, nObs, matchEvery int, lowerHops bool) *DB { // path_json is uppercase; this measures the worst case Carmack flagged). matchPath, fillerPath = `["aa","01fa","bb"]`, `["aa","cc","bb"]` } - conn, err := sql.Open("sqlite", ":memory:") + conn, err := openPostgresTestDB(b) if err != nil { b.Fatal(err) } schema := []string{ - `CREATE TABLE transmissions (id INTEGER PRIMARY KEY, hash TEXT, first_seen TEXT, payload_type INTEGER, from_pubkey TEXT)`, - `CREATE TABLE observers (id TEXT PRIMARY KEY, name TEXT)`, - `CREATE TABLE observations (id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_idx INTEGER, snr REAL, path_json TEXT, timestamp INTEGER)`, + testNativeSQL(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY AUTOINCREMENT, hash TEXT, first_seen TEXT, payload_type INTEGER, from_pubkey TEXT)`, `CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, hash TEXT, first_seen TEXT, payload_type INTEGER, from_pubkey TEXT)`), + testNativeSQL(`CREATE TABLE observers (id TEXT PRIMARY KEY, name TEXT, inactive INTEGER)`, `CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY UNIQUE, id TEXT PRIMARY KEY, name TEXT, inactive INTEGER)`), + testNativeSQL(`CREATE TABLE observations (id INTEGER PRIMARY KEY AUTOINCREMENT, transmission_id INTEGER, observer_idx INTEGER, snr DOUBLE PRECISION, path_json TEXT, timestamp INTEGER)`, `CREATE TABLE observations (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_idx INTEGER, snr DOUBLE PRECISION, path_json TEXT, timestamp INTEGER)`), `CREATE INDEX idx_obs_ts ON observations(timestamp)`, } for _, s := range schema { @@ -47,7 +44,7 @@ func benchReachDB(b *testing.B, nObs, matchEvery int, lowerHops bool) *DB { b.Fatal(err) } for i := 0; i < nObs; i++ { - if _, err := tx.Exec(`INSERT INTO transmissions (id, hash, first_seen, payload_type, from_pubkey) VALUES (?,?,?,5,'')`, + if _, err := tx.Exec(`INSERT INTO transmissions (id, hash, first_seen, payload_type, from_pubkey) VALUES ($1,$2,$3,5,'')`, i, fmt.Sprintf("h%d", i), "2026-06-07T00:00:00Z"); err != nil { b.Fatal(err) } @@ -55,7 +52,7 @@ func benchReachDB(b *testing.B, nObs, matchEvery int, lowerHops bool) *DB { if i%matchEvery == 0 { path = matchPath } - if _, err := tx.Exec(`INSERT INTO observations (id, transmission_id, observer_idx, snr, path_json, timestamp) VALUES (?,?,1,-7.0,?,?)`, + if _, err := tx.Exec(`INSERT INTO observations (id, transmission_id, observer_idx, snr, path_json, timestamp) VALUES ($1,$2,1,-7.0,$3,$4)`, i, i, path, 1000); err != nil { b.Fatal(err) } @@ -63,7 +60,7 @@ func benchReachDB(b *testing.B, nObs, matchEvery int, lowerHops bool) *DB { if err := tx.Commit(); err != nil { b.Fatal(err) } - return &DB{conn: conn} + return &DB{backend: testBackendValue(), conn: conn} } // BenchmarkNodeReachScan measures the windowed scan + path-decode at increasing @@ -155,16 +152,16 @@ func BenchmarkNodeReachAttribute(b *testing.B) { // must surface an error, not a swallowed nil. Lives in this file because // the bench callers in the same file rely on the same signature. func TestScanReachRows_ErrorReturn(t *testing.T) { - conn, err := sql.Open("sqlite", ":memory:") + conn, err := openPostgresTestDB(t) if err != nil { t.Fatalf("open: %v", err) } // PREFLIGHT: async=true reason="test-only in-memory scratch schema, immediately closed" - if _, err := conn.Exec(`CREATE TABLE observations (id INTEGER); CREATE TABLE transmissions (id INTEGER); CREATE TABLE observers (rowid INTEGER, id TEXT)`); err != nil { + if _, err := conn.Exec(`CREATE TABLE observations (id INTEGER); CREATE TABLE transmissions (id INTEGER); CREATE TABLE observers (rowid INTEGER, id TEXT, inactive INTEGER)`); err != nil { t.Fatalf("schema: %v", err) } conn.Close() // force QueryContext to fail - srv := &Server{db: &DB{conn: conn}} + srv := &Server{db: &DB{backend: testBackendValue(), conn: conn}} rows, err := srv.scanReachRows(context.Background(), map[string]bool{"01FA": true}, 0) if err == nil { t.Fatalf("expected error from closed DB, got nil (rows=%d)", len(rows)) diff --git a/cmd/server/node_reach_concurrency_test.go b/cmd/server/node_reach_concurrency_test.go new file mode 100644 index 000000000..b52aa471f --- /dev/null +++ b/cmd/server/node_reach_concurrency_test.go @@ -0,0 +1,48 @@ +package main + +import ( + "net/http" + "net/http/httptest" + "testing" +) + +// GET /api/nodes/{pubkey}/reach is unauthenticated and a cold-cache request +// starts a scan. Only reachMaxConcurrentBuilds scans may run at once; the +// next cold request gets 429 with Retry-After instead of queueing on the +// shared SQLite pool. +func TestNodeReachConcurrentBuildCap(t *testing.T) { + srv, router := setupTestServerWithAPIKey(t, "") + const key = "ab00000000000000000000000000000000000000000000000000000000000001" + + get := func() *httptest.ResponseRecorder { + req := httptest.NewRequest("GET", "/api/nodes/"+key+"/reach?days=7", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + return w + } + + // Fill every build slot as if scans were in flight. + var releases []func() + for i := 0; i < reachMaxConcurrentBuilds; i++ { + rel, ok := srv.reachAcquireBuildSlot() + if !ok { + t.Fatalf("slot %d should be free", i) + } + releases = append(releases, rel) + } + if w := get(); w.Code != http.StatusTooManyRequests { + t.Fatalf("expected 429 while all build slots are busy, got %d (body: %s)", w.Code, w.Body.String()) + } else if w.Header().Get("Retry-After") == "" { + t.Fatalf("429 must carry Retry-After") + } + + // Free the slots: the same request now runs the scan. The key is not a + // known node, so the normal answer is 404 — the point is that it is no + // longer 429. + for _, rel := range releases { + rel() + } + if w := get(); w.Code == http.StatusTooManyRequests { + t.Fatalf("still 429 after slots were released (body: %s)", w.Body.String()) + } +} diff --git a/cmd/server/node_reach_endpoint_test.go b/cmd/server/node_reach_endpoint_test.go index 6094077b0..14aa8eb87 100644 --- a/cmd/server/node_reach_endpoint_test.go +++ b/cmd/server/node_reach_endpoint_test.go @@ -1,7 +1,6 @@ package main import ( - "database/sql" "encoding/json" "net/http" "net/http/httptest" @@ -11,7 +10,6 @@ import ( "time" "github.com/gorilla/mux" - _ "modernc.org/sqlite" ) func serveReach(srv *Server, path string) *httptest.ResponseRecorder { @@ -57,7 +55,7 @@ func resetReachState(t *testing.T, servers ...*Server) { // build the zero-reach case (identifiable node, no matching observations). func newReachIntegrationDB(t *testing.T, obsPath string) (*DB, string) { t.Helper() - conn, err := sql.Open("sqlite", ":memory:") + conn, err := openPostgresTestDB(t) if err != nil { t.Fatal(err) } @@ -66,10 +64,10 @@ func newReachIntegrationDB(t *testing.T, obsPath string) (*DB, string) { b := pk64("ccdd") // successor → B hears us now := time.Now().Unix() stmts := []string{ - `CREATE TABLE nodes (public_key TEXT, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, advert_count INTEGER)`, - `CREATE TABLE transmissions (id INTEGER PRIMARY KEY, from_pubkey TEXT, payload_type INTEGER)`, - `CREATE TABLE observers (id TEXT)`, - `CREATE TABLE observations (id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_idx INTEGER, snr REAL, path_json TEXT, timestamp INTEGER)`, + `CREATE TABLE nodes (public_key TEXT, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, first_seen TEXT, advert_count INTEGER)`, + testNativeSQL(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY AUTOINCREMENT, from_pubkey TEXT, payload_type INTEGER)`, `CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, from_pubkey TEXT, payload_type INTEGER)`), + testNativeSQL(`CREATE TABLE observers (id TEXT, inactive INTEGER)`, `CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY UNIQUE, id TEXT, inactive INTEGER)`), + testNativeSQL(`CREATE TABLE observations (id INTEGER PRIMARY KEY AUTOINCREMENT, transmission_id INTEGER, observer_idx INTEGER, snr DOUBLE PRECISION, path_json TEXT, timestamp INTEGER)`, `CREATE TABLE observations (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_idx INTEGER, snr DOUBLE PRECISION, path_json TEXT, timestamp INTEGER)`), `CREATE TABLE neighbor_edges (node_a TEXT, node_b TEXT, count INTEGER)`, } for _, s := range stmts { @@ -81,19 +79,19 @@ func newReachIntegrationDB(t *testing.T, obsPath string) (*DB, string) { q string args []interface{} }{ - {`INSERT INTO nodes VALUES (?, 'N', 'repeater', 50.9, 5.4, ?, '2026-06-01T00:00:00Z', 3)`, []interface{}{n, "2026-06-07T00:00:00Z"}}, - {`INSERT INTO nodes VALUES (?, 'A', 'repeater', 51.0, 5.5, ?, '2026-06-01T00:00:00Z', 1)`, []interface{}{a, "2026-06-07T00:00:00Z"}}, - {`INSERT INTO nodes VALUES (?, 'B', 'repeater', 51.1, 5.6, ?, '2026-06-01T00:00:00Z', 1)`, []interface{}{b, "2026-06-07T00:00:00Z"}}, + {`INSERT INTO nodes VALUES ($1, 'N', 'repeater', 50.9, 5.4, $2, '2026-06-01T00:00:00Z', 3)`, []interface{}{n, "2026-06-07T00:00:00Z"}}, + {`INSERT INTO nodes VALUES ($1, 'A', 'repeater', 51.0, 5.5, $2, '2026-06-01T00:00:00Z', 1)`, []interface{}{a, "2026-06-07T00:00:00Z"}}, + {`INSERT INTO nodes VALUES ($1, 'B', 'repeater', 51.1, 5.6, $2, '2026-06-01T00:00:00Z', 1)`, []interface{}{b, "2026-06-07T00:00:00Z"}}, {`INSERT INTO observers (id) VALUES ('OBS1')`, nil}, {`INSERT INTO transmissions (id, from_pubkey, payload_type) VALUES (1, '', 5)`, nil}, - {`INSERT INTO observations (id, transmission_id, observer_idx, snr, path_json, timestamp) VALUES (1,1,1,-7.0,?,?)`, []interface{}{obsPath, now}}, + {`INSERT INTO observations (id, transmission_id, observer_idx, snr, path_json, timestamp) VALUES (1,1,1,-7.0,$1,$2)`, []interface{}{obsPath, now}}, } for _, in := range ins { if _, err := conn.Exec(in.q, in.args...); err != nil { t.Fatal(err) } } - return &DB{conn: conn, isV3: true}, n + return &DB{backend: testBackendValue(), conn: conn, isV3: true}, n } func TestClampDays(t *testing.T) { @@ -232,14 +230,21 @@ func TestNodeReach_ShapeAndClamp(t *testing.T) { // shape (empty — no rows needed for shape/clamp assertions). mustExecDB(t, db, `DROP TABLE observations`) // PREFLIGHT: async=true reason="test-only in-memory schema rebuild; not a production migration" - mustExecDB(t, db, `CREATE TABLE observations ( + mustExecDB(t, db, testNativeSQL(`CREATE TABLE observations ( id INTEGER PRIMARY KEY AUTOINCREMENT, transmission_id INTEGER, observer_idx INTEGER, - snr REAL, + snr DOUBLE PRECISION, path_json TEXT, timestamp INTEGER - )`) + )`, `CREATE TABLE observations ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + transmission_id INTEGER, + observer_idx INTEGER, + snr DOUBLE PRECISION, + path_json TEXT, + timestamp INTEGER + )`)) cfg := &Config{} srv := &Server{store: newTestStoreWithDB(t, db, cfg), db: db, cfg: cfg, perfStats: NewPerfStats()} diff --git a/cmd/server/node_reach_test.go b/cmd/server/node_reach_test.go index 606f8e2c7..4ea0454fd 100644 --- a/cmd/server/node_reach_test.go +++ b/cmd/server/node_reach_test.go @@ -2,25 +2,25 @@ package main import ( "context" - "database/sql" "strconv" + "strings" "testing" - _ "modernc.org/sqlite" + "github.com/meshcore-analyzer/dbconfig" ) // newReachScanTestDB builds a minimal observer_idx-schema DB with two rows whose // path contains "01FA" and one that does not, for scanReachRows coverage. func newReachScanTestDB(t *testing.T) *DB { t.Helper() - conn, err := sql.Open("sqlite", ":memory:") + conn, err := openPostgresTestDB(t) if err != nil { t.Fatal(err) } stmts := []string{ - `CREATE TABLE transmissions (id INTEGER PRIMARY KEY, from_pubkey TEXT, payload_type INTEGER)`, - `CREATE TABLE observers (id TEXT)`, - `CREATE TABLE observations (id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_idx INTEGER, snr REAL, path_json TEXT, timestamp INTEGER)`, + testNativeSQL(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY AUTOINCREMENT, from_pubkey TEXT, payload_type INTEGER)`, `CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, from_pubkey TEXT, payload_type INTEGER)`), + testNativeSQL(`CREATE TABLE observers (id TEXT, inactive INTEGER)`, `CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY UNIQUE, id TEXT, inactive INTEGER)`), + testNativeSQL(`CREATE TABLE observations (id INTEGER PRIMARY KEY AUTOINCREMENT, transmission_id INTEGER, observer_idx INTEGER, snr DOUBLE PRECISION, path_json TEXT, timestamp INTEGER)`, `CREATE TABLE observations (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_idx INTEGER, snr DOUBLE PRECISION, path_json TEXT, timestamp INTEGER)`), `INSERT INTO observers (id) VALUES ('OBS1')`, // rowid 1 `INSERT INTO transmissions (id, from_pubkey, payload_type) VALUES (1,'FF00',4),(2,'',5),(3,'',5)`, `INSERT INTO observations (id, transmission_id, observer_idx, snr, path_json, timestamp) VALUES @@ -33,7 +33,7 @@ func newReachScanTestDB(t *testing.T) *DB { t.Fatal(err) } } - return &DB{conn: conn} + return &DB{backend: testBackendValue(), conn: conn} } // resolver that only resolves the exact tokens it's told are unique. @@ -289,3 +289,71 @@ func TestScanReachRows_DecodesRows(t *testing.T) { t.Fatalf("snr not decoded: valid=%v val=%v", got.snrValid, got.snr) } } + +// Upstream #2155 / #2146: plan using the ANALYZE statistics captured from an +// 18.3M-observation, 1.5M-transmission database, without building that corpus. +func TestReachScanStartsFromObservationsTimeIndex(t *testing.T) { + if testBackend(t) != dbconfig.SQLite { + t.Skip("SQLite planner regression; PostgreSQL uses its native join planner") + } + db := newReachScanTestDB(t) + db.conn.SetMaxOpenConns(1) + for _, query := range []string{ + `CREATE INDEX idx_observations_timestamp ON observations(timestamp)`, + `CREATE INDEX idx_observations_tx_ts ON observations(transmission_id, timestamp)`, + `ANALYZE`, + `DELETE FROM sqlite_stat1`, + `INSERT INTO sqlite_stat1 (tbl, idx, stat) VALUES + ('observations', 'idx_observations_timestamp', '18270402 4'), + ('observations', 'idx_observations_tx_ts', '18270402 11 3'), + ('observations', NULL, '18270402'), + ('transmissions', NULL, '1517599'), + ('observers', NULL, '88')`, + `ANALYZE sqlite_master`, + } { + if _, err := db.conn.Exec(query); err != nil { + t.Fatal(err) + } + } + rows, err := db.conn.Query("EXPLAIN QUERY PLAN "+db.reachScanSQL(2), int64(0), `%"01FA"%`, `%"01FA00"%`, 10) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + var steps []string + for rows.Next() { + var id, parent, unused int + var detail string + if err := rows.Scan(&id, &parent, &unused, &detail); err != nil { + t.Fatal(err) + } + steps = append(steps, detail) + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + if len(steps) == 0 || !strings.Contains(steps[0], "o USING INDEX idx_observations_timestamp") { + t.Fatalf("reach scan must start from observations by timestamp; plan: %q", steps) + } + t.Logf("reach plan: %q", steps) +} + +func TestScanReachRows_WindowAndMultipleTokens(t *testing.T) { + db := newReachScanTestDB(t) + for _, query := range []string{ + `INSERT INTO transmissions (id, from_pubkey, payload_type) VALUES (4,NULL,5),(5,'',5)`, + `INSERT INTO observations (id, transmission_id, observer_idx, snr, path_json, timestamp) VALUES + (4,4,NULL,NULL,'["cafe"]',2000), (5,5,1,2,'["CAFE"]',999)`, + } { + if _, err := db.conn.Exec(query); err != nil { + t.Fatal(err) + } + } + rows, err := (&Server{db: db}).scanReachRows(context.Background(), map[string]bool{"01FA": true, "CAFE": true}, 1500) + if err != nil { + t.Fatal(err) + } + if len(rows) != 1 || len(rows[0].path) != 1 || rows[0].path[0] != "CAFE" || rows[0].observerPK != "" || rows[0].fromPubkey != "" || rows[0].snrValid { + t.Fatalf("window, token binding or nullable metadata changed: %+v", rows) + } +} diff --git a/cmd/server/node_resolve.go b/cmd/server/node_resolve.go index bf5bd5ff6..2dcf43600 100644 --- a/cmd/server/node_resolve.go +++ b/cmd/server/node_resolve.go @@ -41,7 +41,7 @@ func (s *Server) handleResolvePrefix(w http.ResponseWriter, r *http.Request) { } // LIMIT 2: we only need to know unique vs ambiguous. nodes.public_key is the // PK and stored lowercase; pfx is validated hex so the LIKE pattern is safe. - rows, err := s.db.conn.Query(`SELECT public_key, COALESCE(name,'') FROM nodes WHERE public_key LIKE ? LIMIT 2`, pfx+"%") + rows, err := s.db.conn.Query(`SELECT public_key, COALESCE(name,'') FROM nodes WHERE public_key LIKE `+s.db.parameter(1)+` LIMIT 2`, pfx+"%") if err != nil { http.Error(w, "query failed", http.StatusInternalServerError) return diff --git a/cmd/server/nodes_pagination_has_more_test.go b/cmd/server/nodes_pagination_has_more_test.go new file mode 100644 index 000000000..9bbb13ce6 --- /dev/null +++ b/cmd/server/nodes_pagination_has_more_test.go @@ -0,0 +1,92 @@ +package main + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "testing" +) + +// TestNodesHasMoreSurvivesPostLimitFiltering pins the contract that lets a +// client paginate /api/nodes safely. +// +// handleNodes applies the geo-filter, blacklist, hidden-prefix and area passes +// AFTER the SQL LIMIT/OFFSET, and rewrites Total to the filtered length. So a +// page that loses a row is short without being the last page, and neither the +// page length nor Total can tell a client whether to ask for another page. +// has_more is computed from the raw SQL page against the real COUNT(*), before +// those passes run, and must therefore stay true on a page that filtering +// shortened. +// +// Anti-tautology: move the hasMore assignment below the filter block (or +// compute it from the filtered slice) and the page-1 assertion fails — that is +// exactly the arrangement that stranded every node behind a filtered row. +func TestNodesHasMoreSurvivesPostLimitFiltering(t *testing.T) { + srv, router := setupTestServer(t) + + // setupTestServer seeds its own fixture nodes; clear them so the page + // boundaries below are exactly the ones this test sets up. + if _, err := srv.db.conn.Exec(`DELETE FROM nodes`); err != nil { + t.Fatalf("clear fixture nodes: %v", err) + } + + // 7 nodes, newest first by last_seen so page order is deterministic. + for i := 0; i < 7; i++ { + name := fmt.Sprintf("visible-%d", i) + if i == 1 { + name = "🚫 hidden-1" // lands inside page 1 (offset 0, limit 3) + } + lastSeen := fmt.Sprintf("2026-06-0%dT00:00:00Z", 7-i) + if _, err := srv.db.conn.Exec(`INSERT INTO nodes + (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) + VALUES ($1, $2, 'repeater', 0, 0, $3, '2026-06-01T00:00:00Z', 1)`, + fmt.Sprintf("deadbeef0000200%d", i), name, lastSeen); err != nil { + t.Fatalf("insert %d: %v", i, err) + } + } + srv.cfg.SetHiddenNamePrefixes([]string{"🚫"}) + + page := func(offset int) NodeListResponse { + req := httptest.NewRequest("GET", fmt.Sprintf("/api/nodes?limit=3&offset=%d", offset), nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("offset %d: status %d body=%s", offset, w.Code, w.Body.String()) + } + var got NodeListResponse + if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil { + t.Fatalf("offset %d: decode: %v", offset, err) + } + return got + } + + // Page 1 is a row short because the hidden node was dropped after the LIMIT. + first := page(0) + if len(first.Nodes) != 2 { + t.Fatalf("page 1: expected 2 rows after filtering, got %d", len(first.Nodes)) + } + if !first.HasMore { + t.Fatalf("page 1: has_more must stay true on a page shortened by filtering " + + "(4 more nodes are waiting) — a client stopping here strands them") + } + + // Walking has_more reaches every visible node, including the last page. + seen := map[string]bool{} + for offset, more := 0, true; more && offset < 100; offset += 3 { + p := page(offset) + for _, n := range p.Nodes { + pk, _ := n["public_key"].(string) + seen[pk] = true + } + more = p.HasMore + } + if len(seen) != 6 { + t.Fatalf("expected all 6 visible nodes across the walk, got %d", len(seen)) + } + + // The final page reports has_more=false rather than relying on a short page. + if last := page(6); last.HasMore { + t.Fatalf("final page: has_more must be false, got true") + } +} diff --git a/cmd/server/nodes_perf_limit2000_test.go b/cmd/server/nodes_perf_limit2000_test.go index e03a7ad38..7ac64d86b 100644 --- a/cmd/server/nodes_perf_limit2000_test.go +++ b/cmd/server/nodes_perf_limit2000_test.go @@ -41,7 +41,7 @@ func TestHandleNodesLimit2000ColdMiss(t *testing.T) { } stmt, err := tx.Prepare(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count, foreign_advert) - VALUES (?, ?, ?, 0, 0, ?, '2026-01-01T00:00:00Z', 1, 0)`) + VALUES ($1, $2, $3, 0, 0, $4, '2026-01-01T00:00:00Z', 1, 0)`) if err != nil { t.Fatal(err) } diff --git a/cmd/server/nodes_perf_test.go b/cmd/server/nodes_perf_test.go index c7e1c5722..b1ac1acb4 100644 --- a/cmd/server/nodes_perf_test.go +++ b/cmd/server/nodes_perf_test.go @@ -43,7 +43,7 @@ func TestHandleNodesPerfLargeFleet(t *testing.T) { } stmt, err := tx.Prepare(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count, foreign_advert) - VALUES (?, ?, ?, 0, 0, ?, '2026-01-01T00:00:00Z', 1, 0)`) + VALUES ($1, $2, $3, 0, 0, $4, '2026-01-01T00:00:00Z', 1, 0)`) if err != nil { t.Fatal(err) } diff --git a/cmd/server/notify_eval.go b/cmd/server/notify_eval.go new file mode 100644 index 000000000..f8e46e65b --- /dev/null +++ b/cmd/server/notify_eval.go @@ -0,0 +1,302 @@ +package main + +import ( + "sort" + "strings" + "time" + + "github.com/meshcore-analyzer/users" +) + +// Node notification evaluator (docs/specs/2026-10-07-node-notifications-design.md). +// evaluateNotifications is a pure function: everything it reads is in +// notifyInput, the time included, so it is tested without a clock or a +// database. The notifier (notify_service.go) gathers the input and acts on +// the result. + +// notifyNode is one analyzer node row the evaluator needs. +type notifyNode struct { + Pubkey string // lowercase hex + Name string + Role string // as stored; compared lowercase + LastSeen string // nodes.last_seen, "" when NULL + BatteryMv *int // latest advert telemetry; nil when the node reports none + Foreign bool // nodes.foreign_advert = 1 +} + +// notifyObserver is one active (not soft-deleted) observer. +type notifyObserver struct { + ID string + Name string + LastSeen string // observers.last_seen, "" when NULL +} + +// notifyInput is everything one evaluation reads. +type notifyInput struct { + Now time.Time + Health HealthThresholds // resolved, as Config.GetHealthThresholds returns it + LowMv int // Config.LowBatteryMv + Accounts map[int64]users.User + Prefs []users.NotifyPrefs + Watches []users.NotifyWatch + States []users.NotifyState + Nodes map[string]notifyNode // watched nodes, plus every foreign node when an admin chose foreign.new + Heard map[string]time.Time // newest packet involving the node in the packet store (node page "Last Heard") + Relayed map[string]time.Time // last relay hop, used for repeaters and rooms only (#1598) + Observers []notifyObserver // nil unless an admin chose observer.offline + // IngestStale: the newest packet in the store is old, so a silent node + // and a silent feed look the same. node.offline and observer.offline + // are then not compared (states unchanged, no changes). + IngestStale bool + // IngestResumedAt: when the last stale period ended (zero: none since + // startup). For a node or observer whose stored state is good, evidence + // older than this counts as heard at this time, so nothing goes offline + // until a full silent window has passed after the feed came back. + IngestResumedAt time.Time +} + +// notifyChange is one transition for one user. +type notifyChange struct { + Event string + Subject string // node pubkey or observer id + Name string // display name at evaluation time + To string // users.NotifyGood, users.NotifyBad or users.NotifyTold + BatteryMv *int // node.battery only + At time.Time +} + +// notifyResult: Changes per user, ordered by event then name; States are +// the rows to write (new or changed), disabled users included; Drop are the +// rows to delete (admin-event rows of users who are no longer admin). +type notifyResult struct { + Changes map[int64][]notifyChange + States []users.NotifyState + Drop []users.NotifyKey +} + +const ( + // batteryHysteresisMv: a low battery is good again only at lowMv + 100. + batteryHysteresisMv = 100 + // foreignBaselineSubject marks that an admin's foreign.new baseline + // exists: the first evaluation after opting in stores every current + // foreign node as told without mailing; afterwards a foreign node + // without a row is new. + foreignBaselineSubject = "*" +) + +var notifyEventRank = map[string]int{ + users.NotifyNodeOffline: 0, users.NotifyNodeBattery: 1, users.NotifyForeignNew: 2, users.NotifyObserverOffline: 3, +} + +// notifyInfra reports the roles whose relay activity counts as heard +// (GetHealthMs and roles.js use the same two). +func notifyInfra(role string) bool { + r := strings.ToLower(role) + return r == "repeater" || r == "room" +} + +// nodeLabel is the node's name made safe for a mail line (mailSafeText), +// or the first 8 hex characters of its key. +func nodeLabel(n notifyNode, known bool, pk string) string { + if name := mailSafeText(n.Name); known && name != "" { + return name + } + if len(pk) > 8 { + return pk[:8] + } + return pk +} + +// nodeOnlineState is good when the node was heard within its role's silent +// window, like roles.js getNodeStatus: the latest of last_seen, the packet +// store's last heard and, for repeaters and rooms, the last relay hop. A +// node missing from the analyzer database is bad. While prev is good, the +// latest evidence is never older than in.IngestResumedAt (the outage grace). +func nodeOnlineState(pk string, n notifyNode, known bool, prev string, in *notifyInput) string { + if !known { + return users.NotifyBad + } + last, _ := parseRelayTS(n.LastSeen) + if t, ok := in.Heard[pk]; ok && t.After(last) { + last = t + } + if notifyInfra(n.Role) { + if t, ok := in.Relayed[pk]; ok && t.After(last) { + last = t + } + } + if prev == users.NotifyGood && last.Before(in.IngestResumedAt) { + last = in.IngestResumedAt + } + _, silentMs := in.Health.GetHealthMs(strings.ToLower(n.Role)) + if last.IsZero() || in.Now.Sub(last) >= time.Duration(silentMs)*time.Millisecond { + return users.NotifyBad + } + return users.NotifyGood +} + +// batteryState applies the low-battery threshold with hysteresis. +func batteryState(mv, lowMv int, prev string) string { + if prev == users.NotifyBad { + if mv >= lowMv+batteryHysteresisMv { + return users.NotifyGood + } + return users.NotifyBad + } + if mv < lowMv { + return users.NotifyBad + } + return users.NotifyGood +} + +// observerState: bad when last_seen is older than observerStaleMinutes, +// good again within observerOnlineMinutes. While prev is good, last_seen +// is never older than resumed (the outage grace, zero for none). +func observerState(o notifyObserver, h HealthThresholds, now, resumed time.Time, prev string) string { + t, ok := parseRelayTS(o.LastSeen) + if prev == users.NotifyGood && !resumed.IsZero() && (!ok || t.Before(resumed)) { + t, ok = resumed, true + } + if !ok { + return users.NotifyBad + } + age := now.Sub(t) + if prev == users.NotifyBad { + if age <= time.Duration(h.ObserverOnlineMinutes)*time.Minute { + return users.NotifyGood + } + return users.NotifyBad + } + if age > time.Duration(h.ObserverStaleMinutes)*time.Minute { + return users.NotifyBad + } + return users.NotifyGood +} + +// evaluateNotifications computes every chosen (user, event, subject) state +// and compares it with the stored one. A subject without a stored state is +// stored without a change (a restart or a new watch never mails); a stored +// state that differs is a change. Users missing from Accounts are skipped. +// A user with watches but no preferences row gets the defaults (enabled, +// node events). A disabled user's states are kept current without changes, +// so turning notifications on again never mails what happened while off. +// Admin events are evaluated only for admins; a non-admin's admin-event +// rows are dropped, like opting out, so a re-promotion starts silently. +func evaluateNotifications(in notifyInput) notifyResult { + prev := make(map[users.NotifyKey]string, len(in.States)) + for _, s := range in.States { + prev[s.NotifyKey] = s.State + } + watched := map[int64][]string{} + for _, w := range in.Watches { + watched[w.UserID] = append(watched[w.UserID], w.Pubkey) + } + var foreign []string + for pk, n := range in.Nodes { + if n.Foreign { + foreign = append(foreign, pk) + } + } + sort.Strings(foreign) + + res := notifyResult{Changes: map[int64][]notifyChange{}} + for _, s := range in.States { + if u, ok := in.Accounts[s.UserID]; ok && u.Role != users.RoleAdmin && users.IsAdminNotifyEvent(s.Event) { + res.Drop = append(res.Drop, s.NotifyKey) + } + } + prefs := make([]users.NotifyPrefs, 0, len(in.Prefs)+len(watched)) + hasPrefs := make(map[int64]bool, len(in.Prefs)) + for _, p := range in.Prefs { + prefs = append(prefs, p) + hasPrefs[p.UserID] = true + } + var defaulted []int64 + for uid := range watched { + if !hasPrefs[uid] { + defaulted = append(defaulted, uid) + } + } + sort.Slice(defaulted, func(i, j int) bool { return defaulted[i] < defaulted[j] }) + for _, uid := range defaulted { + prefs = append(prefs, users.DefaultNotifyPrefs(uid)) + } + mailing := false // Enabled of the user being evaluated + store := func(k users.NotifyKey, state string) { + res.States = append(res.States, users.NotifyState{NotifyKey: k, State: state, ChangedAt: in.Now}) + } + compare := func(k users.NotifyKey, name, state string, mv *int) { + old, had := prev[k] + if had && old == state { + return + } + store(k, state) + if had && mailing { + res.Changes[k.UserID] = append(res.Changes[k.UserID], + notifyChange{Event: k.Event, Subject: k.Subject, Name: name, To: state, BatteryMv: mv, At: in.Now}) + } + } + + for _, p := range prefs { + u, ok := in.Accounts[p.UserID] + if !ok { + continue + } + mailing = p.Enabled + for _, pk := range watched[p.UserID] { + n, known := in.Nodes[pk] + name := nodeLabel(n, known, pk) + if p.Has(users.NotifyNodeOffline) && !in.IngestStale { + k := users.NotifyKey{UserID: p.UserID, Event: users.NotifyNodeOffline, Subject: pk} + compare(k, name, nodeOnlineState(pk, n, known, prev[k], &in), nil) + } + if p.Has(users.NotifyNodeBattery) && known && n.BatteryMv != nil { + k := users.NotifyKey{UserID: p.UserID, Event: users.NotifyNodeBattery, Subject: pk} + compare(k, name, batteryState(*n.BatteryMv, in.LowMv, prev[k]), n.BatteryMv) + } + } + if u.Role != users.RoleAdmin { + continue + } + if p.Has(users.NotifyForeignNew) { + base := users.NotifyKey{UserID: p.UserID, Event: users.NotifyForeignNew, Subject: foreignBaselineSubject} + _, baselined := prev[base] + if !baselined { + store(base, users.NotifyGood) + } + for _, pk := range foreign { + k := users.NotifyKey{UserID: p.UserID, Event: users.NotifyForeignNew, Subject: pk} + if _, told := prev[k]; told { + continue + } + store(k, users.NotifyTold) + if baselined && mailing { + res.Changes[p.UserID] = append(res.Changes[p.UserID], notifyChange{Event: users.NotifyForeignNew, Subject: pk, + Name: nodeLabel(in.Nodes[pk], true, pk), To: users.NotifyTold, At: in.Now}) + } + } + } + if p.Has(users.NotifyObserverOffline) && !in.IngestStale { + for _, o := range in.Observers { + k := users.NotifyKey{UserID: p.UserID, Event: users.NotifyObserverOffline, Subject: o.ID} + name := mailSafeText(o.Name) + if name == "" { + name = o.ID + } + compare(k, name, observerState(o, in.Health, in.Now, in.IngestResumedAt, prev[k]), nil) + } + } + } + for _, list := range res.Changes { + sort.SliceStable(list, func(i, j int) bool { + if a, b := notifyEventRank[list[i].Event], notifyEventRank[list[j].Event]; a != b { + return a < b + } + if list[i].Name != list[j].Name { + return list[i].Name < list[j].Name + } + return list[i].Subject < list[j].Subject + }) + } + return res +} diff --git a/cmd/server/notify_eval_test.go b/cmd/server/notify_eval_test.go new file mode 100644 index 000000000..b3704e353 --- /dev/null +++ b/cmd/server/notify_eval_test.go @@ -0,0 +1,552 @@ +package main + +import ( + "fmt" + "strings" + "testing" + "time" + + "github.com/meshcore-analyzer/users" +) + +var evNow = time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC) + +const ( + evUser = int64(1) + evAdmin = int64(2) + evAdmin2 = int64(3) +) + +var ( + evPkA = strings.Repeat("a", 64) + evPkB = strings.Repeat("b", 64) + evPkF = strings.Repeat("f", 64) + evPkG = strings.Repeat("e", 64) +) + +func evAgo(d time.Duration) string { return evNow.Add(-d).Format(time.RFC3339) } +func evMv(v int) *int { return &v } + +// evInput: one user watching evPkA with the default node events, two +// admins without prefs, no stored states. +func evInput() notifyInput { + return notifyInput{ + Now: evNow, Health: (&Config{}).GetHealthThresholds(), LowMv: 3300, + Accounts: map[int64]users.User{ + evUser: {ID: evUser, Role: users.RoleUser, Status: users.StatusActive}, + evAdmin: {ID: evAdmin, Role: users.RoleAdmin, Status: users.StatusActive}, + evAdmin2: {ID: evAdmin2, Role: users.RoleAdmin, Status: users.StatusActive}, + }, + Prefs: []users.NotifyPrefs{{UserID: evUser, Enabled: true, Events: []string{users.NotifyNodeOffline, users.NotifyNodeBattery}}}, + Watches: []users.NotifyWatch{{UserID: evUser, Pubkey: evPkA}}, + Nodes: map[string]notifyNode{}, + Heard: map[string]time.Time{}, + Relayed: map[string]time.Time{}, + } +} + +func evStateRow(uid int64, event, subject, state string) users.NotifyState { + return users.NotifyState{NotifyKey: users.NotifyKey{UserID: uid, Event: event, Subject: subject}, State: state, ChangedAt: evNow.Add(-time.Hour)} +} + +func evState(res notifyResult, uid int64, event, subject string) (string, bool) { + for _, s := range res.States { + if s.UserID == uid && s.Event == event && s.Subject == subject { + return s.State, true + } + } + return "", false +} + +func TestEvaluateFirstEvaluationStoresWithoutChanges(t *testing.T) { + in := evInput() + in.Watches = append(in.Watches, users.NotifyWatch{UserID: evUser, Pubkey: evPkB}) + in.Nodes[evPkA] = notifyNode{Pubkey: evPkA, Name: "Alpha", Role: "companion", LastSeen: evAgo(48 * time.Hour)} + in.Nodes[evPkB] = notifyNode{Pubkey: evPkB, Name: "Bravo", Role: "companion", LastSeen: evAgo(time.Hour), BatteryMv: evMv(3100)} + res := evaluateNotifications(in) + if len(res.Changes) != 0 { + t.Fatalf("changes on the first evaluation: %+v", res.Changes) + } + for _, c := range []struct{ event, subject, want string }{ + {users.NotifyNodeOffline, evPkA, users.NotifyBad}, + {users.NotifyNodeOffline, evPkB, users.NotifyGood}, + {users.NotifyNodeBattery, evPkB, users.NotifyBad}, + } { + if got, ok := evState(res, evUser, c.event, c.subject); !ok || got != c.want { + t.Errorf("%s %s = %q (stored %v); want %q", c.event, c.subject[:4], got, ok, c.want) + } + } + if _, ok := evState(res, evUser, users.NotifyNodeBattery, evPkA); ok { + t.Error("battery state stored for a node without telemetry") + } + for _, s := range res.States { + if !s.ChangedAt.Equal(evNow) { + t.Errorf("ChangedAt = %v; want the input's Now", s.ChangedAt) + } + } +} + +func TestEvaluateOfflineAndBack(t *testing.T) { + in := evInput() + in.Nodes[evPkA] = notifyNode{Pubkey: evPkA, Name: "Alpha", Role: "companion", LastSeen: evAgo(24 * time.Hour)} + in.States = []users.NotifyState{evStateRow(evUser, users.NotifyNodeOffline, evPkA, users.NotifyGood)} + res := evaluateNotifications(in) + ch := res.Changes[evUser] + if len(ch) != 1 || ch[0].Event != users.NotifyNodeOffline || ch[0].Subject != evPkA || ch[0].To != users.NotifyBad || + ch[0].Name != "Alpha" || !ch[0].At.Equal(evNow) { + t.Fatalf("changes = %+v; want one offline change for Alpha", ch) + } + // One minute inside the 24 h companion window: still online, nothing to write. + in.Nodes[evPkA] = notifyNode{Pubkey: evPkA, Name: "Alpha", Role: "companion", LastSeen: evAgo(24*time.Hour - time.Minute)} + if res := evaluateNotifications(in); len(res.Changes) != 0 || len(res.States) != 0 { + t.Fatalf("unchanged online node produced %+v", res) + } + // Stored bad, advert 30 h old, but the packet store heard it an hour ago. + in.States = []users.NotifyState{evStateRow(evUser, users.NotifyNodeOffline, evPkA, users.NotifyBad)} + in.Nodes[evPkA] = notifyNode{Pubkey: evPkA, Name: "Alpha", Role: "companion", LastSeen: evAgo(30 * time.Hour)} + in.Heard[evPkA] = evNow.Add(-time.Hour) + if ch := evaluateNotifications(in).Changes[evUser]; len(ch) != 1 || ch[0].To != users.NotifyGood { + t.Fatalf("changes = %+v; want one back-online change", ch) + } +} + +func TestEvaluateRelayAwareInfra(t *testing.T) { + cases := []struct { + name, role, lastSeen string + relayed time.Duration + want string + }{ + {"repeater relaying", "repeater", evAgo(80 * time.Hour), time.Hour, users.NotifyGood}, + {"room relaying, role case", "Room", evAgo(80 * time.Hour), time.Hour, users.NotifyGood}, + {"repeater silent exactly 72 h", "repeater", evAgo(72 * time.Hour), 0, users.NotifyBad}, + {"repeater 71 h", "repeater", evAgo(71 * time.Hour), 0, users.NotifyGood}, + {"companion: relay time ignored", "companion", evAgo(30 * time.Hour), time.Hour, users.NotifyBad}, + {"no timestamp at all", "repeater", "", 0, users.NotifyBad}, + } + for _, c := range cases { + in := evInput() + in.Nodes[evPkA] = notifyNode{Pubkey: evPkA, Role: c.role, LastSeen: c.lastSeen} + if c.relayed > 0 { + in.Relayed[evPkA] = evNow.Add(-c.relayed) + } + if got, _ := evState(evaluateNotifications(in), evUser, users.NotifyNodeOffline, evPkA); got != c.want { + t.Errorf("%s: %q; want %q", c.name, got, c.want) + } + } +} + +func TestEvaluateBatteryHysteresis(t *testing.T) { + cases := []struct { + prev string + mv int + want string + change bool + }{ + {"", 3299, users.NotifyBad, false}, + {"", 3300, users.NotifyGood, false}, + {"", 3350, users.NotifyGood, false}, + {users.NotifyGood, 3299, users.NotifyBad, true}, + {users.NotifyGood, 3300, users.NotifyGood, false}, + {users.NotifyBad, 3399, users.NotifyBad, false}, + {users.NotifyBad, 3400, users.NotifyGood, true}, + } + for _, c := range cases { + in := evInput() + in.Nodes[evPkA] = notifyNode{Pubkey: evPkA, Name: "Alpha", Role: "companion", LastSeen: evAgo(time.Hour), BatteryMv: evMv(c.mv)} + in.States = []users.NotifyState{evStateRow(evUser, users.NotifyNodeOffline, evPkA, users.NotifyGood)} + if c.prev != "" { + in.States = append(in.States, evStateRow(evUser, users.NotifyNodeBattery, evPkA, c.prev)) + } + res := evaluateNotifications(in) + got, stored := evState(res, evUser, users.NotifyNodeBattery, evPkA) + if !stored { + got = c.prev + } + ch := res.Changes[evUser] + if got != c.want || (len(ch) == 1) != c.change { + t.Errorf("prev %q, %d mV: state %q, changes %+v; want %q, change %v", c.prev, c.mv, got, ch, c.want, c.change) + continue + } + if c.change && (ch[0].Event != users.NotifyNodeBattery || ch[0].BatteryMv == nil || *ch[0].BatteryMv != c.mv) { + t.Errorf("prev %q, %d mV: change = %+v", c.prev, c.mv, ch[0]) + } + } +} + +func TestEvaluateAbsentNodeIsOfflineOnce(t *testing.T) { + in := evInput() // evPkA is watched but not in the analyzer database + in.States = []users.NotifyState{ + evStateRow(evUser, users.NotifyNodeOffline, evPkA, users.NotifyGood), + evStateRow(evUser, users.NotifyNodeBattery, evPkA, users.NotifyBad), + } + ch := evaluateNotifications(in).Changes[evUser] + if len(ch) != 1 || ch[0].Event != users.NotifyNodeOffline || ch[0].To != users.NotifyBad || ch[0].Name != evPkA[:8] { + t.Fatalf("changes = %+v; want one offline change named by the key prefix", ch) + } + in.States[0] = evStateRow(evUser, users.NotifyNodeOffline, evPkA, users.NotifyBad) + if res := evaluateNotifications(in); len(res.Changes) != 0 || len(res.States) != 0 { + t.Fatalf("absent node evaluated again: %+v", res) + } +} + +func TestEvaluateForeignOncePerAdmin(t *testing.T) { + in := evInput() + in.Prefs[0].Events = append(in.Prefs[0].Events, users.NotifyForeignNew) // a non-admin with an admin event + in.Prefs = append(in.Prefs, users.NotifyPrefs{UserID: evAdmin, Enabled: true, Events: []string{users.NotifyForeignNew}}) + in.Nodes[evPkF] = notifyNode{Pubkey: evPkF, Name: "Foxtrot", Foreign: true, LastSeen: evAgo(time.Hour)} + res := evaluateNotifications(in) + if len(res.Changes[evAdmin]) != 0 { + t.Fatalf("first evaluation mailed: %+v", res.Changes[evAdmin]) + } + if s, ok := evState(res, evAdmin, users.NotifyForeignNew, foreignBaselineSubject); !ok || s != users.NotifyGood { + t.Fatalf("baseline row = %q, %v", s, ok) + } + if s, ok := evState(res, evAdmin, users.NotifyForeignNew, evPkF); !ok || s != users.NotifyTold { + t.Fatalf("existing foreign node = %q, %v; want told", s, ok) + } + if _, ok := evState(res, evUser, users.NotifyForeignNew, evPkF); ok { + t.Fatal("non-admin evaluated for foreign.new") + } + + // Next tick: a second foreign node appears; a second admin opts in now. + in.States = res.States + in.Prefs = append(in.Prefs, users.NotifyPrefs{UserID: evAdmin2, Enabled: true, Events: []string{users.NotifyForeignNew}}) + in.Nodes[evPkG] = notifyNode{Pubkey: evPkG, Name: "Golf", Foreign: true} + res = evaluateNotifications(in) + ch := res.Changes[evAdmin] + if len(ch) != 1 || ch[0].Subject != evPkG || ch[0].To != users.NotifyTold || ch[0].Name != "Golf" { + t.Fatalf("admin changes = %+v; want Golf once", ch) + } + if len(res.Changes[evAdmin2]) != 0 { + t.Fatalf("second admin's first evaluation mailed: %+v", res.Changes[evAdmin2]) + } + + in.States = append(in.States, res.States...) + if res := evaluateNotifications(in); len(res.Changes) != 0 { + t.Fatalf("foreign nodes mailed twice: %+v", res.Changes) + } +} + +func TestEvaluateObserverOfflineAndBack(t *testing.T) { + cases := []struct { + prev string + age time.Duration + want string + change bool + }{ + {"", 2 * time.Hour, users.NotifyGood, false}, + {"", 25 * time.Hour, users.NotifyBad, false}, + {users.NotifyGood, 24 * time.Hour, users.NotifyGood, false}, + {users.NotifyGood, 24*time.Hour + time.Minute, users.NotifyBad, true}, + {users.NotifyBad, 2 * time.Hour, users.NotifyBad, false}, + {users.NotifyBad, 60 * time.Minute, users.NotifyGood, true}, + } + for _, c := range cases { + in := evInput() + in.Prefs = []users.NotifyPrefs{{UserID: evAdmin, Enabled: true, Events: []string{users.NotifyObserverOffline}}} + in.Watches = nil + in.Observers = []notifyObserver{{ID: "OBS1", Name: "Roof", LastSeen: evAgo(c.age)}} + if c.prev != "" { + in.States = []users.NotifyState{evStateRow(evAdmin, users.NotifyObserverOffline, "OBS1", c.prev)} + } + res := evaluateNotifications(in) + got, stored := evState(res, evAdmin, users.NotifyObserverOffline, "OBS1") + if !stored { + got = c.prev + } + ch := res.Changes[evAdmin] + if got != c.want || (len(ch) == 1) != c.change || (c.change && ch[0].Name != "Roof") { + t.Errorf("prev %q, age %v: state %q, changes %+v; want %q, change %v", c.prev, c.age, got, ch, c.want, c.change) + } + } + in := evInput() + in.Prefs = []users.NotifyPrefs{{UserID: evAdmin, Enabled: true, Events: []string{users.NotifyObserverOffline}}} + in.Observers = []notifyObserver{{ID: "OBS2"}} + if s, _ := evState(evaluateNotifications(in), evAdmin, users.NotifyObserverOffline, "OBS2"); s != users.NotifyBad { + t.Fatalf("observer without last_seen = %q; want bad", s) + } +} + +func TestEvaluateOnlyChosenEventsAndKnownAccounts(t *testing.T) { + in := evInput() + in.Prefs[0].Events = []string{users.NotifyNodeBattery, users.NotifyObserverOffline} // observer.offline: the user is no admin + in.Nodes[evPkA] = notifyNode{Pubkey: evPkA, Role: "companion", LastSeen: evAgo(48 * time.Hour), BatteryMv: evMv(4000)} + in.Observers = []notifyObserver{{ID: "OBS1", LastSeen: evAgo(time.Hour)}} + res := evaluateNotifications(in) + if len(res.States) != 1 || res.States[0].Event != users.NotifyNodeBattery { + t.Fatalf("states = %+v; want only node.battery", res.States) + } + in.Accounts = map[int64]users.User{} // the user was deleted after the read + if res := evaluateNotifications(in); len(res.States) != 0 { + t.Fatalf("deleted user evaluated: %+v", res.States) + } +} + +func TestEvaluateChangesAreOrderedPerUser(t *testing.T) { + in := evInput() + in.Watches = []users.NotifyWatch{{UserID: evUser, Pubkey: evPkB}, {UserID: evUser, Pubkey: evPkA}} + in.Nodes[evPkA] = notifyNode{Pubkey: evPkA, Name: "Alpha", Role: "companion", LastSeen: evAgo(48 * time.Hour), BatteryMv: evMv(3000)} + in.Nodes[evPkB] = notifyNode{Pubkey: evPkB, Name: "Bravo", Role: "companion", LastSeen: evAgo(48 * time.Hour)} + in.States = []users.NotifyState{ + evStateRow(evUser, users.NotifyNodeOffline, evPkA, users.NotifyGood), + evStateRow(evUser, users.NotifyNodeOffline, evPkB, users.NotifyGood), + evStateRow(evUser, users.NotifyNodeBattery, evPkA, users.NotifyGood), + } + var got []string + for _, c := range evaluateNotifications(in).Changes[evUser] { + got = append(got, c.Event+" "+c.Name) + } + if want := "node.offline Alpha|node.offline Bravo|node.battery Alpha"; strings.Join(got, "|") != want { + t.Fatalf("order = %q; want %q", strings.Join(got, "|"), want) + } +} + +// 100 users x 50 watches over 2,000 nodes: the spec's Performance figure +// (5,000 state rows per event type). Run with -bench and quote the result +// in the PR. +func BenchmarkEvaluateNotifications(b *testing.B) { + in := notifyInput{Now: evNow, Health: (&Config{}).GetHealthThresholds(), LowMv: 3300, + Accounts: map[int64]users.User{}, Nodes: map[string]notifyNode{}, Heard: map[string]time.Time{}, Relayed: map[string]time.Time{}} + for i := 0; i < 2000; i++ { + pk := fmt.Sprintf("%064x", i) + in.Nodes[pk] = notifyNode{Pubkey: pk, Role: "repeater", LastSeen: evAgo(time.Duration(i) * time.Minute), BatteryMv: evMv(3200 + i%300)} + } + for u := int64(1); u <= 100; u++ { + in.Accounts[u] = users.User{ID: u, Role: users.RoleUser, Status: users.StatusActive} + in.Prefs = append(in.Prefs, users.NotifyPrefs{UserID: u, Enabled: true, Events: users.NodeNotifyEvents}) + for w := 0; w < 50; w++ { + in.Watches = append(in.Watches, users.NotifyWatch{UserID: u, Pubkey: fmt.Sprintf("%064x", (int(u)*37+w*11)%2000)}) + } + } + in.States = evaluateNotifications(in).States + b.ResetTimer() + for i := 0; i < b.N; i++ { + evaluateNotifications(in) + } +} + +// P7: an admin who loses the admin role loses their admin-event states, like +// opting out of those events, so a re-promotion starts a fresh baseline. +func TestEvaluateDemotedAdminDropsAdminEventStates(t *testing.T) { + in := evInput() + in.Prefs = append(in.Prefs, users.NotifyPrefs{UserID: evAdmin, Enabled: true, + Events: []string{users.NotifyNodeOffline, users.NotifyForeignNew, users.NotifyObserverOffline}}) + in.Watches = append(in.Watches, users.NotifyWatch{UserID: evAdmin, Pubkey: evPkA}) + in.Nodes[evPkA] = notifyNode{Pubkey: evPkA, Name: "Alpha", Role: "companion", LastSeen: evAgo(time.Hour)} + in.Nodes[evPkF] = notifyNode{Pubkey: evPkF, Name: "Foxtrot", Foreign: true} + in.Observers = []notifyObserver{{ID: "OBS1", Name: "Roof", LastSeen: evAgo(time.Hour)}} + in.States = evaluateNotifications(in).States + if s, ok := evState(notifyResult{States: in.States}, evAdmin, users.NotifyForeignNew, foreignBaselineSubject); !ok || s != users.NotifyGood { + t.Fatalf("baseline row = %q, %v", s, ok) + } + + // Demoted: the admin-event rows are dropped, the node rows stay, nothing mailed. + admin := in.Accounts[evAdmin] + admin.Role = users.RoleUser + in.Accounts[evAdmin] = admin + in.Nodes[evPkG] = notifyNode{Pubkey: evPkG, Name: "Golf", Foreign: true} + in.Observers[0].LastSeen = evAgo(48 * time.Hour) + res := evaluateNotifications(in) + if len(res.Changes) != 0 || len(res.States) != 0 { + t.Fatalf("demoted admin evaluated: %+v", res) + } + want := map[users.NotifyKey]bool{ + {UserID: evAdmin, Event: users.NotifyForeignNew, Subject: foreignBaselineSubject}: true, + {UserID: evAdmin, Event: users.NotifyForeignNew, Subject: evPkF}: true, + {UserID: evAdmin, Event: users.NotifyObserverOffline, Subject: "OBS1"}: true, + } + if len(res.Drop) != len(want) { + t.Fatalf("drop = %+v; want %d admin-event keys", res.Drop, len(want)) + } + for _, k := range res.Drop { + if !want[k] { + t.Errorf("dropped %+v", k) + } + } + dropped := map[users.NotifyKey]bool{} + for _, k := range res.Drop { + dropped[k] = true + } + var kept []users.NotifyState + for _, s := range in.States { + if !dropped[s.NotifyKey] { + kept = append(kept, s) + } + } + in.States = kept + if res := evaluateNotifications(in); len(res.Drop) != 0 { + t.Fatalf("dropped again: %+v", res.Drop) + } + + // Re-promoted: Golf appeared and the observer went offline meanwhile; the + // first evaluation stores a new baseline and mails nothing. + admin.Role = users.RoleAdmin + in.Accounts[evAdmin] = admin + res = evaluateNotifications(in) + if len(res.Changes) != 0 || len(res.Drop) != 0 { + t.Fatalf("re-promoted admin: changes %+v, drop %+v", res.Changes, res.Drop) + } + for _, c := range []struct{ event, subject, want string }{ + {users.NotifyForeignNew, foreignBaselineSubject, users.NotifyGood}, + {users.NotifyForeignNew, evPkG, users.NotifyTold}, + {users.NotifyObserverOffline, "OBS1", users.NotifyBad}, + } { + if got, ok := evState(res, evAdmin, c.event, c.subject); !ok || got != c.want { + t.Errorf("%s %s = %q (stored %v); want %q", c.event, c.subject, got, ok, c.want) + } + } +} + +// A user who watches nodes but has no preferences row (never opened the +// settings, or created before the notification schema) gets the defaults: +// enabled, node events on, admin events off. +func TestEvaluateWatcherWithoutPrefsGetsDefaults(t *testing.T) { + in := evInput() + in.Prefs = nil + in.Watches = []users.NotifyWatch{{UserID: evAdmin, Pubkey: evPkA}} + in.Nodes[evPkA] = notifyNode{Pubkey: evPkA, Name: "Alpha", Role: "companion", LastSeen: evAgo(48 * time.Hour), BatteryMv: evMv(3000)} + in.Nodes[evPkF] = notifyNode{Pubkey: evPkF, Name: "Foxtrot", Foreign: true} + in.Observers = []notifyObserver{{ID: "OBS1", LastSeen: evAgo(time.Hour)}} + in.States = []users.NotifyState{ + evStateRow(evAdmin, users.NotifyNodeOffline, evPkA, users.NotifyGood), + evStateRow(evAdmin, users.NotifyNodeBattery, evPkA, users.NotifyGood), + } + res := evaluateNotifications(in) + var got []string + for _, c := range res.Changes[evAdmin] { + got = append(got, c.Event+" "+c.To) + } + if want := "node.offline bad|node.battery bad"; strings.Join(got, "|") != want { + t.Fatalf("changes = %q; want %q", strings.Join(got, "|"), want) + } + for _, s := range res.States { + if users.IsAdminNotifyEvent(s.Event) { + t.Errorf("admin event evaluated without opting in: %+v", s) + } + } + in.Accounts = map[int64]users.User{} + if res := evaluateNotifications(in); len(res.States) != 0 { + t.Fatalf("watcher without an account evaluated: %+v", res.States) + } +} + +// A disabled user keeps being evaluated: states follow the node, nothing is +// mailed, and re-enabling does not mail what changed while off. +func TestEvaluateDisabledUserUpdatesStatesSilently(t *testing.T) { + in := evInput() + in.Prefs[0].Enabled = false + in.Prefs = append(in.Prefs, users.NotifyPrefs{UserID: evAdmin, Enabled: false, Events: []string{users.NotifyForeignNew}}) + in.Nodes[evPkA] = notifyNode{Pubkey: evPkA, Name: "Alpha", Role: "companion", LastSeen: evAgo(time.Hour)} + in.Nodes[evPkF] = notifyNode{Pubkey: evPkF, Name: "Foxtrot", Foreign: true} + in.States = evaluateNotifications(in).States + + // While off: Alpha goes offline and Golf appears. + in.Nodes[evPkA] = notifyNode{Pubkey: evPkA, Name: "Alpha", Role: "companion", LastSeen: evAgo(48 * time.Hour)} + in.Nodes[evPkG] = notifyNode{Pubkey: evPkG, Name: "Golf", Foreign: true} + res := evaluateNotifications(in) + if len(res.Changes) != 0 { + t.Fatalf("disabled users mailed: %+v", res.Changes) + } + if s, ok := evState(res, evUser, users.NotifyNodeOffline, evPkA); !ok || s != users.NotifyBad { + t.Fatalf("disabled user's offline state = %q, %v; want bad stored", s, ok) + } + if s, ok := evState(res, evAdmin, users.NotifyForeignNew, evPkG); !ok || s != users.NotifyTold { + t.Fatalf("disabled admin's foreign state = %q, %v; want told stored", s, ok) + } + in.States = append(in.States, res.States...) + + // Re-enabled: the next evaluation has nothing to send. + in.Prefs[0].Enabled = true + in.Prefs[1].Enabled = true + if res := evaluateNotifications(in); len(res.Changes) != 0 || len(res.States) != 0 { + t.Fatalf("re-enabling mailed old changes: %+v", res) + } +} + +// While ingest is stale nobody can tell a silent node from a silent +// MQTT feed: offline comparisons pause with their states unchanged; +// battery and foreign nodes go on. +func TestEvaluateIngestStalePausesOfflineChecks(t *testing.T) { + in := evInput() + in.IngestStale = true + in.Watches = append(in.Watches, users.NotifyWatch{UserID: evUser, Pubkey: evPkB}) + in.Prefs = append(in.Prefs, users.NotifyPrefs{UserID: evAdmin, Enabled: true, Events: []string{users.NotifyForeignNew, users.NotifyObserverOffline}}) + in.Nodes[evPkA] = notifyNode{Pubkey: evPkA, Name: "Alpha", Role: "companion", LastSeen: evAgo(48 * time.Hour), BatteryMv: evMv(3100)} + in.Nodes[evPkF] = notifyNode{Pubkey: evPkF, Name: "Foxtrot", Foreign: true} + in.Observers = []notifyObserver{{ID: "OBS1", Name: "Roof", LastSeen: evAgo(48 * time.Hour)}} + in.States = []users.NotifyState{ + evStateRow(evUser, users.NotifyNodeOffline, evPkA, users.NotifyGood), + evStateRow(evUser, users.NotifyNodeBattery, evPkA, users.NotifyGood), + evStateRow(evAdmin, users.NotifyObserverOffline, "OBS1", users.NotifyGood), + evStateRow(evAdmin, users.NotifyForeignNew, foreignBaselineSubject, users.NotifyGood), + } + res := evaluateNotifications(in) + for _, k := range []struct { + uid int64 + event, subject string + }{{evUser, users.NotifyNodeOffline, evPkA}, {evUser, users.NotifyNodeOffline, evPkB}, {evAdmin, users.NotifyObserverOffline, "OBS1"}} { + if s, ok := evState(res, k.uid, k.event, k.subject); ok { + t.Errorf("%s %s written as %q while ingest is stale", k.event, k.subject, s) + } + } + if ch := res.Changes[evUser]; len(ch) != 1 || ch[0].Event != users.NotifyNodeBattery || ch[0].To != users.NotifyBad { + t.Fatalf("user changes = %+v; want only the battery", ch) + } + if ch := res.Changes[evAdmin]; len(ch) != 1 || ch[0].Event != users.NotifyForeignNew || ch[0].Subject != evPkF { + t.Fatalf("admin changes = %+v; want only the foreign node", ch) + } + + in.IngestStale = false + res = evaluateNotifications(in) + if s, _ := evState(res, evUser, users.NotifyNodeOffline, evPkA); s != users.NotifyBad { + t.Fatalf("offline check after ingest resumed = %q; want bad", s) + } + if s, _ := evState(res, evAdmin, users.NotifyObserverOffline, "OBS1"); s != users.NotifyBad { + t.Fatalf("observer check after ingest resumed = %q; want bad", s) + } +} + +// After a feed outage, evidence older than the recovery counts as heard at +// the recovery: no node or observer goes offline until a full silent window +// has passed since then. A state that is already bad is not lifted by it. +func TestEvaluateGraceAfterIngestResumes(t *testing.T) { + in := evInput() + in.Prefs = append(in.Prefs, users.NotifyPrefs{UserID: evAdmin, Enabled: true, Events: []string{users.NotifyObserverOffline}}) + in.Watches = append(in.Watches, users.NotifyWatch{UserID: evUser, Pubkey: evPkB}) + in.Nodes[evPkA] = notifyNode{Pubkey: evPkA, Name: "Alpha", Role: "companion", LastSeen: evAgo(25 * time.Hour)} + in.Nodes[evPkB] = notifyNode{Pubkey: evPkB, Name: "Bravo", Role: "companion", LastSeen: evAgo(48 * time.Hour)} + in.Observers = []notifyObserver{{ID: "OBS1", Name: "Roof", LastSeen: evAgo(25 * time.Hour)}, {ID: "OBS2", Name: "Mast", LastSeen: evAgo(48 * time.Hour)}} + in.States = []users.NotifyState{ + evStateRow(evUser, users.NotifyNodeOffline, evPkA, users.NotifyGood), + evStateRow(evUser, users.NotifyNodeOffline, evPkB, users.NotifyBad), + evStateRow(evAdmin, users.NotifyObserverOffline, "OBS1", users.NotifyGood), + evStateRow(evAdmin, users.NotifyObserverOffline, "OBS2", users.NotifyBad), + } + in.IngestResumedAt = evNow.Add(-10 * time.Minute) + if res := evaluateNotifications(in); len(res.Changes) != 0 || len(res.States) != 0 { + t.Fatalf("10 minutes after recovery: changes %+v, states %+v; want none", res.Changes, res.States) + } + + // One silent window (24 h companions, observerStaleMinutes 1440) plus a + // minute after recovery, still no new evidence: offline. + in.Now = in.IngestResumedAt.Add(24*time.Hour + time.Minute) + res := evaluateNotifications(in) + if ch := res.Changes[evUser]; len(ch) != 1 || ch[0].Subject != evPkA || ch[0].To != users.NotifyBad { + t.Fatalf("node changes after the window = %+v; want Alpha offline", ch) + } + if ch := res.Changes[evAdmin]; len(ch) != 1 || ch[0].Subject != "OBS1" || ch[0].To != users.NotifyBad { + t.Fatalf("observer changes after the window = %+v; want Roof offline", ch) + } + + // A node heard after the recovery behaves as without the grace. + in.Now = evNow + in.Nodes[evPkB] = notifyNode{Pubkey: evPkB, Name: "Bravo", Role: "companion", LastSeen: evAgo(5 * time.Minute)} + in.Observers[1].LastSeen = evAgo(5 * time.Minute) + res = evaluateNotifications(in) + if ch := res.Changes[evUser]; len(ch) != 1 || ch[0].Subject != evPkB || ch[0].To != users.NotifyGood { + t.Fatalf("node heard after recovery = %+v; want Bravo back online", ch) + } + if ch := res.Changes[evAdmin]; len(ch) != 1 || ch[0].Subject != "OBS2" || ch[0].To != users.NotifyGood { + t.Fatalf("observer heard after recovery = %+v; want Mast back online", ch) + } +} diff --git a/cmd/server/notify_handlers.go b/cmd/server/notify_handlers.go new file mode 100644 index 000000000..4d0b856c5 --- /dev/null +++ b/cmd/server/notify_handlers.go @@ -0,0 +1,312 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "log" + "net/http" + "regexp" + "strconv" + "strings" + "time" + + "github.com/gorilla/mux" + "github.com/meshcore-analyzer/users" +) + +// Node notification routes (docs/specs/2026-10-07-node-notifications-design.md). +// Registered only when userManagement.notifications.enabled (see +// registerAuthRoutes). + +type notifyPrefsRequest struct { + Enabled bool `json:"enabled"` + Events []string `json:"events"` +} + +type notifyWatchJSON struct { + Pubkey string `json:"pubkey"` + Name string `json:"name"` // "" when unnamed or no longer in the analyzer database + Known bool `json:"known"` // false once the node left the analyzer database + CreatedAt string `json:"createdAt"` +} + +type notifyLimitsJSON struct { + MaxWatches int `json:"maxWatches"` + PerUserPerDay int `json:"perUserPerDay"` + MailsLast24h int `json:"mailsLast24h"` +} + +// notifyAccountJSON is the caller's notification state: GET answers it, and +// so does every change, so the client can replace its copy. +type notifyAccountJSON struct { + Enabled bool `json:"enabled"` + Events []string `json:"events"` + AvailableEvents []string `json:"availableEvents"` + Watches []notifyWatchJSON `json:"watches"` + Limits notifyLimitsJSON `json:"limits"` +} + +type watchMyNodesJSON struct { + Added int `json:"added"` + Already int `json:"already"` + Skipped int `json:"skipped"` // malformed, not in the analyzer database, or over the limit + Account notifyAccountJSON `json:"account"` +} + +func (s *Server) registerNotifyRoutes(r *mux.Router) { + r.HandleFunc("/api/account/notifications", s.withUser(s.handleNotifyGet)).Methods("GET") + r.HandleFunc("/api/account/notifications", s.withUser(s.handleNotifyPut)).Methods("PUT") + r.HandleFunc("/api/account/notifications/watches/{pubkey}", s.withUser(s.handleWatchPut)).Methods("PUT") + r.HandleFunc("/api/account/notifications/watches/{pubkey}", s.withUser(s.handleWatchDelete)).Methods("DELETE") + r.HandleFunc("/api/account/notifications/watch-my-nodes", s.withUser(s.handleWatchMyNodes)).Methods("POST") + // No session: the link comes from a mail, and the provider's one-click + // POST (List-Unsubscribe-Post) carries neither a cookie nor our Origin. + r.HandleFunc("/api/notifications/unsubscribe", s.handleUnsubscribeGet).Methods("GET") + r.HandleFunc("/api/notifications/unsubscribe", s.handleUnsubscribePost).Methods("POST") +} + +var notifyPubkeyRE = regexp.MustCompile(`^[0-9a-f]{64}$`) + +// notifyPubkey lowercases raw and reports whether it is a 64-hex-character key. +func notifyPubkey(raw string) (string, bool) { + pk := strings.ToLower(strings.TrimSpace(raw)) + return pk, notifyPubkeyRE.MatchString(pk) +} + +func availableNotifyEvents(u *users.User) []string { + out := append([]string{}, users.NodeNotifyEvents...) + if u.Role == users.RoleAdmin { + out = append(out, users.AdminNotifyEvents...) + } + return out +} + +func notifyInternal(w http.ResponseWriter, op string, uid int64, err error) { + log.Printf("[notify] %s for user #%d: %v", op, uid, err) + writeError(w, http.StatusInternalServerError, "internal error") +} + +// notifyAccount builds the caller's state; it creates the default +// preferences row on first use. A demoted admin's stored admin events are +// not shown (and are not evaluated). +func (s *Server) notifyAccount(u *users.User) (notifyAccountJSON, error) { + n, st, ns := s.auth.notify, s.auth.st, s.auth.set.notify + p, err := st.NotifyPrefsFor(u.ID) + if err != nil { + return notifyAccountJSON{}, err + } + ws, err := st.WatchesFor(u.ID) + if err != nil { + return notifyAccountJSON{}, err + } + pks := make([]string, len(ws)) + for i, w := range ws { + pks[i] = w.Pubkey + } + nodes, err := n.src.nodes(pks, false) + if err != nil { + return notifyAccountJSON{}, err + } + _, perUser, err := st.NotifyMailCounts(n.now().Add(-24 * time.Hour)) + if err != nil { + return notifyAccountJSON{}, err + } + out := notifyAccountJSON{Enabled: p.Enabled, Events: []string{}, AvailableEvents: availableNotifyEvents(u), + Watches: make([]notifyWatchJSON, 0, len(ws)), + Limits: notifyLimitsJSON{MaxWatches: ns.maxWatchesPerUser, PerUserPerDay: ns.perUserPerDay, MailsLast24h: perUser[u.ID]}} + for _, e := range p.Events { + if u.Role == users.RoleAdmin || !users.IsAdminNotifyEvent(e) { + out.Events = append(out.Events, e) + } + } + for _, w := range ws { + nd, known := nodes[w.Pubkey] + out.Watches = append(out.Watches, notifyWatchJSON{Pubkey: w.Pubkey, Name: nd.Name, Known: known, CreatedAt: rfc3339(w.CreatedAt)}) + } + return out, nil +} + +func (s *Server) writeNotifyAccount(w http.ResponseWriter, u *users.User) { + out, err := s.notifyAccount(u) + if err != nil { + notifyInternal(w, "account state", u.ID, err) + return + } + writeJSON(w, out) +} + +func (s *Server) handleNotifyGet(w http.ResponseWriter, _ *http.Request, u *users.User, _ *users.Session) { + s.writeNotifyAccount(w, u) +} + +func (s *Server) handleNotifyPut(w http.ResponseWriter, r *http.Request, u *users.User, _ *users.Session) { + var req notifyPrefsRequest + if !decodeJSON(w, r, &req) { + return + } + for _, e := range req.Events { + if !users.ValidNotifyEvent(e) { + writeError(w, http.StatusBadRequest, "unknown event type") + return + } + if users.IsAdminNotifyEvent(e) && u.Role != users.RoleAdmin { + writeError(w, http.StatusForbidden, "only admins can choose this event") + return + } + } + p, err := s.auth.st.SetNotifyPrefs(u.ID, req.Enabled, req.Events) + if err != nil { + notifyInternal(w, "save preferences", u.ID, err) + return + } + s.auth.audit(idPtr(u.ID), "notify.prefs", idPtr(u.ID), + map[string]string{"enabled": strconv.FormatBool(p.Enabled), "events": strings.Join(p.Events, ",")}) + s.writeNotifyAccount(w, u) +} + +func (s *Server) handleWatchPut(w http.ResponseWriter, r *http.Request, u *users.User, _ *users.Session) { + pk, ok := notifyPubkey(mux.Vars(r)["pubkey"]) + if !ok { + writeError(w, http.StatusBadRequest, "invalid node key: expected 64 hex characters") + return + } + nodes, err := s.auth.notify.src.nodes([]string{pk}, false) + if err != nil { + notifyInternal(w, "look up node", u.ID, err) + return + } + if _, found := nodes[pk]; !found { + writeError(w, http.StatusNotFound, "node not found") + return + } + if _, err := s.auth.st.NotifyPrefsFor(u.ID); err != nil { + notifyInternal(w, "preferences", u.ID, err) + return + } + limit := s.auth.set.notify.maxWatchesPerUser + if err := s.auth.st.AddWatch(u.ID, pk, limit); err != nil { + if errors.Is(err, users.ErrWatchLimit) { + writeError(w, http.StatusConflict, fmt.Sprintf("you watch the maximum of %d nodes; remove one first", limit)) + return + } + notifyInternal(w, "add watch", u.ID, err) + return + } + s.writeNotifyAccount(w, u) +} + +func (s *Server) handleWatchDelete(w http.ResponseWriter, r *http.Request, u *users.User, _ *users.Session) { + pk, ok := notifyPubkey(mux.Vars(r)["pubkey"]) + if !ok { + writeError(w, http.StatusBadRequest, "invalid node key: expected 64 hex characters") + return + } + if err := s.auth.st.RemoveWatch(u.ID, pk); err != nil { + notifyInternal(w, "remove watch", u.ID, err) + return + } + s.writeNotifyAccount(w, u) +} + +// myNodesPubkeys returns the pubkeys of the synced "meshcore-my-nodes" list +// (settings sync; [{pubkey, name, addedAt}]) in list order; nil when the +// document or the key is absent or does not decode. +func myNodesPubkeys(doc string) []string { + if doc == "" { + return nil + } + var d settingsDoc + if json.Unmarshal([]byte(doc), &d) != nil { + return nil + } + raw, ok := d.Keys["meshcore-my-nodes"] + if !ok { + return nil + } + var items []struct { + Pubkey string `json:"pubkey"` + } + if json.Unmarshal([]byte(raw), &items) != nil { + return nil + } + out := make([]string, 0, len(items)) + for _, it := range items { + out = append(out, it.Pubkey) + } + return out +} + +func (s *Server) handleWatchMyNodes(w http.ResponseWriter, _ *http.Request, u *users.User, _ *users.Session) { + st := s.auth.st + doc, _, err := st.GetSettings(u.ID) + if err != nil { + notifyInternal(w, "read synced settings", u.ID, err) + return + } + var res watchMyNodesJSON + seen := map[string]bool{} + var valid []string + for _, raw := range myNodesPubkeys(doc) { + pk, ok := notifyPubkey(raw) + if !ok { + res.Skipped++ + continue + } + if !seen[pk] { + seen[pk] = true + valid = append(valid, pk) + } + } + nodes, err := s.auth.notify.src.nodes(valid, false) + if err != nil { + notifyInternal(w, "look up nodes", u.ID, err) + return + } + known := make([]string, 0, len(valid)) + for _, pk := range valid { + if _, ok := nodes[pk]; ok { + known = append(known, pk) + } else { + res.Skipped++ + } + } + if _, err := st.NotifyPrefsFor(u.ID); err != nil { + notifyInternal(w, "preferences", u.ID, err) + return + } + added, already, over, err := st.AddWatches(u.ID, known, s.auth.set.notify.maxWatchesPerUser) + if err != nil { + notifyInternal(w, "add watches", u.ID, err) + return + } + res.Added, res.Already, res.Skipped = added, already, res.Skipped+over + if res.Account, err = s.notifyAccount(u); err != nil { + notifyInternal(w, "account state", u.ID, err) + return + } + writeJSON(w, res) +} + +// handleUnsubscribeGet sends a mail client or a scanner that follows the +// List-Unsubscribe URL to the confirm view. A GET never changes anything. +func (s *Server) handleUnsubscribeGet(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, s.auth.link("unsubscribe", r.URL.Query().Get("token")), http.StatusSeeOther) +} + +// handleUnsubscribePost turns notifications off for the token's owner: the +// confirm button and the provider's one-click POST. The token only does this. +func (s *Server) handleUnsubscribePost(w http.ResponseWriter, r *http.Request) { + uid, was, err := s.auth.st.DisableNotifyByToken(r.URL.Query().Get("token")) + if err != nil { + if !errors.Is(err, users.ErrTokenInvalid) { + log.Printf("[notify] unsubscribe: %v", err) + } + writeTokenError(w, err) + return + } + if was { + s.auth.audit(idPtr(uid), "notify.unsubscribe", idPtr(uid), map[string]string{"via": "link"}) + } + writeJSON(w, okResponse{OK: true, Message: "Notifications are off. Turn them back on from your account page."}) +} diff --git a/cmd/server/notify_handlers_test.go b/cmd/server/notify_handlers_test.go new file mode 100644 index 000000000..732dd8ad5 --- /dev/null +++ b/cmd/server/notify_handlers_test.go @@ -0,0 +1,273 @@ +package main + +import ( + "encoding/json" + "net/http/httptest" + "reflect" + "strings" + "testing" + "time" + + "github.com/meshcore-analyzer/users" +) + +func notifyPath(pk string) string { return "/api/account/notifications/watches/" + pk } + +func TestNotifyRoutesAbsentWhenOff(t *testing.T) { + f := newNotifyFixture(t, notifySettings{}) + for _, rt := range []struct{ method, path string }{ + {"GET", "/api/account/notifications"}, {"PUT", "/api/account/notifications"}, + {"PUT", notifyPath(evPkA)}, {"DELETE", notifyPath(evPkA)}, + {"POST", "/api/account/notifications/watch-my-nodes"}, + {"GET", "/api/notifications/unsubscribe?token=x"}, {"POST", "/api/notifications/unsubscribe?token=x"}, + } { + if w := f.do(rt.method, rt.path, nil); w.Code != 404 { + t.Errorf("%s %s = %d with notifications off; want 404", rt.method, rt.path, w.Code) + } + } +} + +func TestNotifyGetDefaultsAndAdminEvents(t *testing.T) { + f := newNotifyFixture(t, defaultNotifySettings()) + boss := f.registerAndActivate(t, "admin@example.org", "Boss", pw) + pat := f.registerAndActivate(t, "pat@example.org", "Pat", pw) + expectStatus(t, f.do("GET", "/api/account/notifications", nil), 401) + w := f.do("GET", "/api/account/notifications", nil, as(pat)) + expectStatus(t, w, 200) + want := notifyAccountJSON{Enabled: true, Events: []string{"node.offline", "node.battery"}, AvailableEvents: []string{"node.offline", "node.battery"}, + Watches: []notifyWatchJSON{}, Limits: notifyLimitsJSON{MaxWatches: 50, PerUserPerDay: 20}} + if got := decode[notifyAccountJSON](t, w); !reflect.DeepEqual(got, want) { + t.Fatalf("user defaults = %+v; want %+v", got, want) + } + if !strings.Contains(w.Body.String(), `"watches":[]`) { + t.Fatalf("empty watch list is not an array: %s", w.Body.String()) + } + adm := decode[notifyAccountJSON](t, f.do("GET", "/api/account/notifications", nil, as(boss))) + if !reflect.DeepEqual(adm.AvailableEvents, []string{"node.offline", "node.battery", "foreign.new", "observer.offline"}) || + !reflect.DeepEqual(adm.Events, []string{"node.offline", "node.battery"}) { + t.Fatalf("admin = %+v", adm) + } +} + +func TestNotifyPutPrefs(t *testing.T) { + f := newNotifyFixture(t, defaultNotifySettings()) + boss := f.registerAndActivate(t, "admin@example.org", "Boss", pw) + pat := f.registerAndActivate(t, "pat@example.org", "Pat", pw) + put := func(c *client, body any) *httptest.ResponseRecorder { + return f.do("PUT", "/api/account/notifications", body, as(c)) + } + expectStatus(t, put(&client{cookie: pat.cookie}, notifyPrefsRequest{Enabled: true}), 403) + w := put(pat, notifyPrefsRequest{Enabled: false, Events: []string{"node.battery"}}) + expectStatus(t, w, 200) + if got := decode[notifyAccountJSON](t, w); got.Enabled || !reflect.DeepEqual(got.Events, []string{"node.battery"}) { + t.Fatalf("after PUT = %+v", got) + } + entries, err := f.st.AuditList(users.AuditFilter{Actions: []string{"notify.prefs"}}) + if err != nil || len(entries) != 1 || *entries[0].ActorUserID != pat.me.ID || entries[0].Detail["enabled"] != "false" || entries[0].Detail["events"] != "node.battery" { + t.Fatalf("audit = %+v, %v", entries, err) + } + if w := put(pat, notifyPrefsRequest{Enabled: true, Events: []string{"node.reboot"}}); w.Code != 400 || !strings.Contains(w.Body.String(), "unknown event type") { + t.Errorf("unknown event: %d %s", w.Code, w.Body.String()) + } + if w := put(pat, notifyPrefsRequest{Enabled: true, Events: []string{"foreign.new"}}); w.Code != 403 || !strings.Contains(w.Body.String(), "only admins") { + t.Errorf("admin event by a user: %d %s", w.Code, w.Body.String()) + } + if w := put(pat, map[string]any{"enabled": true, "x": 1}); w.Code != 400 { + t.Errorf("unknown field: %d", w.Code) + } + w = put(boss, notifyPrefsRequest{Enabled: true, Events: []string{"observer.offline", "foreign.new"}}) + if got := decode[notifyAccountJSON](t, w); w.Code != 200 || !reflect.DeepEqual(got.Events, []string{"foreign.new", "observer.offline"}) { + t.Fatalf("admin PUT: %d %+v", w.Code, got) + } +} + +func TestWatchPutAndDelete(t *testing.T) { + ns := defaultNotifySettings() + ns.maxWatchesPerUser = 1 + f := newNotifyFixture(t, ns) + f.setNode(evPkA, "Alpha", "repeater", time.Hour, nil) + f.setNode(evPkB, "Bravo", "companion", time.Hour, nil) + pat := f.registerAndActivate(t, "pat@example.org", "Pat", pw) + expectStatus(t, f.do("PUT", notifyPath(evPkA), nil), 401) + expectStatus(t, f.do("PUT", notifyPath(evPkA), nil, as(&client{cookie: pat.cookie})), 403) + for _, bad := range []string{"xyz", evPkA[:63], evPkA + "0", strings.Repeat("g", 64)} { + if w := f.do("PUT", notifyPath(bad), nil, as(pat)); w.Code != 400 { + t.Errorf("PUT %q = %d; want 400", bad, w.Code) + } + } + if w := f.do("PUT", notifyPath(evPkG), nil, as(pat)); w.Code != 404 || !strings.Contains(w.Body.String(), "node not found") { + t.Fatalf("unknown node: %d %s", w.Code, w.Body.String()) + } + w := f.do("PUT", notifyPath(strings.ToUpper(evPkA)), nil, as(pat)) + expectStatus(t, w, 200) + got := decode[notifyAccountJSON](t, w) + if len(got.Watches) != 1 || got.Watches[0].Pubkey != evPkA || got.Watches[0].Name != "Alpha" || !got.Watches[0].Known { + t.Fatalf("after watching = %+v", got.Watches) + } + expectStatus(t, f.do("PUT", notifyPath(evPkA), nil, as(pat)), 200) + if w := f.do("PUT", notifyPath(evPkB), nil, as(pat)); w.Code != 409 || !strings.Contains(w.Body.String(), "you watch the maximum of 1 nodes") { + t.Fatalf("over the limit: %d %s", w.Code, w.Body.String()) + } + delete(f.src.nodeMap, evPkA) // the node left the analyzer database + if got := decode[notifyAccountJSON](t, f.do("GET", "/api/account/notifications", nil, as(pat))); got.Watches[0].Known || got.Watches[0].Name != "" { + t.Fatalf("vanished node = %+v", got.Watches[0]) + } + w = f.do("DELETE", notifyPath(evPkA), nil, as(pat)) + expectStatus(t, w, 200) + if got := decode[notifyAccountJSON](t, w); len(got.Watches) != 0 { + t.Fatalf("after DELETE = %+v", got.Watches) + } + expectStatus(t, f.do("DELETE", notifyPath(evPkA), nil, as(pat)), 200) + expectStatus(t, f.do("DELETE", notifyPath("xyz"), nil, as(pat)), 400) + if entries, _ := f.st.AuditList(users.AuditFilter{Actions: []string{"notify.*"}}); len(entries) != 0 { + t.Fatalf("watch changes were audited: %+v", entries) + } +} + +func TestMyNodesPubkeys(t *testing.T) { + for _, doc := range []string{"", "not json", `{"v":1,"keys":{}}`, `{"v":1,"keys":{"meshcore-my-nodes":"not json"}}`} { + if got := myNodesPubkeys(doc); got != nil { + t.Errorf("%q = %v; want nil", doc, got) + } + } + got := myNodesPubkeys(`{"v":1,"keys":{"meshcore-my-nodes":"[{\"pubkey\":\"ab\",\"name\":\"x\",\"addedAt\":1},{\"name\":\"no key\"}]"}}`) + if !reflect.DeepEqual(got, []string{"ab", ""}) { + t.Fatalf("myNodesPubkeys = %v", got) + } +} + +func TestWatchMyNodes(t *testing.T) { + ns := defaultNotifySettings() + ns.maxWatchesPerUser = 1 + f := newNotifyFixture(t, ns) + f.setNode(evPkA, "Alpha", "repeater", time.Hour, nil) + f.setNode(evPkB, "Bravo", "companion", time.Hour, nil) + pat := f.registerAndActivate(t, "pat@example.org", "Pat", pw) + post := func() watchMyNodesJSON { + t.Helper() + w := f.do("POST", "/api/account/notifications/watch-my-nodes", nil, as(pat)) + expectStatus(t, w, 200) + return decode[watchMyNodesJSON](t, w) + } + if r := post(); r.Added != 0 || r.Already != 0 || r.Skipped != 0 { + t.Fatalf("without synced settings = %+v", r) + } + list, _ := json.Marshal([]map[string]string{{"pubkey": strings.ToUpper(evPkA)}, {"pubkey": evPkB}, {"pubkey": evPkG}, {"pubkey": "bad"}, {"pubkey": evPkA}}) + doc, _ := json.Marshal(settingsDoc{V: 1, Keys: map[string]string{"meshcore-my-nodes": string(list)}}) + if _, err := f.st.PutSettings(pat.me.ID, users.SettingsVersion{}, string(doc)); err != nil { + t.Fatal(err) + } + // A watched, B over the limit, G not in the analyzer, "bad" malformed, the second A a duplicate. + if r := post(); r.Added != 1 || r.Already != 0 || r.Skipped != 3 || len(r.Account.Watches) != 1 || r.Account.Watches[0].Pubkey != evPkA { + t.Fatalf("first copy = %+v", r) + } + if r := post(); r.Added != 0 || r.Already != 1 || r.Skipped != 3 { + t.Fatalf("second copy = %+v", r) + } + expectStatus(t, f.do("POST", "/api/account/notifications/watch-my-nodes", nil, as(&client{cookie: pat.cookie})), 403) +} + +func TestUnsubscribe(t *testing.T) { + f := newNotifyFixture(t, defaultNotifySettings()) + pat := f.registerAndActivate(t, "pat@example.org", "Pat", pw) + p, err := f.st.NotifyPrefsFor(pat.me.ID) + if err != nil { + t.Fatal(err) + } + q := "/api/notifications/unsubscribe?token=" + p.UnsubToken + w := f.do("GET", q, nil) + if w.Code != 303 || w.Header().Get("Location") != testBase+"/#/account/unsubscribe?token="+p.UnsubToken { + t.Fatalf("GET = %d, Location %q", w.Code, w.Header().Get("Location")) + } + if again, _ := f.st.NotifyPrefsFor(pat.me.ID); !again.Enabled { + t.Fatal("a GET (a link scanner) turned notifications off") + } + // The provider's one-click POST: form body, a foreign Origin, no session. + w = f.do("POST", q, nil, header("Origin", "https://mail.example.net"), header("Content-Type", "application/x-www-form-urlencoded")) + expectStatus(t, w, 200) + if r := decode[okResponse](t, w); !r.OK || !strings.Contains(r.Message, "Notifications are off") { + t.Fatalf("POST = %+v", r) + } + if again, _ := f.st.NotifyPrefsFor(pat.me.ID); again.Enabled { + t.Fatal("still enabled after the POST") + } + expectStatus(t, f.do("POST", q, nil), 200) + entries, err := f.st.AuditList(users.AuditFilter{Actions: []string{"notify.unsubscribe"}}) + if err != nil || len(entries) != 1 || *entries[0].TargetUserID != pat.me.ID || entries[0].Detail["via"] != "link" { + t.Fatalf("audit = %+v, %v; want one row for the change", entries, err) + } + for _, bad := range []string{"/api/notifications/unsubscribe?token=nope", "/api/notifications/unsubscribe"} { + if w := f.do("POST", bad, nil); w.Code != 410 { + t.Errorf("POST %s = %d; want 410", bad, w.Code) + } + } +} + +func TestClientConfigAdvertisesNotifications(t *testing.T) { + srv, router := setupTestServer(t) + a, _ := newTestAuthService(t) + a.notify = newNotifier(a, &fakeNotifySource{}, time.Now) + srv.auth = a + w := httptest.NewRecorder() + router.ServeHTTP(w, httptest.NewRequest("GET", "/api/config/client", nil)) + if !strings.Contains(w.Body.String(), `"userManagement":{"enabled":true,"notifications":true}`) { + t.Fatalf("client config = %s", w.Body.String()) + } +} + +func TestAdminStatsNotifyBlock(t *testing.T) { + f := newNotifyFixture(t, defaultNotifySettings()) + boss := f.registerAndActivate(t, "admin@example.org", "Boss", pw) + f.watcher(t, "pat@example.org", "Pat", evPkA, evPkB) + if _, err := f.st.LogMail(nil, "gone@example.org", users.NotifyMailPurpose, ""); err != nil { + t.Fatal(err) + } + w := f.do("GET", "/api/admin/stats", nil, as(boss)) + expectStatus(t, w, 200) + st := decode[adminStatsJSON](t, w) + if st.Notify == nil || *st.Notify != (adminNotifyJSON{MailsLast24h: 1, MaxMailsPerDay: 300, Watches: 2, WatchingUsers: 1}) { + t.Fatalf("notify block = %+v", st.Notify) + } + off, offBoss, _ := adminFixture(t) + if w := off.do("GET", "/api/admin/stats", nil, as(offBoss)); strings.Contains(w.Body.String(), `"notify"`) { + t.Fatalf("notify block with notifications off: %s", w.Body.String()) + } +} + +// users.db never holds case variants: every route lowercases the key first. +func TestWatchRoutesStoreLowercaseKeys(t *testing.T) { + f := newNotifyFixture(t, defaultNotifySettings()) + f.setNode(evPkA, "Alpha", "repeater", time.Hour, nil) + f.setNode(evPkB, "Bravo", "companion", time.Hour, nil) + pat := f.registerAndActivate(t, "pat@example.org", "Pat", pw) + stored := func() []string { + t.Helper() + ws, err := f.st.WatchesFor(pat.me.ID) + if err != nil { + t.Fatal(err) + } + out := []string{} + for _, w := range ws { + out = append(out, w.Pubkey) + } + return out + } + expectStatus(t, f.do("PUT", notifyPath(strings.ToUpper(evPkA)), nil, as(pat)), 200) + expectStatus(t, f.do("PUT", notifyPath(evPkA), nil, as(pat)), 200) + if got := stored(); !reflect.DeepEqual(got, []string{evPkA}) { + t.Fatalf("after PUT upper then lower = %v", got) + } + expectStatus(t, f.do("DELETE", notifyPath(strings.ToUpper(evPkA)), nil, as(pat)), 200) + if got := stored(); len(got) != 0 { + t.Fatalf("DELETE with an uppercase key left %v", got) + } + list, _ := json.Marshal([]map[string]string{{"pubkey": strings.ToUpper(evPkB)}}) + doc, _ := json.Marshal(settingsDoc{V: 1, Keys: map[string]string{"meshcore-my-nodes": string(list)}}) + if _, err := f.st.PutSettings(pat.me.ID, users.SettingsVersion{}, string(doc)); err != nil { + t.Fatal(err) + } + expectStatus(t, f.do("POST", "/api/account/notifications/watch-my-nodes", nil, as(pat)), 200) + if got := stored(); !reflect.DeepEqual(got, []string{evPkB}) { + t.Fatalf("watch-my-nodes stored %v", got) + } +} diff --git a/cmd/server/notify_service.go b/cmd/server/notify_service.go new file mode 100644 index 000000000..4e3d2282c --- /dev/null +++ b/cmd/server/notify_service.go @@ -0,0 +1,357 @@ +package main + +import ( + "context" + "fmt" + "log" + "net/url" + "sort" + "strings" + "time" + + "github.com/meshcore-analyzer/mailer" + "github.com/meshcore-analyzer/users" +) + +// notifier evaluates node notifications on a timer and mails the changes +// (docs/specs/2026-10-07-node-notifications-design.md). authService.notify +// is nil unless userManagement.notifications.enabled. +type notifier struct { + a *authService + src notifySource + now func() time.Time + // ingestStale is the last tick's verdict, so the pause and the resume + // are each logged once; ingestResumedAt is when the last stale period + // ended (the offline grace, see notifyInput). In memory only: a restart + // inside the grace loses it. Only the loop goroutine touches them. + ingestStale bool + ingestResumedAt time.Time +} + +func newNotifier(a *authService, src notifySource, now func() time.Time) *notifier { + return ¬ifier{a: a, src: src, now: now} +} + +// notifySendTimeout bounds one mail send inside a tick. +const notifySendTimeout = 30 * time.Second + +// notifyIngestStaleAfter: when the newest packet in the store is older, +// offline checks pause (an MQTT or ingestor outage would otherwise mail +// every watcher "offline" and later "back online"). +const notifyIngestStaleAfter = 30 * time.Minute + +// checkIngest reports whether ingest is stale at now and logs when that +// verdict changes. +func (n *notifier) checkIngest(now time.Time) bool { + newest := n.src.newestPacket() + stale := newest.IsZero() || now.Sub(newest) > notifyIngestStaleAfter + switch { + case stale && !n.ingestStale: + since := "(no packets)" + if !newest.IsZero() { + since = newest.UTC().Format(time.RFC3339) + } + log.Printf("[notify] ingest stale since %s; offline checks paused", since) + case !stale && n.ingestStale: + n.ingestResumedAt = now + log.Printf("[notify] ingest fresh again; offline checks resumed") + } + n.ingestStale = stale + return stale +} + +// loop evaluates every interval until stop closes. The first evaluation +// runs one interval after startup. A tick in flight sees its context +// cancelled when stop closes, so a slow mail provider does not hold up +// the shutdown. +func (n *notifier) loop(stop <-chan struct{}, every time.Duration) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go func() { + select { + case <-stop: + cancel() + case <-ctx.Done(): + } + }() + t := time.NewTicker(every) + defer t.Stop() + for { + select { + case <-stop: + return + case <-t.C: + n.safeTick(ctx) + } + } +} + +// safeTick runs one evaluation; a panic is logged and the next tick runs. +func (n *notifier) safeTick(ctx context.Context) { + defer func() { + if r := recover(); r != nil { + log.Printf("[notify] evaluation panicked, the next one runs as planned: %v", r) + } + }() + n.tick(ctx) +} + +// tick reads all preferences, watches and states (one query each), the +// accounts and the analyzer snapshot they need, evaluates, deletes the +// dropped states and writes the new ones, and then mails each user's +// changes. Users without a preferences row and users with notifications +// off are evaluated too (the evaluator gives the first the defaults and +// keeps the second's states current without changes). A failed read or +// state write ends the tick without mail; the next tick starts over. A +// tick that evaluates logs one line with its counts, its duration and how +// long LastHeardMap held the packet store's read lock. +func (n *notifier) tick(ctx context.Context) { + if !n.src.ready() { + return + } + started := time.Now() + now := n.now() + stale := n.checkIngest(now) + st := n.a.st + prefs, err := st.AllNotifyPrefs() + if err != nil { + log.Printf("[notify] read preferences: %v", err) + return + } + stored, err := st.AllWatches() + if err != nil { + log.Printf("[notify] read watches: %v", err) + return + } + if len(prefs) == 0 && len(stored) == 0 { + return + } + states, err := st.AllNotifyStates() + if err != nil { + log.Printf("[notify] read states: %v", err) + return + } + // The analyzer snapshot is keyed by lowercase pubkey; normalise the + // watches once so every lookup and state subject uses that form. + watches := make([]users.NotifyWatch, 0, len(stored)) + watchSeen := make(map[users.NotifyKey]bool, len(stored)) + seen := map[string]bool{} + var pubkeys []string + for _, w := range stored { + w.Pubkey = strings.ToLower(w.Pubkey) + k := users.NotifyKey{UserID: w.UserID, Subject: w.Pubkey} + if watchSeen[k] { + continue + } + watchSeen[k] = true + watches = append(watches, w) + if !seen[w.Pubkey] { + seen[w.Pubkey] = true + pubkeys = append(pubkeys, w.Pubkey) + } + } + byUser := make(map[int64]users.NotifyPrefs, len(prefs)) + var ids []int64 + for _, p := range prefs { + byUser[p.UserID] = p + ids = append(ids, p.UserID) + } + noPrefs := map[int64]bool{} + for _, w := range watches { + if _, ok := byUser[w.UserID]; !ok && !noPrefs[w.UserID] { + noPrefs[w.UserID] = true + ids = append(ids, w.UserID) + } + } + accounts, err := st.UsersByID(ids) + if err != nil { + log.Printf("[notify] read accounts: %v", err) + return + } + wantForeign, wantObservers := false, false + for _, p := range prefs { + if accounts[p.UserID].Role != users.RoleAdmin { + continue + } + wantForeign = wantForeign || p.Has(users.NotifyForeignNew) + wantObservers = wantObservers || p.Has(users.NotifyObserverOffline) + } + nodes, err := n.src.nodes(pubkeys, wantForeign) + if err != nil { + log.Printf("[notify] read nodes: %v", err) + return + } + var infra []string + for _, pk := range pubkeys { + if nd, ok := nodes[pk]; ok && notifyInfra(nd.Role) { + infra = append(infra, pk) + } + } + var observers []notifyObserver + if wantObservers { + if observers, err = n.src.observers(); err != nil { + log.Printf("[notify] read observers: %v", err) + return + } + } + heard, lock := n.src.lastHeard(pubkeys) + res := evaluateNotifications(notifyInput{Now: now, Health: n.src.health(), LowMv: n.src.lowBatteryMv(), + Accounts: accounts, Prefs: prefs, Watches: watches, States: states, Nodes: nodes, + Heard: heard, Relayed: n.src.lastRelayed(infra), Observers: observers, IngestStale: stale, IngestResumedAt: n.ingestResumedAt}) + if err := st.DeleteNotifyStates(res.Drop); err != nil { + log.Printf("[notify] delete dropped states, nothing mailed this time: %v", err) + return + } + if len(res.States) > 0 { + if err := st.WriteNotifyStates(res.States); err != nil { + log.Printf("[notify] write states, nothing mailed this time: %v", err) + return + } + } + mails := n.deliver(ctx, now, res.Changes, accounts, byUser) + changes := 0 + for _, list := range res.Changes { + changes += len(list) + } + log.Printf("[notify] tick: users=%d changes=%d mails=%d took=%.2fms lock=%.2fms", + len(accounts), changes, mails, msFloat(time.Since(started)), msFloat(lock)) +} + +func msFloat(d time.Duration) float64 { return float64(d.Microseconds()) / 1000 } + +// notifySkipReason says why a user's changes are not mailed; "" means send. +// Skipped changes are already stored and are never mailed later. +func notifySkipReason(u users.User, p users.NotifyPrefs, userMails, totalMails int, ns notifySettings) string { + switch { + case u.Status != users.StatusActive: + return "account not active" + case u.EmailBouncing: + return "address bounces" + case !p.Enabled: + return "notifications off" + case userMails >= ns.perUserPerDay: + return "daily limit for this user reached" + case totalMails >= ns.maxMailsPerDay: + return "daily limit for this instance reached" + } + return "" +} + +// deliver sends one mail per user with changes, within the limits counted +// from mail_log over the last 24 hours (this tick's mails included). A user +// with changes but no preferences row was evaluated with the defaults; the +// row (and so the unsubscribe token) is created before the first mail. +// Once ctx is cancelled (shutdown) the remaining users are not mailed; +// their changes are already stored, like any other skipped change. It +// returns the number of mails sent. +func (n *notifier) deliver(ctx context.Context, now time.Time, changes map[int64][]notifyChange, + accounts map[int64]users.User, prefs map[int64]users.NotifyPrefs) int { + if len(changes) == 0 { + return 0 + } + total, perUser, err := n.a.st.NotifyMailCounts(now.Add(-24 * time.Hour)) + if err != nil { + log.Printf("[notify] count mails, nothing mailed this time: %v", err) + return 0 + } + sent := 0 + ids := make([]int64, 0, len(changes)) + for uid := range changes { + ids = append(ids, uid) + } + sort.Slice(ids, func(i, j int) bool { return ids[i] < ids[j] }) + for i, uid := range ids { + if ctx.Err() != nil { + log.Printf("[notify] shutting down: %d user(s) with changes not mailed", len(ids)-i) + return sent + } + u := accounts[uid] + p, ok := prefs[uid] + if !ok { + p = users.DefaultNotifyPrefs(uid) + } + if why := notifySkipReason(u, p, perUser[uid], total, n.a.set.notify); why != "" { + log.Printf("[notify] user #%d: %d change(s) not mailed: %s", uid, len(changes[uid]), why) + continue + } + if !ok { + if p, err = n.a.st.NotifyPrefsFor(uid); err != nil { + log.Printf("[notify] user #%d: create preferences, %d change(s) not mailed: %v", uid, len(changes[uid]), err) + continue + } + } + sendCtx, cancel := context.WithTimeout(ctx, notifySendTimeout) + err := n.a.sendMail(sendCtx, &u, users.NotifyMailPurpose, n.a.notifyMail(u, p, changes[uid])) + cancel() + if err != nil { + continue // logged by sendMail; the changes stay recorded + } + total++ + perUser[uid]++ + sent++ + } + return sent +} + +// notifyMail is the one mail for all of a user's changes in one evaluation. +func (a *authService) notifyMail(u users.User, p users.NotifyPrefs, changes []notifyChange) mailer.Message { + base := a.set.baseURL.String() + lines := make([]mailLine, 0, len(changes)) + for _, c := range changes { + lines = append(lines, mailLine{text: notifyChangeText(c), url: notifySubjectURL(base, c)}) + } + subject := fmt.Sprintf("%d changes on your watched nodes", len(changes)) + if len(changes) == 1 { + subject = "1 change on your watched nodes" + } + msg := a.render(u.Email, u.DisplayName, "notify", mailContent{ + subject: subject, greeting: "Hello " + u.DisplayName + ",", + paragraphs: []string{"Changes since the last check:"}, + lines: lines, + actionLabel: "Manage notifications", actionURL: base + "/#/account?section=notifications", + footer: &mailLine{text: "You get this mail because you asked " + a.set.baseURL.Host + + " to watch these nodes. Turn all notification mails off with one click:", url: a.link("unsubscribe", p.UnsubToken)}, + }) + msg.Headers = map[string]string{ + "List-Unsubscribe": "<" + base + "/api/notifications/unsubscribe?token=" + url.QueryEscape(p.UnsubToken) + ">", + "List-Unsubscribe-Post": "List-Unsubscribe=One-Click", + } + return msg +} + +// notifyChangeText is one line of the mail: name, what happened, when (UTC). +func notifyChangeText(c notifyChange) string { + bad := c.To == users.NotifyBad + var what string + switch c.Event { + case users.NotifyNodeOffline: + what = "back online" + if bad { + what = "offline" + } + case users.NotifyNodeBattery: + what = "battery recovered" + if bad { + what = "battery low" + } + if c.BatteryMv != nil { + what += fmt.Sprintf(" (%d mV)", *c.BatteryMv) + } + case users.NotifyForeignNew: + what = "new foreign node" + case users.NotifyObserverOffline: + what = "observer back online" + if bad { + what = "observer offline" + } + } + return c.Name + ": " + what + ", " + c.At.UTC().Format("2006-01-02 15:04 UTC") +} + +// notifySubjectURL links a change to its node or observer page. +func notifySubjectURL(base string, c notifyChange) string { + if c.Event == users.NotifyObserverOffline { + return base + "/#/observers/" + url.PathEscape(c.Subject) + } + return base + "/#/nodes/" + url.PathEscape(c.Subject) +} diff --git a/cmd/server/notify_service_test.go b/cmd/server/notify_service_test.go new file mode 100644 index 000000000..69ac180ff --- /dev/null +++ b/cmd/server/notify_service_test.go @@ -0,0 +1,729 @@ +package main + +import ( + "context" + "errors" + "regexp" + "strings" + "testing" + "time" + + "github.com/gorilla/mux" + "github.com/meshcore-analyzer/mailer" + "github.com/meshcore-analyzer/users" +) + +type fakeNotifySource struct { + isReady bool + hs HealthThresholds + low int + nodeMap map[string]notifyNode + heard map[string]time.Time + relayed map[string]time.Time + obs []notifyObserver + newestAt func() time.Time // newest packet in the store; nil means none + lock time.Duration // what lastHeard reports as its lock time + panicNext bool + heardAsked []string + foreignAsked int + observersAsked int +} + +func (f *fakeNotifySource) ready() bool { return f.isReady } +func (f *fakeNotifySource) health() HealthThresholds { return f.hs } +func (f *fakeNotifySource) lowBatteryMv() int { return f.low } +func (f *fakeNotifySource) nodes(pubkeys []string, withForeign bool) (map[string]notifyNode, error) { + if f.panicNext { + f.panicNext = false + panic("fake source panic") + } + out := map[string]notifyNode{} + for _, pk := range pubkeys { + if n, ok := f.nodeMap[pk]; ok { + out[pk] = n + } + } + if withForeign { + f.foreignAsked++ + for pk, n := range f.nodeMap { + if n.Foreign { + out[pk] = n + } + } + } + return out, nil +} +func (f *fakeNotifySource) lastHeard(pks []string) (map[string]time.Time, time.Duration) { + f.heardAsked = append([]string(nil), pks...) + return f.heard, f.lock +} +func (f *fakeNotifySource) lastRelayed([]string) map[string]time.Time { return f.relayed } +func (f *fakeNotifySource) newestPacket() time.Time { + if f.newestAt == nil { + return time.Time{} + } + return f.newestAt() +} +func (f *fakeNotifySource) observers() ([]notifyObserver, error) { + f.observersAsked++ + return f.obs, nil +} + +type notifyClock struct{ t time.Time } + +func (c *notifyClock) Now() time.Time { return c.t } +func (c *notifyClock) Advance(d time.Duration) { c.t = c.t.Add(d) } + +func defaultNotifySettings() notifySettings { + return notifySettings{enabled: true, interval: 5 * time.Minute, perUserPerDay: 20, maxMailsPerDay: 300, maxWatchesPerUser: 50} +} + +type notifyFixture struct { + *authFixture + n *notifier + src *fakeNotifySource + clk *notifyClock +} + +// newNotifyFixture is newAuthFixture with notifications set to ns, a fake +// analyzer source and one fixed clock for the notifier and users.db. +// admin@example.org is a config admin. +func newNotifyFixture(t *testing.T, ns notifySettings) *notifyFixture { + t.Helper() + a, fake, ownerURL := newTestAuthServiceWithURL(t, postgresTestDSN(t), "admin@example.org") + a.set.notify = ns + clk := ¬ifyClock{t: time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC)} + a.st.SetClock(clk.Now) + src := &fakeNotifySource{isReady: true, hs: (&Config{}).GetHealthThresholds(), low: 3300, + nodeMap: map[string]notifyNode{}, heard: map[string]time.Time{}, relayed: map[string]time.Time{}, newestAt: clk.Now} + if ns.enabled { + a.notify = newNotifier(a, src, clk.Now) + } + srv := &Server{cfg: &Config{APIKey: testAPIKey}, perfStats: NewPerfStats(), auth: a} + r := mux.NewRouter() + srv.registerAuthRoutes(r) + return ¬ifyFixture{authFixture: &authFixture{srv: srv, router: r, fake: fake, st: a.st, ownerURL: ownerURL}, n: a.notify, src: src, clk: clk} +} + +// watcher registers and activates a user with default prefs watching pubkeys. +func (f *notifyFixture) watcher(t *testing.T, email, name string, pubkeys ...string) *client { + t.Helper() + c := f.registerAndActivate(t, email, name, pw) + if _, err := f.st.NotifyPrefsFor(c.me.ID); err != nil { + t.Fatal(err) + } + if _, _, _, err := f.st.AddWatches(c.me.ID, pubkeys, 0); err != nil { + t.Fatal(err) + } + return c +} + +// setNode puts a node in the fake analyzer, last seen lastSeen before now. +func (f *notifyFixture) setNode(pk, name, role string, lastSeen time.Duration, battery *int) { + f.src.nodeMap[pk] = notifyNode{Pubkey: pk, Name: name, Role: role, LastSeen: f.clk.t.Add(-lastSeen).Format(time.RFC3339), BatteryMv: battery} +} + +func (f *notifyFixture) tick() { f.n.tick(context.Background()) } + +func (f *notifyFixture) notifyMails() []mailer.Message { + var out []mailer.Message + for _, m := range f.fake.Sent() { + if m.Tag == "notify" { + out = append(out, m) + } + } + return out +} + +func TestNotifierMailsAChangeAfterTheSilentFirstTick(t *testing.T) { + f := newNotifyFixture(t, defaultNotifySettings()) + pat := f.watcher(t, "pat@example.org", "Pat", evPkA) + f.setNode(evPkA, "Alpha", "companion", time.Hour, nil) + f.tick() + if m := f.notifyMails(); len(m) != 0 { + t.Fatalf("first tick mailed: %+v", m) + } + f.clk.Advance(25 * time.Hour) + f.tick() + m := f.notifyMails() + if len(m) != 1 { + t.Fatalf("mails = %d; want 1", len(m)) + } + p, _ := f.st.NotifyPrefsFor(pat.me.ID) + if m[0].To != "pat@example.org" || m[0].Subject != "[CoreScope] 1 change on your watched nodes" || + !strings.Contains(m[0].Text, "Alpha: offline, 2026-10-08 13:00 UTC") || + !strings.Contains(m[0].Text, testBase+"/#/nodes/"+evPkA) || + !strings.Contains(m[0].Text, testBase+"/#/account?section=notifications") || + !strings.Contains(m[0].Text, testBase+"/#/account/unsubscribe?token="+p.UnsubToken) { + t.Fatalf("mail = %+v", m[0]) + } + if m[0].Headers["List-Unsubscribe"] != "<"+testBase+"/api/notifications/unsubscribe?token="+p.UnsubToken+">" || + m[0].Headers["List-Unsubscribe-Post"] != "List-Unsubscribe=One-Click" { + t.Fatalf("headers = %v", m[0].Headers) + } + if total, per, err := f.st.NotifyMailCounts(f.clk.t.Add(-time.Hour)); err != nil || total != 1 || per[pat.me.ID] != 1 { + t.Fatalf("mail_log notify rows = %d, %v, %v", total, per, err) + } + f.clk.Advance(5 * time.Minute) + f.tick() + if len(f.notifyMails()) != 1 { + t.Fatal("an unchanged state was mailed again") + } + f.src.heard[evPkA] = f.clk.t.Add(-time.Minute) + f.tick() + if m := f.notifyMails(); len(m) != 2 || !strings.Contains(m[1].Text, "Alpha: back online") { + t.Fatalf("back-online mail = %+v", m) + } +} + +func TestNotifierOneMailPerUserPerTick(t *testing.T) { + f := newNotifyFixture(t, defaultNotifySettings()) + f.watcher(t, "pat@example.org", "Pat", evPkA, evPkB) + f.watcher(t, "quinn@example.org", "Quinn", evPkA) + f.setNode(evPkA, "Alpha", "companion", time.Hour, nil) + f.setNode(evPkB, "Bravo", "companion", time.Hour, nil) + f.tick() + f.clk.Advance(25 * time.Hour) + f.tick() + m := f.notifyMails() + if len(m) != 2 { + t.Fatalf("mails = %d; want one per user", len(m)) + } + byTo := map[string]mailer.Message{m[0].To: m[0], m[1].To: m[1]} + pat, quinn := byTo["pat@example.org"], byTo["quinn@example.org"] + if pat.Subject != "[CoreScope] 2 changes on your watched nodes" || !strings.Contains(pat.Text, "Alpha: offline") || !strings.Contains(pat.Text, "Bravo: offline") { + t.Fatalf("Pat's mail = %+v", pat) + } + if quinn.Subject != "[CoreScope] 1 change on your watched nodes" || strings.Contains(quinn.Text, "Bravo") { + t.Fatalf("Quinn's mail = %+v", quinn) + } +} + +func TestNotifierSkipsAreNeverMailedLater(t *testing.T) { + cases := []struct { + name string + tune func(*notifySettings) + block func(*notifyFixture, int64) error + lift func(*notifyFixture, int64) error + }{ + {"notifications off", nil, + func(f *notifyFixture, uid int64) error { + _, err := f.st.SetNotifyPrefs(uid, false, users.NodeNotifyEvents) + return err + }, + func(f *notifyFixture, uid int64) error { + _, err := f.st.SetNotifyPrefs(uid, true, users.NodeNotifyEvents) + return err + }}, + {"address bounces", nil, + func(f *notifyFixture, uid int64) error { return f.st.SetEmailBouncing(uid, true) }, + func(f *notifyFixture, uid int64) error { return f.st.SetEmailBouncing(uid, false) }}, + {"account disabled", nil, + func(f *notifyFixture, uid int64) error { return f.st.SetStatus(uid, users.StatusDisabled) }, + func(f *notifyFixture, uid int64) error { return f.st.SetStatus(uid, users.StatusActive) }}, + {"per-user limit", func(ns *notifySettings) { ns.perUserPerDay = 1 }, + func(f *notifyFixture, uid int64) error { + _, err := f.st.LogMail(&uid, "pat@example.org", users.NotifyMailPurpose, "") + return err + }, + func(f *notifyFixture, uid int64) error { f.clk.Advance(25 * time.Hour); return nil }}, + {"instance limit", func(ns *notifySettings) { ns.maxMailsPerDay = 1 }, + func(f *notifyFixture, uid int64) error { + _, err := f.st.LogMail(nil, "gone@example.org", users.NotifyMailPurpose, "") + return err + }, + func(f *notifyFixture, uid int64) error { f.clk.Advance(25 * time.Hour); return nil }}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + ns := defaultNotifySettings() + if c.tune != nil { + c.tune(&ns) + } + f := newNotifyFixture(t, ns) + pat := f.watcher(t, "pat@example.org", "Pat", evPkA) + f.setNode(evPkA, "Alpha", "companion", time.Hour, nil) + f.tick() + // Advance first, so a blocking mail_log row lies inside the + // rolling 24 hours of the next tick. + f.clk.Advance(25 * time.Hour) + if err := c.block(f, pat.me.ID); err != nil { + t.Fatal(err) + } + f.tick() + if m := f.notifyMails(); len(m) != 0 { + t.Fatalf("mailed while blocked: %+v", m) + } + if err := c.lift(f, pat.me.ID); err != nil { + t.Fatal(err) + } + f.tick() + if m := f.notifyMails(); len(m) != 0 { + t.Fatalf("a skipped change was mailed later: %+v", m) + } + }) + } +} + +func TestNotifierRestartAndNewWatchStaySilent(t *testing.T) { + f := newNotifyFixture(t, defaultNotifySettings()) + pat := f.watcher(t, "pat@example.org", "Pat", evPkA) + f.setNode(evPkA, "Alpha", "companion", time.Hour, nil) + f.tick() + f.clk.Advance(25 * time.Hour) + f.tick() + if len(f.notifyMails()) != 1 { + t.Fatal("no mail for the change") + } + restarted := newNotifier(f.srv.auth, f.src, f.clk.Now) // the states live in users.db + f.setNode(evPkB, "Bravo", "companion", 48*time.Hour, nil) + if _, _, _, err := f.st.AddWatches(pat.me.ID, []string{evPkB}, 0); err != nil { + t.Fatal(err) + } + f.clk.Advance(5 * time.Minute) + restarted.tick(context.Background()) + if m := f.notifyMails(); len(m) != 1 { + t.Fatalf("the restart or the new watch mailed: %+v", m[1:]) + } +} + +func TestNotifierSendFailureIsNotRetried(t *testing.T) { + f := newNotifyFixture(t, defaultNotifySettings()) + f.watcher(t, "pat@example.org", "Pat", evPkA) + f.setNode(evPkA, "Alpha", "companion", time.Hour, nil) + f.tick() + f.fake.SetSendErr(errors.New("provider down")) + f.clk.Advance(25 * time.Hour) + f.tick() + f.fake.SetSendErr(nil) + f.clk.Advance(5 * time.Minute) + f.tick() + if m := f.notifyMails(); len(m) != 0 { + t.Fatalf("a failed send was retried: %+v", m) + } + if total, _, _ := f.st.NotifyMailCounts(f.clk.t.Add(-48 * time.Hour)); total != 0 { + t.Fatalf("mail_log has %d notify rows for a mail that never left", total) + } +} + +func TestNotifierStateWriteFailureSendsNothing(t *testing.T) { + f := newNotifyFixture(t, defaultNotifySettings()) + f.watcher(t, "pat@example.org", "Pat", evPkA) + f.setNode(evPkA, "Alpha", "companion", time.Hour, nil) + f.tick() + f.execDB(t, testNativeSQL(`CREATE TRIGGER notify_state_fail BEFORE UPDATE ON notification_state BEGIN SELECT RAISE(ABORT,'boom'); END`, `CREATE FUNCTION notify_state_fail() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'boom'; END $$; CREATE TRIGGER notify_state_fail BEFORE UPDATE ON notification_state FOR EACH ROW EXECUTE FUNCTION notify_state_fail()`)) + f.clk.Advance(25 * time.Hour) + f.tick() + if m := f.notifyMails(); len(m) != 0 { + t.Fatalf("mailed although the states were not written: %+v", m) + } + f.execDB(t, testNativeSQL(`DROP TRIGGER notify_state_fail`, `DROP TRIGGER notify_state_fail ON notification_state`)) + f.tick() + if m := f.notifyMails(); len(m) != 1 { + t.Fatalf("mails after the write works again = %d; want 1", len(m)) + } +} + +func TestNotifierWaitsForTheAnalyzer(t *testing.T) { + f := newNotifyFixture(t, defaultNotifySettings()) + f.watcher(t, "pat@example.org", "Pat", evPkA) + f.setNode(evPkA, "Alpha", "companion", time.Hour, nil) + f.src.isReady = false + f.tick() + if states, _ := f.st.AllNotifyStates(); len(states) != 0 { + t.Fatalf("evaluated while the store was loading: %+v", states) + } + f.src.isReady = true + f.tick() + if states, _ := f.st.AllNotifyStates(); len(states) != 1 { + t.Fatalf("states once ready = %+v", states) + } +} + +func TestNotifierPanicIsRecovered(t *testing.T) { + f := newNotifyFixture(t, defaultNotifySettings()) + f.watcher(t, "pat@example.org", "Pat", evPkA) + f.setNode(evPkA, "Alpha", "companion", time.Hour, nil) + f.src.panicNext = true + f.n.safeTick(context.Background()) + f.tick() + if states, _ := f.st.AllNotifyStates(); len(states) != 1 { + t.Fatalf("the tick after the panic did not run: %+v", states) + } +} + +func TestNotifierReadsForeignAndObserversOnlyForAdmins(t *testing.T) { + f := newNotifyFixture(t, defaultNotifySettings()) + f.watcher(t, "pat@example.org", "Pat", evPkA) + f.setNode(evPkA, "Alpha", "companion", time.Hour, nil) + f.tick() + if f.src.foreignAsked != 0 || f.src.observersAsked != 0 { + t.Fatalf("admin data read without an admin asking: foreign %d, observers %d", f.src.foreignAsked, f.src.observersAsked) + } + boss := f.registerAndActivate(t, "admin@example.org", "Boss", pw) + if _, err := f.st.SetNotifyPrefs(boss.me.ID, true, []string{users.NotifyForeignNew}); err != nil { + t.Fatal(err) + } + f.tick() + if f.src.foreignAsked != 1 || f.src.observersAsked != 0 { + t.Fatalf("foreign %d, observers %d; want 1, 0", f.src.foreignAsked, f.src.observersAsked) + } + if err := f.st.SetRole(boss.me.ID, users.RoleUser); err != nil { + t.Fatal(err) + } + f.tick() + if f.src.foreignAsked != 1 { + t.Fatal("foreign nodes read for a demoted admin") + } +} + +func TestNotifyMailEscapesNodeNames(t *testing.T) { + f := newNotifyFixture(t, defaultNotifySettings()) + f.watcher(t, "pat@example.org", "Pat", evPkA) + f.setNode(evPkA, `Bad & "co"`, "companion", time.Hour, nil) + f.tick() + f.clk.Advance(25 * time.Hour) + f.tick() + m := f.notifyMails() + if len(m) != 1 || strings.Contains(m[0].HTML, "Bad") || !strings.Contains(m[0].HTML, "<b>Bad</b> & "co"") { + t.Fatalf("HTML = %q", m[0].HTML) + } +} + +// Names come from whoever owns a node or runs an observer: a line break +// must not forge a line in the mail, a bidi override must not reorder it. +func TestNotifyMailSanitisesNames(t *testing.T) { + f := newNotifyFixture(t, defaultNotifySettings()) + f.watcher(t, "pat@example.org", "Pat", evPkA) + admin := f.registerAndActivate(t, "admin@example.org", "Ada", pw) + if _, err := f.st.SetNotifyPrefs(admin.me.ID, true, []string{users.NotifyObserverOffline}); err != nil { + t.Fatal(err) + } + f.setNode(evPkA, "x\r\nAccount suspended, sign in: https://evil.example\t\u202egnp.exe", "companion", time.Hour, nil) + seen := f.clk.t.Add(-time.Minute).Format(time.RFC3339) + f.src.obs = []notifyObserver{{ID: "OBS1", Name: "Roof\n\u2066Fake line\u2069", LastSeen: seen}, {ID: "OBS2", Name: "\n\u202e", LastSeen: seen}} + f.tick() + f.clk.Advance(25 * time.Hour) + f.tick() + m := f.notifyMails() + if len(m) != 2 { + t.Fatalf("mails = %d, want 2", len(m)) + } + for _, msg := range m { + for _, part := range []string{msg.Text, msg.HTML} { + for _, bad := range []string{"\r", "\t", "\u202e", "\u2066", "\u2069", "\nAccount", "\nFake"} { + if strings.Contains(part, bad) { + t.Fatalf("mail part contains %q:\n%s", bad, part) + } + } + } + } + body := m[0].Text + m[1].Text + for _, want := range []string{"- x Account suspended, sign in: https://evil.example gnp.exe: offline,", + "- Roof Fake line: observer offline,", "- OBS2: observer offline,"} { + if !strings.Contains(body, want) { + t.Fatalf("text lacks %q:\n%s", want, body) + } + } +} + +func TestMailSafeText(t *testing.T) { + cases := map[string]string{ + "plain": "plain", + " a \n\n b\r\tc ": "a b c", + "a\u202eb\u2066c\u2069d": "a b c d", + "a\u202ab\u202dc": "a b c", + "a\u0085b\x7fc": "a b c", + "\n\u202a": "", + "Zo\u00eb \u00e9": "Zo\u00eb \u00e9", + } + for in, want := range cases { + if got := mailSafeText(in); got != want { + t.Errorf("mailSafeText(%q) = %q; want %q", in, got, want) + } + } +} + +func TestNotifyChangeText(t *testing.T) { + at := time.Date(2026, 10, 7, 9, 5, 0, 0, time.UTC) + cases := []struct { + c notifyChange + want string + }{ + {notifyChange{Event: users.NotifyNodeOffline, Name: "A", To: users.NotifyBad, At: at}, "A: offline, 2026-10-07 09:05 UTC"}, + {notifyChange{Event: users.NotifyNodeOffline, Name: "A", To: users.NotifyGood, At: at}, "A: back online, 2026-10-07 09:05 UTC"}, + {notifyChange{Event: users.NotifyNodeBattery, Name: "A", To: users.NotifyBad, BatteryMv: evMv(3250), At: at}, "A: battery low (3250 mV), 2026-10-07 09:05 UTC"}, + {notifyChange{Event: users.NotifyNodeBattery, Name: "A", To: users.NotifyGood, BatteryMv: evMv(3410), At: at}, "A: battery recovered (3410 mV), 2026-10-07 09:05 UTC"}, + {notifyChange{Event: users.NotifyForeignNew, Name: "F", To: users.NotifyTold, At: at}, "F: new foreign node, 2026-10-07 09:05 UTC"}, + {notifyChange{Event: users.NotifyObserverOffline, Name: "O", To: users.NotifyBad, At: at}, "O: observer offline, 2026-10-07 09:05 UTC"}, + {notifyChange{Event: users.NotifyObserverOffline, Name: "O", To: users.NotifyGood, At: at}, "O: observer back online, 2026-10-07 09:05 UTC"}, + } + for _, c := range cases { + if got := notifyChangeText(c.c); got != c.want { + t.Errorf("%q; want %q", got, c.want) + } + } + if got := notifySubjectURL("https://e.org", notifyChange{Event: users.NotifyObserverOffline, Subject: "AB CD"}); got != "https://e.org/#/observers/AB%20CD" { + t.Errorf("observer URL = %q", got) + } + if got := notifySubjectURL("https://e.org", notifyChange{Event: users.NotifyNodeOffline, Subject: evPkA}); got != "https://e.org/#/nodes/"+evPkA { + t.Errorf("node URL = %q", got) + } +} + +func TestNotifySkipReason(t *testing.T) { + ns := defaultNotifySettings() + ok := users.User{Status: users.StatusActive} + on := users.NotifyPrefs{Enabled: true} + cases := []struct { + u users.User + p users.NotifyPrefs + mine, total int + want string + }{ + {ok, on, 0, 0, ""}, + {users.User{Status: users.StatusDisabled}, on, 0, 0, "account not active"}, + {users.User{Status: users.StatusActive, EmailBouncing: true}, on, 0, 0, "address bounces"}, + {ok, users.NotifyPrefs{}, 0, 0, "notifications off"}, + {ok, on, 20, 0, "daily limit for this user reached"}, + {ok, on, 19, 300, "daily limit for this instance reached"}, + {ok, on, 19, 299, ""}, + } + for _, c := range cases { + if got := notifySkipReason(c.u, c.p, c.mine, c.total, ns); got != c.want { + t.Errorf("%+v %+v %d %d = %q; want %q", c.u, c.p, c.mine, c.total, got, c.want) + } + } +} + +func TestInitUserManagementStartsTheNotifierOnlyWhenEnabled(t *testing.T) { + for _, on := range []bool{false, true} { + a, _ := newTestAuthService(t) + um := validUM() + um.DatabaseURL = a.set.databaseURL + backupOff := false + um.Backup = &UsersBackupConfig{Enabled: &backupOff} + if on { + um.Notifications = &NotificationsConfig{Enabled: true} + } + srv := &Server{cfg: &Config{UserManagement: um}} + if err := srv.initUserManagement(testDatabaseDSN(t)); err != nil { + t.Fatal(err) + } + if (srv.auth.notify != nil) != on { + t.Fatalf("notifications %v: notifier = %v", on, srv.auth.notify) + } + done := make(chan struct{}) + go func() { srv.closeUserManagement(); close(done) }() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("closeUserManagement did not stop the notifier") + } + } +} + +func TestNotifierDeletesAdminStatesOfADemotedAdmin(t *testing.T) { + f := newNotifyFixture(t, defaultNotifySettings()) + boss := f.registerAndActivate(t, "admin@example.org", "Boss", pw) + if _, err := f.st.SetNotifyPrefs(boss.me.ID, true, []string{users.NotifyForeignNew}); err != nil { + t.Fatal(err) + } + f.src.nodeMap[evPkB] = notifyNode{Pubkey: evPkB, Name: "Far", Foreign: true} + f.tick() + if states, _ := f.st.AllNotifyStates(); len(states) != 2 { + t.Fatalf("baseline states = %+v; want the sentinel and Far", states) + } + if err := f.st.SetRole(boss.me.ID, users.RoleUser); err != nil { + t.Fatal(err) + } + f.tick() + if states, _ := f.st.AllNotifyStates(); len(states) != 0 { + t.Fatalf("states of a demoted admin = %+v; want none", states) + } + if err := f.st.SetRole(boss.me.ID, users.RoleAdmin); err != nil { + t.Fatal(err) + } + f.tick() + if m := f.notifyMails(); len(m) != 0 { + t.Fatalf("re-promotion mailed: %+v", m) + } +} + +func TestNotifierMailsAUserWithoutAPrefsRow(t *testing.T) { + f := newNotifyFixture(t, defaultNotifySettings()) + pat := f.registerAndActivate(t, "pat@example.org", "Pat", pw) + if _, _, _, err := f.st.AddWatches(pat.me.ID, []string{evPkA}, 0); err != nil { + t.Fatal(err) + } + f.setNode(evPkA, "Alpha", "companion", time.Hour, nil) + f.tick() + if states, _ := f.st.AllNotifyStates(); len(states) != 1 { + t.Fatalf("states = %+v; want the default node.offline baseline", states) + } + if prefs, _ := f.st.AllNotifyPrefs(); len(prefs) != 0 { + t.Fatalf("a silent tick created prefs: %+v", prefs) + } + f.clk.Advance(25 * time.Hour) + f.tick() + m := f.notifyMails() + p, err := f.st.NotifyPrefsFor(pat.me.ID) + if err != nil { + t.Fatal(err) + } + if len(m) != 1 || p.UnsubToken == "" || !strings.Contains(m[0].Text, "unsubscribe?token="+p.UnsubToken) || + m[0].Headers["List-Unsubscribe"] != "<"+testBase+"/api/notifications/unsubscribe?token="+p.UnsubToken+">" { + t.Fatalf("mail = %+v, prefs = %+v", m, p) + } +} + +func TestNotifierNormalisesMixedCaseWatches(t *testing.T) { + f := newNotifyFixture(t, defaultNotifySettings()) + pat := f.watcher(t, "pat@example.org", "Pat", strings.ToUpper(evPkA)) + if _, _, _, err := f.st.AddWatches(pat.me.ID, []string{evPkA}, 0); err != nil { + t.Fatal(err) + } + f.setNode(evPkA, "Alpha", "companion", time.Hour, nil) // keyed lowercase, like NotifyNodes + f.src.heard[evPkA] = f.clk.t.Add(-time.Minute) + f.tick() + if len(f.src.heardAsked) != 1 || f.src.heardAsked[0] != evPkA { + t.Fatalf("last heard asked for %v; want [%s]", f.src.heardAsked, evPkA) + } + states, _ := f.st.AllNotifyStates() + if len(states) != 1 || states[0].Subject != evPkA || states[0].State != users.NotifyGood { + t.Fatalf("states = %+v; want one good row under the lowercase key", states) + } + delete(f.src.heard, evPkA) + f.clk.Advance(25 * time.Hour) + f.tick() + if m := f.notifyMails(); len(m) != 1 || !strings.Contains(m[0].Text, "Alpha: offline") || + strings.Count(m[0].Text, "Alpha:") != 1 { + t.Fatalf("mail = %+v; want one Alpha offline line", m) + } +} + +// cancelAfterSend cancels the tick's context once a mail went out, like a +// shutdown arriving mid-delivery. +type cancelAfterSend struct { + mailer.Mailer + cancel context.CancelFunc +} + +func (c cancelAfterSend) Send(ctx context.Context, m mailer.Message) (string, error) { + id, err := c.Mailer.Send(ctx, m) + c.cancel() + return id, err +} + +func TestNotifierStopsDeliveringAfterShutdown(t *testing.T) { + f := newNotifyFixture(t, defaultNotifySettings()) + f.watcher(t, "pat@example.org", "Pat", evPkA) + f.watcher(t, "quinn@example.org", "Quinn", evPkA) + f.setNode(evPkA, "Alpha", "companion", time.Hour, nil) + f.tick() + f.clk.Advance(25 * time.Hour) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + f.n.a.mail = cancelAfterSend{Mailer: f.n.a.mail, cancel: cancel} + f.n.tick(ctx) + if m := f.notifyMails(); len(m) != 1 { + t.Fatalf("mails after a shutdown mid-delivery = %d; want 1", len(m)) + } +} + +func TestNotifierPausesOfflineChecksWhileIngestIsStale(t *testing.T) { + f := newNotifyFixture(t, defaultNotifySettings()) + f.watcher(t, "pat@example.org", "Pat", evPkA) + f.setNode(evPkA, "Alpha", "companion", time.Hour, nil) + f.tick() // baseline: online + lastPacket := f.clk.t + f.src.newestAt = func() time.Time { return lastPacket } + + logs := captureLog(func() { + f.clk.Advance(25 * time.Hour) // feed down for a day: Alpha looks offline + f.tick() + f.clk.Advance(5 * time.Minute) + f.tick() + }) + if m := f.notifyMails(); len(m) != 0 { + t.Fatalf("mailed %d while ingest is stale", len(m)) + } + paused := "[notify] ingest stale since " + lastPacket.UTC().Format(time.RFC3339) + "; offline checks paused" + if n := strings.Count(logs, paused); n != 1 { + t.Fatalf("pause logged %d times; want once:\n%s", n, logs) + } + + f.src.newestAt = f.clk.Now // feed back, Alpha still silent + logs = captureLog(f.tick) + if m := f.notifyMails(); len(m) != 0 { + t.Fatalf("mailed %d on the first tick after recovery; offline waits one silent window", len(m)) + } + if n := strings.Count(logs, "[notify] ingest fresh again; offline checks resumed"); n != 1 { + t.Fatalf("resume logged %d times; want once:\n%s", n, logs) + } + f.clk.Advance(23 * time.Hour) + f.tick() + if m := f.notifyMails(); len(m) != 0 { + t.Fatalf("mailed %d inside the window after recovery", len(m)) + } + f.clk.Advance(time.Hour + time.Minute) // a full companion window after recovery + f.tick() + if m := f.notifyMails(); len(m) != 1 || !strings.Contains(m[0].Text, "Alpha: offline") { + t.Fatalf("mails one window after recovery = %+v; want Alpha offline", m) + } +} + +// A store that is already stale at startup gets the same grace once fresh. +func TestNotifierGraceWhenStaleAtStartup(t *testing.T) { + f := newNotifyFixture(t, defaultNotifySettings()) + f.watcher(t, "pat@example.org", "Pat", evPkA) + f.setNode(evPkA, "Alpha", "companion", time.Hour, nil) + f.tick() // baseline: online (states persist in users.db) + f.n = newNotifier(f.n.a, f.src, f.clk.Now) // restart + f.clk.Advance(25 * time.Hour) + old := f.clk.t.Add(-25 * time.Hour) + f.src.newestAt = func() time.Time { return old } + f.tick() // stale at startup + f.src.newestAt = f.clk.Now + f.tick() + if m := f.notifyMails(); len(m) != 0 { + t.Fatalf("mailed %d right after a stale startup recovered", len(m)) + } + f.clk.Advance(24*time.Hour + time.Minute) + f.tick() + if m := f.notifyMails(); len(m) != 1 { + t.Fatalf("mails one window after recovery = %d; want 1", len(m)) + } +} + +func TestNotifierTreatsAnEmptyStoreAsStale(t *testing.T) { + f := newNotifyFixture(t, defaultNotifySettings()) + f.watcher(t, "pat@example.org", "Pat", evPkA) + f.setNode(evPkA, "Alpha", "companion", time.Hour, nil) + f.src.newestAt = nil + if logs := captureLog(f.tick); !strings.Contains(logs, "[notify] ingest stale since (no packets); offline checks paused") { + t.Fatalf("log = %q", logs) + } + if s, err := f.st.AllNotifyStates(); err != nil || len(s) != 0 { + t.Fatalf("states with no packets in the store = %+v, %v", s, err) + } +} + +func TestNotifierLogsOneLinePerTick(t *testing.T) { + f := newNotifyFixture(t, defaultNotifySettings()) + f.watcher(t, "pat@example.org", "Pat", evPkA, evPkB) + f.watcher(t, "quinn@example.org", "Quinn", evPkB) + f.setNode(evPkA, "Alpha", "companion", time.Hour, nil) + f.setNode(evPkB, "Bravo", "companion", time.Hour, nil) + f.src.lock = 1500 * time.Microsecond + logs := captureLog(f.tick) + if !regexp.MustCompile(`\[notify\] tick: users=2 changes=0 mails=0 took=\d+\.\d\dms lock=1\.50ms`).MatchString(logs) { + t.Fatalf("first tick log = %q", logs) + } + f.clk.Advance(25 * time.Hour) + f.src.newestAt = f.clk.Now + logs = captureLog(f.tick) + if n := strings.Count(logs, "[notify] tick:"); n != 1 || !strings.Contains(logs, "users=2 changes=3 mails=2 ") { + t.Fatalf("second tick log (%d lines) = %q", n, logs) + } +} diff --git a/cmd/server/notify_source.go b/cmd/server/notify_source.go new file mode 100644 index 000000000..d9a3409d2 --- /dev/null +++ b/cmd/server/notify_source.go @@ -0,0 +1,205 @@ +package main + +import ( + "database/sql" + "strings" + "time" +) + +// notifySource is the analyzer side the notifier reads. serverNotifySource +// is the production one; tests substitute a fake. +type notifySource interface { + ready() bool // false while the packet store is still loading after startup + health() HealthThresholds + lowBatteryMv() int + nodes(pubkeys []string, withForeign bool) (map[string]notifyNode, error) + lastHeard(pubkeys []string) (map[string]time.Time, time.Duration) // and how long the store lock was held + lastRelayed(pubkeys []string) map[string]time.Time + observers() ([]notifyObserver, error) + newestPacket() time.Time // zero when the store holds no packet +} + +type serverNotifySource struct{ s *Server } + +// ready needs /api/healthz readiness (best-observation pick and neighbor +// graph done) and the finished startup load: hot window and background +// fill. Readiness alone can come while the newest packets are still +// loading. +func (src serverNotifySource) ready() bool { + if readiness.Load() == 0 || src.s.store == nil { + return false + } + select { + case <-src.s.store.StartupLoadDone(): + return true + default: + return false + } +} + +func (src serverNotifySource) health() HealthThresholds { return src.s.cfg.GetHealthThresholds() } +func (src serverNotifySource) lowBatteryMv() int { return src.s.cfg.LowBatteryMv() } + +func (src serverNotifySource) nodes(pubkeys []string, withForeign bool) (map[string]notifyNode, error) { + return src.s.db.NotifyNodes(pubkeys, withForeign) +} + +func (src serverNotifySource) lastHeard(pubkeys []string) (map[string]time.Time, time.Duration) { + if src.s.store == nil { + return map[string]time.Time{}, 0 + } + return src.s.store.LastHeardMap(pubkeys) +} + +// lastRelayed reads the cached relay metrics (the map /api/nodes uses; the +// background recomputer refreshes it). Pass repeaters and rooms only. +func (src serverNotifySource) lastRelayed(pubkeys []string) map[string]time.Time { + if src.s.store == nil || len(pubkeys) == 0 { + return map[string]time.Time{} + } + return relayTimes(src.s.store.GetRepeaterRelayInfoMap(src.s.cfg.GetHealthThresholds().RelayActiveHours), pubkeys) +} + +func (src serverNotifySource) newestPacket() time.Time { + if src.s.store == nil { + return time.Time{} + } + return src.s.store.NewestFirstSeen() +} + +func (src serverNotifySource) observers() ([]notifyObserver, error) { + list, err := src.s.db.GetObservers() + if err != nil { + return nil, err + } + return notifyObserversFrom(list), nil +} + +// relayTimes parses LastRelayed for the given keys; keys without a relay +// time are absent. +func relayTimes(m map[string]RepeaterRelayInfo, pubkeys []string) map[string]time.Time { + out := make(map[string]time.Time, len(pubkeys)) + for _, pk := range pubkeys { + info, ok := lookupRelayInfo(m, pk) + if !ok || info.LastRelayed == "" { + continue + } + if t, ok := parseRelayTS(info.LastRelayed); ok { + out[pk] = t + } + } + return out +} + +func notifyObserversFrom(list []Observer) []notifyObserver { + out := make([]notifyObserver, 0, len(list)) + for _, o := range list { + n := notifyObserver{ID: o.ID} + if o.Name != nil { + n.Name = *o.Name + } + if o.LastSeen != nil { + n.LastSeen = *o.LastSeen + } + out = append(out, n) + } + return out +} + +// notifyNodeChunk bounds the IN list of one query, far below SQLite's +// variable limit. +const notifyNodeChunk = 500 + +const notifyNodeCols = `SELECT public_key, COALESCE(name, ''), COALESCE(role, ''), COALESCE(last_seen, ''), battery_mv, COALESCE(foreign_advert, 0) FROM nodes` + +// NotifyNodes reads the node rows notifications need: the given pubkeys +// (primary-key lookups, in chunks) and, with withForeign, every node with +// foreign_advert = 1 (partial index idx_nodes_foreign_advert). Keyed by +// lowercase pubkey. Read-only, on the server's mode=ro handle. +func (db *DB) NotifyNodes(pubkeys []string, withForeign bool) (map[string]notifyNode, error) { + out := make(map[string]notifyNode, len(pubkeys)) + scan := func(rows *sql.Rows) error { + defer rows.Close() + for rows.Next() { + var n notifyNode + var mv sql.NullInt64 + var foreign int + if err := rows.Scan(&n.Pubkey, &n.Name, &n.Role, &n.LastSeen, &mv, &foreign); err != nil { + return err + } + n.Pubkey = strings.ToLower(n.Pubkey) + if mv.Valid { + v := int(mv.Int64) + n.BatteryMv = &v + } + n.Foreign = foreign != 0 + out[n.Pubkey] = n + } + return rows.Err() + } + for i := 0; i < len(pubkeys); i += notifyNodeChunk { + chunk := pubkeys[i:min(i+notifyNodeChunk, len(pubkeys))] + args := make([]any, len(chunk)) + for j, pk := range chunk { + args[j] = pk + } + rows, err := db.conn.Query(notifyNodeCols+` WHERE public_key IN (`+db.sqlPlaceholders(len(chunk))+`)`, args...) + if err != nil { + return nil, err + } + if err := scan(rows); err != nil { + return nil, err + } + } + if withForeign { + rows, err := db.conn.Query(notifyNodeCols + ` WHERE foreign_advert = 1`) + if err != nil { + return nil, err + } + if err := scan(rows); err != nil { + return nil, err + } + } + return out, nil +} + +// LastHeardMap returns, per pubkey with packets in the store, the newest +// FirstSeen among them: the node page's "Last Heard" (GetNodeHealth) in +// bulk, and how long the read lock was held (logged once per tick). One +// read lock, O(packets of the given nodes), string compares only; parsing +// happens after the lock is released. +func (s *PacketStore) LastHeardMap(pubkeys []string) (map[string]time.Time, time.Duration) { + latest := make(map[string]string, len(pubkeys)) + s.mu.RLock() + locked := time.Now() + for _, pk := range pubkeys { + for _, tx := range s.byNode[pk] { + if tx != nil && tx.FirstSeen > latest[pk] { + latest[pk] = tx.FirstSeen + } + } + } + held := time.Since(locked) + s.mu.RUnlock() + out := make(map[string]time.Time, len(latest)) + for pk, ts := range latest { + if t, ok := parseRelayTS(ts); ok { + out[pk] = t + } + } + return out, held +} + +// NewestFirstSeen is the FirstSeen of the newest transmission in the +// store, zero when there is none or it does not parse. O(1): s.packets is +// kept sorted by FirstSeen with the newest at the tail. +func (s *PacketStore) NewestFirstSeen() time.Time { + s.mu.RLock() + var ts string + if n := len(s.packets); n > 0 && s.packets[n-1] != nil { + ts = s.packets[n-1].FirstSeen + } + s.mu.RUnlock() + t, _ := parseRelayTS(ts) + return t +} diff --git a/cmd/server/notify_source_test.go b/cmd/server/notify_source_test.go new file mode 100644 index 000000000..46fef0941 --- /dev/null +++ b/cmd/server/notify_source_test.go @@ -0,0 +1,151 @@ +package main + +import ( + "fmt" + "testing" + "time" +) + +func TestDBNotifyNodes(t *testing.T) { + db := setupTestDB(t) + for _, q := range []string{ + fmt.Sprintf(`INSERT INTO nodes (public_key, name, role, last_seen, battery_mv, foreign_advert) VALUES ('%s', 'Alpha', 'repeater', '2026-10-07T11:00:00Z', 3712, 0)`, evPkA), + fmt.Sprintf(`INSERT INTO nodes (public_key, name, role, last_seen, foreign_advert) VALUES ('%s', NULL, 'companion', NULL, 0)`, evPkB), + fmt.Sprintf(`INSERT INTO nodes (public_key, name, role, last_seen, foreign_advert) VALUES ('%s', 'Foxtrot', 'repeater', '2026-10-07T10:00:00Z', 1)`, evPkF), + } { + if _, err := db.conn.Exec(q); err != nil { + t.Fatal(err) + } + } + got, err := db.NotifyNodes([]string{evPkA, evPkB, evPkG}, false) + if err != nil { + t.Fatal(err) + } + a, b := got[evPkA], got[evPkB] + if len(got) != 2 || a.Pubkey != evPkA || a.Name != "Alpha" || a.Role != "repeater" || a.LastSeen != "2026-10-07T11:00:00Z" || + a.BatteryMv == nil || *a.BatteryMv != 3712 || a.Foreign { + t.Fatalf("NotifyNodes = %+v", got) + } + if b.Name != "" || b.LastSeen != "" || b.BatteryMv != nil { + t.Fatalf("NULL columns = %+v", b) + } + got, err = db.NotifyNodes([]string{evPkA}, true) + if err != nil || len(got) != 2 || !got[evPkF].Foreign || got[evPkF].Name != "Foxtrot" { + t.Fatalf("with foreign = %+v, %v", got, err) + } + if got, err := db.NotifyNodes(nil, false); err != nil || len(got) != 0 { + t.Fatalf("no keys = %+v, %v", got, err) + } +} + +func TestDBNotifyNodesChunksLongLists(t *testing.T) { + db := setupTestDB(t) + keys := make([]string, 0, 1200) + for i := 0; i < 1200; i++ { + keys = append(keys, fmt.Sprintf("%064x", i)) + } + for _, i := range []int{3, 600, 1199} { + if _, err := db.conn.Exec(`INSERT INTO nodes (public_key, role) VALUES ($1, 'companion')`, keys[i]); err != nil { + t.Fatal(err) + } + } + got, err := db.NotifyNodes(keys, false) + if err != nil || len(got) != 3 { + t.Fatalf("1200 keys over %d-key chunks = %d nodes, %v; want 3", notifyNodeChunk, len(got), err) + } +} + +func TestPacketStoreLastHeardMap(t *testing.T) { + s := &PacketStore{byNode: map[string][]*StoreTx{ + evPkA: {{FirstSeen: "2026-10-07T10:00:00Z"}, {FirstSeen: "2026-10-07T11:30:00.000Z"}, nil, {FirstSeen: "2026-10-07T09:00:00Z"}}, + evPkB: {{FirstSeen: "not a time"}}, + }} + got, lock := s.LastHeardMap([]string{evPkA, evPkB, evPkF}) + if len(got) != 1 || !got[evPkA].Equal(time.Date(2026, 10, 7, 11, 30, 0, 0, time.UTC)) { + t.Fatalf("LastHeardMap = %v", got) + } + if lock < 0 || lock > time.Second { + t.Fatalf("lock time = %v", lock) + } +} + +// 5,000 watched nodes with 20 packets each, the spec's upper figure. Quote +// the result in the PR: this runs under the store's read lock. +func BenchmarkLastHeardMap(b *testing.B) { + s := &PacketStore{byNode: map[string][]*StoreTx{}} + keys := make([]string, 0, 5000) + for i := 0; i < 5000; i++ { + pk := fmt.Sprintf("%064x", i) + keys = append(keys, pk) + for j := 0; j < 20; j++ { + s.byNode[pk] = append(s.byNode[pk], &StoreTx{FirstSeen: evNow.Add(-time.Duration(j) * time.Minute).Format(time.RFC3339)}) + } + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + s.LastHeardMap(keys) + } +} + +func TestRelayTimes(t *testing.T) { + m := map[string]RepeaterRelayInfo{ + evPkA: {LastRelayed: "2026-10-07T11:00:00Z"}, + evPkB: {}, + } + got := relayTimes(m, []string{evPkA, evPkB, evPkF}) + if len(got) != 1 || !got[evPkA].Equal(time.Date(2026, 10, 7, 11, 0, 0, 0, time.UTC)) { + t.Fatalf("relayTimes = %v", got) + } +} + +func TestNotifyObserversFrom(t *testing.T) { + name, seen := "Roof", "2026-10-07T11:00:00Z" + got := notifyObserversFrom([]Observer{{ID: "OBS1", Name: &name, LastSeen: &seen}, {ID: "OBS2"}}) + if len(got) != 2 || got[0] != (notifyObserver{ID: "OBS1", Name: "Roof", LastSeen: seen}) || got[1] != (notifyObserver{ID: "OBS2"}) { + t.Fatalf("notifyObserversFrom = %+v", got) + } +} + +func TestServerNotifySourceWithoutStore(t *testing.T) { + src := serverNotifySource{s: &Server{cfg: &Config{}}} + if heard, lock := src.lastHeard([]string{evPkA}); len(heard) != 0 || lock != 0 || len(src.lastRelayed([]string{evPkA})) != 0 { + t.Fatal("a server without a packet store reported times") + } + if src.lowBatteryMv() != 3300 || src.health().InfraSilentHours != 72 { + t.Fatal("defaults not taken from the config") + } +} + +// The startup load (hot window plus background fill) must be finished, not +// only /api/healthz readiness: readiness can be reached while the newest +// packets are still loading, which would mail "offline" then "back online". +func TestServerNotifySourceReadyWaitsForTheStartupLoad(t *testing.T) { + readiness.Store(1) + defer readiness.Store(0) + store := &PacketStore{clockSkew: &ClockSkewEngine{}} + src := serverNotifySource{s: &Server{cfg: &Config{}, store: store}} + if src.ready() { + t.Fatal("ready before the startup load finished") + } + store.signalStartupLoadDone() + if !src.ready() { + t.Fatal("not ready after readiness and the startup load") + } + readiness.Store(0) + if src.ready() { + t.Fatal("ready without readiness") + } + if (serverNotifySource{s: &Server{cfg: &Config{}}}).ready() { + t.Fatal("ready without a packet store") + } +} + +func TestPacketStoreNewestFirstSeen(t *testing.T) { + if got := (&PacketStore{}).NewestFirstSeen(); !got.IsZero() { + t.Fatalf("empty store = %v", got) + } + s := &PacketStore{packets: []*StoreTx{{FirstSeen: "2026-10-07T10:00:00Z"}, {FirstSeen: "2026-10-07T11:30:00.000Z"}}} + if got := s.NewestFirstSeen(); !got.Equal(time.Date(2026, 10, 7, 11, 30, 0, 0, time.UTC)) { + t.Fatalf("NewestFirstSeen = %v", got) + } +} diff --git a/cmd/server/obs_raw_hex_test.go b/cmd/server/obs_raw_hex_test.go new file mode 100644 index 000000000..557e50ffc --- /dev/null +++ b/cmd/server/obs_raw_hex_test.go @@ -0,0 +1,246 @@ +package main + +import ( + "encoding/json" + "net/http/httptest" + "testing" + "time" + + "github.com/gorilla/mux" +) + +// Issue #1999: the packet-detail API returned the transmission's canonical +// frame for every observation, so the browser's hex view could not show the +// selected observation's actual bytes and could contradict the path_json shown +// beside it. +// +// The store drops observations.raw_hex on purpose (#881, ~98MB on a 1.7M +// observation store) on the assumption that one content hash means one frame. +// The firmware hashes payload and type independently of the relay path, so +// observations of a single transmission legitimately differ. Measured on a +// production DB: of the 3000 most recent transmissions, 1977 had more than one +// observation and 1844 of those held genuinely different frames, up to 51 for +// one packet. +// +// The fix reads them back on the detail path only, one query per request. + +// seedDistinctFrames inserts one transmission and three observations whose +// stored frames differ in their path bytes, plus one with no stored frame at +// all so the canonical fallback stays covered. Returns the hash and the +// observation ids in insertion order. +func seedDistinctFrames(t *testing.T, db *DB, hash string) (string, []int) { + t.Helper() + now := time.Now().UTC().Format(time.RFC3339) + if _, err := db.conn.Exec(`INSERT INTO transmissions + (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) + VALUES ('CANON0000', $1, $2, 1, 5, '{"type":"CHAN"}')`, hash, now); err != nil { + t.Fatalf("insert transmission: %v", err) + } + var txID int + if err := db.conn.QueryRow("SELECT id FROM transmissions WHERE hash = $1", hash).Scan(&txID); err != nil { + t.Fatalf("lookup tx id: %v", err) + } + + frames := []struct { + pathJSON string + rawHex interface{} // nil means SQL NULL: no stored frame + }{ + {`["AA","BB"]`, "FRAME2HOPS"}, + {`["AA","BB","CC"]`, "FRAME3HOPS"}, + {`["AA"]`, "FRAME1HOP"}, + {`["DD"]`, nil}, + } + var ids []int + for i, f := range frames { + var id int64 + err := db.conn.QueryRow(`INSERT INTO observations + (transmission_id, observer_idx, snr, rssi, path_json, timestamp, raw_hex) + VALUES ($1, 1, 5.0, -90, $2, $3, $4) RETURNING id`, + txID, f.pathJSON, time.Now().Unix()-int64(i), f.rawHex).Scan(&id) + if err != nil { + t.Fatalf("insert observation %d: %v", i, err) + } + + if err != nil { + t.Fatalf("last insert id: %v", err) + } + ids = append(ids, int(id)) + } + return hash, ids +} + +func TestObservationRawHexForHash(t *testing.T) { + db := setupTestDB(t) + // Production sets this in detectSchema (db.go) when the column exists; the + // test schema has the column but the helper does not run detection. + db.hasObsRawHex = true + + hash, ids := seedDistinctFrames(t, db, "1999aaaabbbbcccc") + + got := db.ObservationRawHexForHash(hash) + if len(got) != 3 { + t.Fatalf("got %d frames, want 3 (the fourth observation stores none): %v", len(got), got) + } + want := map[int]string{ids[0]: "FRAME2HOPS", ids[1]: "FRAME3HOPS", ids[2]: "FRAME1HOP"} + for id, wantHex := range want { + if got[id] != wantHex { + t.Errorf("observation %d: got %q, want %q", id, got[id], wantHex) + } + } + // The observation with a NULL frame must be absent rather than empty, so + // the caller falls back to the transmission's canonical bytes. + if _, present := got[ids[3]]; present { + t.Errorf("observation %d has no stored frame and must not appear in the map", ids[3]) + } + + // Unknown hash: no rows, no error, no panic. + if got := db.ObservationRawHexForHash("0000000000000000"); len(got) != 0 { + t.Errorf("unknown hash returned %d frames, want 0", len(got)) + } +} + +// TestObservationRawHexForHashRespectsSchemaFlag pins the guard: on a schema +// without observations.raw_hex the query must not be attempted at all, because +// it would be a SQL error rather than an empty result. +func TestObservationRawHexForHashRespectsSchemaFlag(t *testing.T) { + db := setupTestDB(t) + db.hasObsRawHex = false + hash, _ := seedDistinctFrames(t, db, "1999ddddeeeeffff") + if got := db.ObservationRawHexForHash(hash); got != nil { + t.Errorf("hasObsRawHex=false must return nil, got %v", got) + } +} + +// TestPacketDetailExposesPerObservationFrames is the regression the issue asks +// for: the detail response must give each observation its own bytes, and fall +// back to the transmission's only where none are stored. +func TestPacketDetailExposesPerObservationFrames(t *testing.T) { + srv, router := setupTestServer(t) + srv.db.hasObsRawHex = true + + // Inserted after store.Load(), so this transmission is DB-only and the + // handler takes its DB-fallback path. Both paths converge on the same + // backfill, and the DB path is the one that can be set up deterministically. + hash, ids := seedDistinctFrames(t, srv.db, "1999112233445566") + + req := httptest.NewRequest("GET", "/api/packets/"+hash, nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != 200 { + t.Fatalf("got %d, want 200 (body: %s)", w.Code, w.Body.String()) + } + + var body map[string]interface{} + if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil { + t.Fatal(err) + } + obsList, _ := body["observations"].([]interface{}) + if len(obsList) != 4 { + t.Fatalf("got %d observations, want 4", len(obsList)) + } + + byID := map[int]string{} + for _, raw := range obsList { + m, ok := raw.(map[string]interface{}) + if !ok { + t.Fatalf("observation is not an object: %T", raw) + } + idF, ok := m["id"].(float64) // JSON numbers decode as float64 + if !ok { + t.Fatalf("observation has no numeric id: %v", m["id"]) + } + hx, _ := m["raw_hex"].(string) + byID[int(idF)] = hx + } + + for i, wantHex := range []string{"FRAME2HOPS", "FRAME3HOPS", "FRAME1HOP"} { + if got := byID[ids[i]]; got != wantHex { + t.Errorf("observation %d: raw_hex = %q, want its own frame %q", ids[i], got, wantHex) + } + } + // The one without stored bytes gets the transmission's canonical frame. On + // this path that is new: the DB observation query selects no raw_hex, so + // before the fix the field was missing from those observations entirely. + if got := byID[ids[3]]; got != "CANON0000" { + t.Errorf("observation %d: raw_hex = %q, want the canonical fallback %q", ids[3], got, "CANON0000") + } + + // The whole point: the frames are not all the same value. + distinct := map[string]bool{} + for _, hx := range byID { + distinct[hx] = true + } + if len(distinct) < 3 { + t.Errorf("only %d distinct frames across 4 observations (%v) — the canonical frame is still being repeated", len(distinct), byID) + } +} + +// TestPacketDetailExposesPerObservationFramesFromStore covers the path almost +// every real request takes: the transmission IS in the in-memory store. +// +// This is the case a DB-fallback-only test misses. enrichObsWithTx already puts +// the transmission's canonical frame into every observation map, so a backfill +// that skipped observations which "already have" raw_hex would skip all of them +// and leave the defect untouched on the main path. A stored frame has to win +// over that placeholder. +func TestPacketDetailExposesPerObservationFramesFromStore(t *testing.T) { + db := setupTestDB(t) + seedTestData(t, db) + db.hasObsRawHex = true + + // Seed BEFORE the store loads, so the store holds this transmission and the + // handler never reaches its DB fallback. + hash, ids := seedDistinctFrames(t, db, "1999aabbccdd0011") + + cfg := &Config{Port: 3000} + srv := NewServer(db, cfg, NewHub()) + store := NewPacketStore(db, nil) + if err := store.Load(); err != nil { + t.Fatalf("store.Load: %v", err) + } + if !store.WaitIndexesReady(5 * time.Second) { + t.Fatal("background indexes never became ready") + } + srv.store = store + router := mux.NewRouter() + srv.RegisterRoutes(router) + + if got := srv.store.GetPacketByHash(hash); got == nil { + t.Fatal("precondition failed: the store does not hold the seeded transmission") + } + + req := httptest.NewRequest("GET", "/api/packets/"+hash, nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != 200 { + t.Fatalf("got %d, want 200 (body: %s)", w.Code, w.Body.String()) + } + var body map[string]interface{} + if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil { + t.Fatal(err) + } + obsList, _ := body["observations"].([]interface{}) + if len(obsList) != 4 { + t.Fatalf("got %d observations, want 4", len(obsList)) + } + + byID := map[int]string{} + for _, raw := range obsList { + m, _ := raw.(map[string]interface{}) + idF, ok := m["id"].(float64) + if !ok { + t.Fatalf("observation has no numeric id: %v", m["id"]) + } + hx, _ := m["raw_hex"].(string) + byID[int(idF)] = hx + } + + for i, wantHex := range []string{"FRAME2HOPS", "FRAME3HOPS", "FRAME1HOP"} { + if got := byID[ids[i]]; got != wantHex { + t.Errorf("store path, observation %d: raw_hex = %q, want its own frame %q (a canonical value here means the backfill was skipped)", ids[i], got, wantHex) + } + } + if got := byID[ids[3]]; got != "CANON0000" { + t.Errorf("store path, observation %d: raw_hex = %q, want the canonical fallback %q", ids[3], got, "CANON0000") + } +} diff --git a/cmd/server/observer_blacklist_test.go b/cmd/server/observer_blacklist_test.go index ec6b2b473..0457e25a7 100644 --- a/cmd/server/observer_blacklist_test.go +++ b/cmd/server/observer_blacklist_test.go @@ -48,8 +48,8 @@ func TestConfigIsObserverBlacklistedNil(t *testing.T) { func TestObserverBlacklistFiltersHandleObservers(t *testing.T) { db := setupTestDB(t) - db.conn.Exec("INSERT OR IGNORE INTO observers (id, name, iata, last_seen) VALUES ('goodobs', 'GoodObs', 'SFO', datetime('now'))") - db.conn.Exec("INSERT OR IGNORE INTO observers (id, name, iata, last_seen) VALUES ('badobs', 'BadObs', 'LAX', datetime('now'))") + db.conn.Exec(testNativeSQL(`INSERT INTO observers (id, name, iata, last_seen) VALUES ('goodobs', 'GoodObs', 'SFO', strftime('%Y-%m-%dT%H:%M:%SZ','now')) ON CONFLICT DO NOTHING`, "INSERT INTO observers (id, name, iata, last_seen) VALUES ('goodobs', 'GoodObs', 'SFO', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')) ON CONFLICT DO NOTHING")) + db.conn.Exec(testNativeSQL(`INSERT INTO observers (id, name, iata, last_seen) VALUES ('badobs', 'BadObs', 'LAX', strftime('%Y-%m-%dT%H:%M:%SZ','now')) ON CONFLICT DO NOTHING`, "INSERT INTO observers (id, name, iata, last_seen) VALUES ('badobs', 'BadObs', 'LAX', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')) ON CONFLICT DO NOTHING")) cfg := &Config{ ObserverBlacklist: []string{"badobs"}, @@ -89,7 +89,7 @@ func TestObserverBlacklistFiltersHandleObservers(t *testing.T) { func TestObserverBlacklistFiltersObserverDetail(t *testing.T) { db := setupTestDB(t) - db.conn.Exec("INSERT OR IGNORE INTO observers (id, name, iata, last_seen) VALUES ('badobs', 'BadObs', 'LAX', datetime('now'))") + db.conn.Exec(testNativeSQL(`INSERT INTO observers (id, name, iata, last_seen) VALUES ('badobs', 'BadObs', 'LAX', strftime('%Y-%m-%dT%H:%M:%SZ','now')) ON CONFLICT DO NOTHING`, "INSERT INTO observers (id, name, iata, last_seen) VALUES ('badobs', 'BadObs', 'LAX', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')) ON CONFLICT DO NOTHING")) cfg := &Config{ ObserverBlacklist: []string{"badobs"}, @@ -108,7 +108,7 @@ func TestObserverBlacklistFiltersObserverDetail(t *testing.T) { func TestNoObserverBlacklistPassesAll(t *testing.T) { db := setupTestDB(t) - db.conn.Exec("INSERT OR IGNORE INTO observers (id, name, iata, last_seen) VALUES ('someobs', 'SomeObs', 'SFO', datetime('now'))") + db.conn.Exec(testNativeSQL(`INSERT INTO observers (id, name, iata, last_seen) VALUES ('someobs', 'SomeObs', 'SFO', strftime('%Y-%m-%dT%H:%M:%SZ','now')) ON CONFLICT DO NOTHING`, "INSERT INTO observers (id, name, iata, last_seen) VALUES ('someobs', 'SomeObs', 'SFO', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')) ON CONFLICT DO NOTHING")) cfg := &Config{} srv := NewServer(db, cfg, NewHub()) diff --git a/cmd/server/observer_count_1888_test.go b/cmd/server/observer_count_1888_test.go index 89d4bd225..65aa4d29c 100644 --- a/cmd/server/observer_count_1888_test.go +++ b/cmd/server/observer_count_1888_test.go @@ -27,18 +27,18 @@ func seedObserversForCount(t *testing.T, db *DB) { live := []string{"live1", "live2", "live3"} for _, id := range live { if _, err := db.conn.Exec(`INSERT INTO observers (id, name, last_seen, first_seen, packet_count, inactive) - VALUES (?, ?, ?, ?, 10, 0)`, id, "Observer "+id, now, now); err != nil { + VALUES ($1, $2, $3, $4, 10, 0)`, id, "Observer "+id, now, now); err != nil { t.Fatalf("insert %s: %v", id, err) } } // One explicitly soft-deleted, one with a NULL flag — GetObservers treats // NULL as live, so only the inactive=1 row must be excluded. if _, err := db.conn.Exec(`INSERT INTO observers (id, name, last_seen, first_seen, packet_count, inactive) - VALUES ('gone1', 'Gone One', ?, ?, 5, 1)`, now, now); err != nil { + VALUES ('gone1', 'Gone One', $1, $2, 5, 1)`, now, now); err != nil { t.Fatalf("insert gone1: %v", err) } if _, err := db.conn.Exec(`INSERT INTO observers (id, name, last_seen, first_seen, packet_count, inactive) - VALUES ('nullflag', 'Null Flag', ?, ?, 5, NULL)`, now, now); err != nil { + VALUES ('nullflag', 'Null Flag', $1, $2, 5, NULL)`, now, now); err != nil { t.Fatalf("insert nullflag: %v", err) } } diff --git a/cmd/server/observer_naive_clock_1478_test.go b/cmd/server/observer_naive_clock_1478_test.go index e0d369c24..5e749367f 100644 --- a/cmd/server/observer_naive_clock_1478_test.go +++ b/cmd/server/observer_naive_clock_1478_test.go @@ -30,7 +30,7 @@ func TestHandleObservers_Issue1478_SurfacesRecentNaiveSkew(t *testing.T) { _, err := srv.db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count, clock_skew_seconds, clock_skew_count_24h, clock_last_naive_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)`, "naive-obs-1", "California Pi", "SFO", now.Format(time.RFC3339), now.Add(-7*24*time.Hour).Format(time.RFC3339), 42, -28800, 17, recent) @@ -91,7 +91,7 @@ func TestHandleObservers_Issue1478_DecaysAfter24h(t *testing.T) { _, err := srv.db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count, clock_skew_seconds, clock_skew_count_24h, clock_last_naive_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)`, "naive-obs-old", "Fixed Pi", "LAX", now.Format(time.RFC3339), now.Add(-30*24*time.Hour).Format(time.RFC3339), 99, -28800, 5, stale) @@ -136,7 +136,7 @@ func TestHandleObserverDetail_Issue1478_IncludesClockNaiveFields(t *testing.T) { _, err := srv.db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count, clock_skew_seconds, clock_skew_count_24h, clock_last_naive_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)`, "naive-obs-detail", "Detail Pi", "SJC", now.Format(time.RFC3339), now.Add(-2*24*time.Hour).Format(time.RFC3339), 7, 25200, 3, recent) diff --git a/cmd/server/observer_planner_test.go b/cmd/server/observer_planner_test.go new file mode 100644 index 000000000..d49208966 --- /dev/null +++ b/cmd/server/observer_planner_test.go @@ -0,0 +1,211 @@ +package main + +import ( + "context" + "database/sql" + "fmt" + "path/filepath" + "reflect" + "sort" + "strings" + "testing" + "time" + + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/dbschema/legacy" +) + +func TestGetObserverPacketCountsWindowAndIdentity(t *testing.T) { + db := setupTestDB(t) + for _, query := range []string{ + `INSERT INTO observers(rowid,id,inactive) VALUES(1,'obs-a',0),(2,'obs-b',1)`, + `INSERT INTO transmissions(id,raw_hex,hash,first_seen) VALUES(1,'00','counts','2026-01-01')`, + `INSERT INTO observations(transmission_id,observer_idx,timestamp) VALUES + (1,1,100),(1,1,101),(1,1,102),(1,2,101),(1,NULL,101),(1,99,101)`, + } { + if _, err := db.conn.Exec(query); err != nil { + t.Fatal(err) + } + } + // Strict cutoff and inner-join identity semantics are shared by both engines. + want := map[string]int{"obs-a": 2, "obs-b": 1} + if got := db.GetObserverPacketCounts(100); !reflect.DeepEqual(got, want) { + t.Fatalf("observer counts=%v, want=%v", got, want) + } +} + +// Match the B corpus's cardinalities: 128 observers, 16 observations per +// transmission on average, eight days, and one eighth in the recent window. +// SQL-only data exercises the real endpoint pipeline without a PacketStore. +func observerPlannerFixture(t testing.TB, observations int) (*sql.DB, string, int64, map[string]int) { + t.Helper() + if testBackend(t) != dbconfig.SQLite { + t.Skip("SQLite identity migration and planner fixture") + } + path := filepath.Join(t.TempDir(), "observers.db") + dsn, err := legacy.WriterDSN(path) + if err != nil { + t.Fatal(err) + } + w, err := sql.Open("sqlite3", dsn) + if err != nil { + t.Fatal(err) + } + w.SetMaxOpenConns(1) + t.Cleanup(func() { w.Close() }) + if err = legacy.ApplyBase(w); err != nil { + t.Fatal(err) + } + if err = legacy.Apply(w, nil); err != nil { + t.Fatal(err) + } + if err = dbschema.EnsureSQLiteObserverTimeIndex(context.Background(), w); err != nil { + t.Fatal(err) + } + epoch := time.Now().Unix() + tx, err := w.Begin() + if err != nil { + t.Fatal(err) + } + defer tx.Rollback() + for _, query := range []string{ + `WITH RECURSIVE n(i) AS (VALUES(1) UNION ALL SELECT i+1 FROM n WHERE i<128) + INSERT INTO observers(rowid,id,name,iata,last_seen,first_seen,packet_count) + SELECT i,printf('%064x',i),printf('Synthetic observer %d',i),'AAA','2026-01-01','2026-01-01',0 FROM n`, + `INSERT INTO nodes(public_key,name,role,lat,lon) SELECT id,name,'repeater',20,30 FROM observers`, + fmt.Sprintf(`WITH RECURSIVE n(i) AS (VALUES(1) UNION ALL SELECT i+1 FROM n WHERE i<%d) + INSERT INTO transmissions(id,raw_hex,hash,first_seen) SELECT i,'00',printf('fixture-%%d',i),'2026-01-01' FROM n`, observations/16), + fmt.Sprintf(`WITH RECURSIVE n(i) AS (VALUES(0) UNION ALL SELECT i+1 FROM n WHERE i+1<%d) + INSERT INTO observations(transmission_id,observer_idx,timestamp,path_json) + SELECT i/16+1,i%%128+1,CASE WHEN (i/160)%%8=0 THEN %d-2700+(i/16)%%1800 + ELSE %d-((i/160)%%8)*86400-43200+(i/16)%%3600 END,'[]' FROM n`, observations, epoch, epoch), + } { + if _, err = tx.Exec(query); err != nil { + t.Fatal(err) + } + } + if err = tx.Commit(); err != nil { + t.Fatal(err) + } + if _, err = w.Exec(`ANALYZE`); err != nil { + t.Fatal(err) + } + want := make(map[string]int) + for i := 0; i < observations; i++ { + if (i/160)%8 == 0 { + want[fmt.Sprintf("%064x", i%128+1)]++ + } + } + return w, path, epoch - 3600, want +} + +func observerPlannerReader(t testing.TB, path string, since int64, want map[string]int) *DB { + t.Helper() + db, err := OpenDB(path) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { db.Close() }) + if got := db.GetObserverPacketCounts(since); !reflect.DeepEqual(got, want) { + t.Fatalf("recent observation counts changed: got %d observer groups, want %d", len(got), len(want)) + } + return db +} + +func observerAggregatePlan(t testing.TB, db *DB, since int64) []string { + t.Helper() + rows, err := db.conn.Query("EXPLAIN QUERY PLAN "+db.observerPacketCountsSQL(), since) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + var plan []string + for rows.Next() { + var id, parent, unused int + var detail string + if err := rows.Scan(&id, &parent, &unused, &detail); err != nil { + t.Fatal(err) + } + plan = append(plan, detail) + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + return plan +} + +func TestObserverAggregatePlannerAfterSQLiteUpgrade(t *testing.T) { + for _, repair := range []bool{false, true} { + name := "legacy adoption" + if repair { + name = "already rebuilt without statistics" + } + t.Run(name, func(t *testing.T) { + w, path, since, want := observerPlannerFixture(t, 128000) + if repair { + if err := dbschema.ApplySQLite(w, nil); err != nil { + t.Fatal(err) + } + if _, err := w.Exec(`DELETE FROM sqlite_stat1 WHERE tbl='observers'; ANALYZE sqlite_schema`); err != nil { + t.Fatal(err) + } + } + if err := dbschema.ApplySQLite(w, nil); err != nil { + t.Fatal(err) + } + // A new reader is important: the old writer can retain pre-rebuild + // planner estimates and conceal the statistics lost by DROP TABLE. + db := observerPlannerReader(t, path, since, want) + plan := observerAggregatePlan(t, db, since) + if strings.Contains(strings.Join(plan, "\n"), "TEMP B-TREE FOR GROUP BY") { + t.Fatalf("observer statistics lost after upgrade; aggregate sorts the observation window: %q", plan) + } + if !strings.Contains(strings.Join(plan, "\n"), "observer_idx=? AND timestamp>?") { + t.Fatalf("aggregate no longer uses observer/time range lookups: %q", plan) + } + }) + } +} + +// Run explicitly with -benchtime=50x. Reports uncached response-build latency, +// not full concurrent HTTP latency or a replacement for the paired workload. +func BenchmarkObserverListSQLiteUpgrade(b *testing.B) { + w, path, since, want := observerPlannerFixture(b, 2048000) + var reference ObserverListResponse + for _, stage := range []string{"legacy", "upgraded"} { + if stage == "upgraded" { + if err := dbschema.ApplySQLite(w, nil); err != nil { + b.Fatal(err) + } + } + b.Run(stage, func(b *testing.B) { + db := observerPlannerReader(b, path, since, want) + srv := &Server{db: db, cfg: &Config{}} + b.Logf("observations=2048000 transmissions=128000 observers=128 recent=%d; plan=%q", 256000, observerAggregatePlan(b, db, since)) + response, err := srv.buildObserversDefaultResponse() + if err != nil || len(response.Observers) != 128 { + b.Fatalf("response rows=%d err=%v", len(response.Observers), err) + } + response.ServerTime = "" + if stage == "legacy" { + reference = response + } else if !reflect.DeepEqual(reference, response) { + b.Fatal("observer response semantics changed after identity migration") + } + durations := make([]time.Duration, 0, b.N) + b.ResetTimer() + for i := 0; i < b.N; i++ { + start := time.Now() + if _, err := srv.buildObserversDefaultResponse(); err != nil { + b.Fatal(err) + } + durations = append(durations, time.Since(start)) + } + b.StopTimer() + sort.Slice(durations, func(i, j int) bool { return durations[i] < durations[j] }) + b.ReportMetric(float64(durations[(len(durations)-1)/2])/float64(time.Millisecond), "p50-ms") + b.ReportMetric(float64(durations[(len(durations)*95+99)/100-1])/float64(time.Millisecond), "p95-ms") + }) + } +} diff --git a/cmd/server/observers_canrelay_tristate_1290_test.go b/cmd/server/observers_canrelay_tristate_1290_test.go index 146f7ae9e..5073edcda 100644 --- a/cmd/server/observers_canrelay_tristate_1290_test.go +++ b/cmd/server/observers_canrelay_tristate_1290_test.go @@ -39,19 +39,19 @@ func TestObservers_CanRelayTriState_Issue1290(t *testing.T) { // Legacy: never received repeat field. can_relay=DEFAULT 1, seen=0. if _, err := srv.store.db.conn.Exec( `INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('legacy-obs', 'Legacy', 'SJC', ?, '2026-01-01T00:00:00Z', 1)`, now); err != nil { + VALUES ('legacy-obs', 'Legacy', 'SJC', $1, '2026-01-01T00:00:00Z', 1)`, now); err != nil { t.Fatalf("seed legacy: %v", err) } // Repeater: ingestor wrote can_relay=1, seen=1. if _, err := srv.store.db.conn.Exec( `INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count, can_relay, can_relay_seen) - VALUES ('rep-obs', 'Repeater', 'SFO', ?, '2026-01-01T00:00:00Z', 1, 1, 1)`, now); err != nil { + VALUES ('rep-obs', 'Repeater', 'SFO', $1, '2026-01-01T00:00:00Z', 1, 1, 1)`, now); err != nil { t.Fatalf("seed repeater: %v", err) } // Listener: ingestor wrote can_relay=0, seen=1. if _, err := srv.store.db.conn.Exec( `INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count, can_relay, can_relay_seen) - VALUES ('lst-obs', 'Listener', 'OAK', ?, '2026-01-01T00:00:00Z', 1, 0, 1)`, now); err != nil { + VALUES ('lst-obs', 'Listener', 'OAK', $1, '2026-01-01T00:00:00Z', 1, 0, 1)`, now); err != nil { t.Fatalf("seed listener: %v", err) } diff --git a/cmd/server/openapi.go b/cmd/server/openapi.go index 3644247e6..92af6072f 100644 --- a/cmd/server/openapi.go +++ b/cmd/server/openapi.go @@ -12,10 +12,15 @@ import ( // routeMeta holds metadata for a single API route. type routeMeta struct { - Summary string `json:"summary"` - Description string `json:"description,omitempty"` - Tag string `json:"tag"` - Auth bool `json:"auth,omitempty"` + Summary string `json:"summary"` + Description string `json:"description,omitempty"` + Tag string `json:"tag"` + Auth bool `json:"auth,omitempty"` + // Session marks routes that also accept the user-management session + // cookie (optional feature). With Auth too, either credential works. + // Set on the 3 documented requireAdmin routes (Auth: true) and on the + // user-management routes; other requireAdmin routes are undocumented. + Session bool `json:"session,omitempty"` QueryParams []paramMeta `json:"queryParams,omitempty"` // Response, when non-nil, is the OpenAPI schema object for the 200 // application/json response body. Routes without it fall back to the @@ -44,14 +49,64 @@ func routeDescriptions() map[string]routeMeta { "GET /api/config/geo-filter": {Summary: "Get geo-filter configuration", Tag: "config"}, // Admin / system - "GET /api/health": {Summary: "Health check", Description: "Returns server health, uptime, and memory stats.", Tag: "admin"}, - "GET /api/stats": {Summary: "Network statistics", Description: "Returns aggregate stats (node counts, packet counts, observer counts). Cached for 10s.", Tag: "admin"}, - "GET /api/perf": {Summary: "Performance statistics", Description: "Returns per-endpoint request timing and slow query log.", Tag: "admin"}, - "GET /api/mqtt/status": {Summary: "MQTT source status", Description: "Returns per-MQTT-source connection state and counters (lastConnectUnix, lastPacketUnix, packetsTotal, etc.). Broker URL passwords are masked. Sourced from the ingestor stats file; empty list when unavailable. (#1043)", Tag: "admin"}, - "POST /api/perf/reset": {Summary: "Reset performance stats", Tag: "admin", Auth: true}, + "GET /api/health": {Summary: "Health check", Description: "Returns server health, uptime, and memory stats.", Tag: "admin"}, + "GET /api/stats": {Summary: "Network statistics", Description: "Returns aggregate stats (node counts, packet counts, observer counts). Cached for 10s.", Tag: "admin"}, + "GET /api/perf": {Summary: "Performance statistics", Description: "Returns per-endpoint request timing and slow query log.", Tag: "admin"}, + "GET /api/perf/database": {Summary: "Selected database diagnostics", Description: "Native metrics for the selected SQLite or PostgreSQL backend. engine identifies the backend; unavailable metrics stay null. Database and row samples refresh every 30 seconds, while connection pool use remains live. SQLite reports WAL bytes, pages, free space, and planner statistics availability without checkpointing or changing the database.", Tag: "admin"}, + "GET /api/perf/postgres": {Summary: "PostgreSQL database statistics", Description: "Returns databaseBytes and PostgreSQL block-cache hit ratio (cacheHitRate, 0..1 or null without available reads), sampled with table counts once per 30 seconds and shared with /api/perf. sampledAt and sampleIntervalSeconds disclose freshness; stale marks a failed refresh. Before a first successful sample, unavailable size/counts are null and error is set. After a refresh failure, the last storage/count sample remains with stale=true and the unavailable block ratio is null. Pool usage and waits remain live. Block-cache counters are cumulative since the PostgreSQL statistics reset and exclude the operating-system cache.", Tag: "admin"}, + "GET /api/mqtt/status": {Summary: "MQTT source status", Description: "Returns per-MQTT-source connection state and counters (lastConnectUnix, lastPacketUnix, packetsTotal, etc.). Broker URL passwords are masked. Sourced from the ingestor stats file; empty list when unavailable. (#1043)", Tag: "admin"}, + "POST /api/perf/reset": {Summary: "Reset performance stats", Tag: "admin", Auth: true, Session: true}, // "POST /api/admin/prune" removed in #1283 (ingestor owns prune). - "GET /api/debug/affinity": {Summary: "Debug neighbor affinity scores", Tag: "admin", Auth: true}, - "GET /api/backup": {Summary: "Download SQLite backup", Description: "Streams a consistent SQLite snapshot of the analyzer DB (VACUUM INTO). Response is application/octet-stream with attachment filename corescope-backup-.db.", Tag: "admin", Auth: true}, + "GET /api/debug/affinity": {Summary: "Debug neighbor affinity scores", Tag: "admin", Auth: true, Session: true}, + "GET /api/backup": {Summary: "Download PostgreSQL backup", Description: "Streams a consistent PostgreSQL custom-format pg_dump archive of telemetry. Restore with pg_restore into a separate database. Response is application/octet-stream with attachment filename corescope-backup-.dump.", Tag: "admin", Auth: true, Session: true}, + + // User management (optional; routes exist only when userManagement.enabled) + "POST /api/auth/register": {Summary: "Register an account", Description: "Creates a pending account and mails an activation link. The response is identical whether or not the address is already registered.", Tag: "users"}, + "POST /api/auth/activate": {Summary: "Activate an account", Description: "Request body {token, password}: the mailed activation token and the account password chosen at (the newest) registration. A wrong password answers 401 and leaves the token usable; attempts are rate-limited per account. On success consumes the token, activates the account and starts a session.", Tag: "users"}, + "POST /api/auth/login": {Summary: "Log in", Description: "Email + password. Sets the cs_session cookie. Rate-limited per IP and per address.", Tag: "users"}, + "POST /api/auth/logout": {Summary: "Log out", Tag: "users"}, + "GET /api/auth/me": {Summary: "Current user", Description: "Returns the logged-in user and the CSRF token, or 401.", Tag: "users", Session: true}, + "POST /api/auth/forgot": {Summary: "Request a password reset", Tag: "users"}, + "POST /api/auth/reset": {Summary: "Reset the password", Description: "Consumes the mailed reset token and ends all sessions of the user.", Tag: "users"}, + "PATCH /api/account": {Summary: "Update profile", Tag: "users", Session: true}, + "DELETE /api/account": {Summary: "Delete own account", Tag: "users", Session: true}, + "POST /api/account/password": {Summary: "Change password", Tag: "users", Session: true}, + "POST /api/account/email": {Summary: "Request an address change", Tag: "users", Session: true}, + "POST /api/account/confirm-email": {Summary: "Confirm an address change", Tag: "users"}, + "GET /api/account/sessions": {Summary: "List own sessions", Tag: "users", Session: true}, + "DELETE /api/account/sessions/{id}": {Summary: "Revoke one own session", Tag: "users", Session: true}, + "GET /api/account/settings": {Summary: "Get own synced settings", Description: "Returns {revision, generation, doc, allowlist}. doc is {v: 1, keys: {: }} or null at revision 0. generation is a random id of the stored document (empty at revision 0); revisions restart at 1 after a DELETE, the generation does not repeat. allowlist lists the keys the server accepts: {key, kind: set|scalar, id?}, where id names the field that identifies an item of a set (absent: the item itself).", Tag: "users", Session: true}, + "PUT /api/account/settings": {Summary: "Replace own synced settings", Description: "Body {baseRevision, baseGeneration, doc}. 200 {revision, generation}. 409 {revision, generation, doc} when baseRevision is not the stored revision or, with a stored document, baseGeneration is not its generation. baseRevision 0 without a stored document starts a new generation. 400 when doc is not {v: 1, keys} or holds a key that is never synced or not allowlisted. 413 above 256 KiB. 429 above 60 PUT requests per hour per user.", Tag: "users", Session: true}, + "DELETE /api/account/settings": {Summary: "Delete own synced settings", Description: "Removes the stored document; the next PUT (baseRevision 0) starts a new one in a new generation.", Tag: "users", Session: true}, + "GET /api/account/export": {Summary: "Download own account data", Description: "One JSON file (Content-Disposition attachment corescope-account-.json): {formatVersion: 1, exportedAt, instance, profile, sessions, settings, proposals, notifications: {prefs, watches}, audit, mail}. Every row, no caps. Leaves out the password hash and every session, link and unsubscribe token. Audit rows name other accounts by id only. Includes activatedAt, activatedBy (id) and each mail row's to-address. Records a user.export audit row after the file is built.", Tag: "users", Session: true}, + "GET /api/admin/users": {Summary: "List users (admin)", Tag: "users", Session: true, QueryParams: []paramMeta{{Name: "status", Description: "pending | active | disabled", Type: "string"}, {Name: "role", Description: "user | admin", Type: "string"}, {Name: "q", Description: "Substring of email or display name", Type: "string"}, {Name: "bouncing", Description: "1 = only addresses whose mail bounces", Type: "string"}}}, + "GET /api/admin/users/{id}": {Summary: "User detail with sessions, mail log and audit (admin)", Tag: "users", Session: true}, + "DELETE /api/admin/users/{id}": {Summary: "Delete a user (admin)", Tag: "users", Session: true}, + "POST /api/admin/users/{id}/disable": {Summary: "Disable a user (admin)", Tag: "users", Session: true}, + "POST /api/admin/users/{id}/enable": {Summary: "Enable a user (admin)", Tag: "users", Session: true}, + "POST /api/admin/users/{id}/role": {Summary: "Change a user's role (admin)", Tag: "users", Session: true}, + "POST /api/admin/users/{id}/resend-activation": {Summary: "Resend the activation mail (admin)", Tag: "users", Session: true}, + "POST /api/admin/users/{id}/activate": {Summary: "Activate a pending user manually (admin)", Description: "For when mail keeps failing. The address stays unverified; recorded as activatedBy + audit row.", Tag: "users", Session: true}, + "POST /api/admin/users/{id}/mail/{mailId}/refresh": {Summary: "Pull delivery events for one mail from the provider (admin)", Tag: "users", Session: true}, + "GET /api/admin/audit": {Summary: "Global audit log (admin)", Description: "Newest first, keyset-paginated: {entries, next}. Pass next as before for the following page; next is null on the last page. Entry: {id, at, action, actor, target, detail}; actor and target are {id, displayName, email}, {id, deleted: true} for a removed account, or null for the system. Login rows (user.login, user.login.failed) are kept 90 days. 400 on an invalid parameter.", Tag: "users", Session: true, QueryParams: []paramMeta{{Name: "action", Description: "One action, or a group ending in .* (user.login.* matches user.login and user.login.failed)", Type: "string"}, {Name: "user", Description: "User id, as actor or target", Type: "integer"}, {Name: "from", Description: "RFC 3339 time, inclusive", Type: "string"}, {Name: "to", Description: "RFC 3339 time, inclusive", Type: "string"}, {Name: "before", Description: "Only entries with a smaller id (next of the previous page)", Type: "integer"}, {Name: "limit", Description: "Page size, default 100, capped at 500", Type: "integer"}}}, + "GET /api/admin/stats": {Summary: "User figures for the admin overview (admin)", Description: "Accounts by status, active admins, pending accounts older than 24 hours (stuckPending), bouncing addresses, registrations in 7 and 30 days and per UTC day for 30 days (newPerDay, oldest first), users active in 7 and 30 days, logins and failed logins in 24 hours, mail sent in 7 days by latest delivery status (mail7d), and accounts with 5 or more failed logins in 24 hours (guessing: [{userId, displayName, failed}]).", Tag: "users", Session: true}, + "GET /api/admin/users-backup": {Summary: "Download accounts backup (admin)", Description: "A consistent PostgreSQL custom-format pg_dump archive of accounts as application/octet-stream, attachment corescope-users-.dump (UTC). It holds every password hash and address: store it encrypted. Records a user.backup audit row. The analyzer database has its own route, GET /api/backup.", Tag: "users", Session: true}, + "POST /api/proposals": {Summary: "Propose a hashtag channel", Description: "Body {kind: \"hashtag_channel\", subject}. subject is trimmed and gets a leading # when missing; at most 31 UTF-8 bytes including the #, no control or invisible characters (ZWJ allowed), not Public. 201 with the proposal {id, kind, subject, status, note, createdAt, decidedAt}. 400 invalid kind or name, 409 already pending, already approved or rejected, or already decrypted on this instance (a name in config.json hashChannels or channelKeys, case-sensitive), 429 when the caller created userManagement.channelProposals.perUserPerDay proposals in 24 hours or maxPending proposals wait for review. Registered only when userManagement.channelProposals.enabled.", Tag: "users", Session: true}, + "GET /api/account/proposals": {Summary: "List own proposals", Description: "The caller's proposals, newest first: [{id, kind, subject, status, note, createdAt, decidedAt}]. note is the reviewer's reason.", Tag: "users", Session: true}, + "GET /api/admin/proposals": {Summary: "List proposals (admin)", Description: "Newest first, at most 500: the proposal fields plus proposer and reviewer ({id, displayName, email}, or null once the account is deleted). 400 on an invalid filter.", Tag: "users", Session: true, QueryParams: []paramMeta{{Name: "status", Description: "pending | approved | rejected | revoked", Type: "string"}, {Name: "kind", Description: "hashtag_channel", Type: "string"}}}, + "POST /api/admin/proposals/{id}/approve": {Summary: "Approve a proposal (admin)", Description: "Body {note} (at most 500 characters). Pending only (409 otherwise). The ingestor decrypts the channel from its next refresh (at most a minute) and /api/channels lists it in approvedChannels. 409 when userManagement.channelProposals.maxApproved channels are approved.", Tag: "users", Session: true}, + "POST /api/admin/proposals/{id}/reject": {Summary: "Reject a proposal (admin)", Description: "Body {note}. Pending only (409 otherwise). The name cannot be proposed again until the row is pruned, 90 days after the decision.", Tag: "users", Session: true}, + "POST /api/admin/proposals/{id}/revoke": {Summary: "Revoke an approved proposal (admin)", Description: "Body {note}. Approved only (409 otherwise). The ingestor stops decrypting the channel from its next refresh unless the channel is configured; stored messages stay. The name can be proposed again.", Tag: "users", Session: true}, + "POST /api/mail/brevo/webhook": {Summary: "Brevo delivery-event webhook", Description: "Authenticated with Authorization: Bearer . Registered only when the secret is set.", Tag: "users"}, + + // Node notifications (user management, part E) + "GET /api/account/notifications": {Summary: "Get own notification settings", Description: "Returns {enabled, events, availableEvents, watches: [{pubkey, name, known, createdAt}], limits: {maxWatches, perUserPerDay, mailsLast24h}}. Creates the default settings (on, node.offline and node.battery) on first use. known is false once the node left the analyzer database. Registered only when userManagement.notifications.enabled.", Tag: "users", Session: true}, + "PUT /api/account/notifications": {Summary: "Change own notification settings", Description: "Body {enabled, events}. events from node.offline, node.battery and, for admins, foreign.new and observer.offline. Answers the GET body. 400 unknown event type, 403 an admin event chosen by a non-admin. Events left out lose their stored state, so choosing them again starts without a mail.", Tag: "users", Session: true}, + "PUT /api/account/notifications/watches/{pubkey}": {Summary: "Watch a node", Description: "Answers the GET body. 400 when pubkey is not 64 hex characters, 404 when the node is not in the analyzer database, 409 when userManagement.notifications.maxWatchesPerUser nodes are watched. Watching a watched node is 200.", Tag: "users", Session: true}, + "DELETE /api/account/notifications/watches/{pubkey}": {Summary: "Stop watching a node", Description: "Answers the GET body; removing a node that is not watched is 200. 400 when pubkey is not 64 hex characters.", Tag: "users", Session: true}, + "POST /api/account/notifications/watch-my-nodes": {Summary: "Watch the synced My nodes", Description: "Copies the pubkeys of the caller's synced meshcore-my-nodes setting into the watch list, up to the limit. Answers {added, already, skipped, account}; skipped counts malformed keys, nodes not in the analyzer database and nodes over the limit; account is the GET body.", Tag: "users", Session: true}, + "GET /api/notifications/unsubscribe": {Summary: "Unsubscribe link (redirect)", Description: "303 to /#/account/unsubscribe?token=, the confirm view. Changes nothing, so link scanners cannot unsubscribe anyone.", Tag: "users", QueryParams: []paramMeta{{Name: "token", Description: "The unsubscribe token from the mail", Type: "string"}}}, + "POST /api/notifications/unsubscribe": {Summary: "Turn notification mails off", Description: "No session and no Origin check: the confirm button and the mail provider's List-Unsubscribe-Post one-click both land here. 200 {ok, message}, also when already off; 410 for an unknown token. The token can do nothing else.", Tag: "users", QueryParams: []paramMeta{{Name: "token", Description: "The unsubscribe token from the mail", Type: "string"}}}, // Packets "GET /api/packets": {Summary: "List packets", Description: "Returns decoded packets with filtering, sorting, and pagination.", Tag: "packets", @@ -87,8 +142,23 @@ func routeDescriptions() map[string]routeMeta { "GET /api/nodes/{pubkey}/health": {Summary: "Get node health", Tag: "nodes"}, "GET /api/nodes/{pubkey}/paths": {Summary: "Get node routing paths", Tag: "nodes"}, "GET /api/nodes/{pubkey}/analytics": {Summary: "Get node analytics", Description: "Per-node packet counts, timing, and RF stats.", Tag: "nodes"}, + "GET /api/nodes/{pubkey}/hop_analytics": {Summary: "Get node hop counts", Description: "One entry per flood packet the node forwarded in the window, with the hop count its flood.max check saw (the node's zero-based index in the path) and tags (flood, scoped or unscoped, advert). DIRECT packets are excluded. Every observation in the window is read. A colliding path prefix is attributed only when the previous hop's neighbor_edges neighbors leave this node as the one candidate; the server's resolved-path pick is not used. Packets carrying this node's prefix that cannot be attributed are counted in `ambiguous`.", Tag: "nodes", + QueryParams: []paramMeta{ + {Name: "days", Description: "Lookback window in days (default 7, clamped 1-365)", Type: "integer"}, + }}, "GET /api/nodes/{pubkey}/neighbors": {Summary: "Get node neighbors", Description: "Returns the queried node's first-hop neighbors with affinity scores and observation metadata (count, SNR, distance, observers). Ambiguous edges carry candidate pubkeys.", Tag: "nodes", Response: schemaRef("NodeNeighborsResponse")}, + "GET /api/rf-noise": {Summary: "RF noise-floor hex grid", Description: "GeoJSON hex cells of the LoRa noise floor measured by mobile clients along their tracks, from client_rf_samples. Lower (more negative) dBm is quieter, the opposite direction to the SNR-coloured coverage layer. Stationary samples are excluded: a parked companion logs hundreds of samples at one point and would otherwise define the cell. Gated on clientRfSamples.", Tag: "coverage", + QueryParams: []paramMeta{ + {Name: "bbox", Description: "Bounding box as minLat,minLon,maxLat,maxLon", Type: "string", Required: true}, + {Name: "z", Description: "Leaflet zoom level; sets the hex resolution", Type: "integer"}, + {Name: "days", Description: "Look-back window in days (1-30, default 7)", Type: "integer"}, + }}, + "GET /api/scope-audit": {Summary: "Network-wide scope audit", Description: "For every repeater that has answered a declared-regions request: the regions it declares, which of those it has NOT been observed forwarding in the window, which scopes it forwards without declaring, and whether it forwards unscoped floods while omitting the '*' wildcard. '*' is never listed as a region — it governs unscoped floods, not a scope. Repeaters never successfully asked are absent rather than shown as declaring nothing. Rows with missing regions sort first; a short window is weak evidence, since a quiet region simply has no traffic.", Tag: "analytics", + QueryParams: []paramMeta{ + {Name: "window", Description: "Time window: 1h, 24h, or 7d (default 24h)", Type: "string"}, + }}, + // Analytics "GET /api/analytics/rf": {Summary: "RF analytics", Description: "SNR/RSSI distributions and statistics.", Tag: "analytics"}, "GET /api/analytics/topology": {Summary: "Network topology", Description: "Hop-count distribution and route analysis.", Tag: "analytics"}, @@ -100,9 +170,17 @@ func routeDescriptions() map[string]routeMeta { "GET /api/analytics/subpaths-bulk": {Summary: "Bulk subpath analysis", Tag: "analytics"}, "GET /api/analytics/subpath-detail": {Summary: "Subpath detail", Tag: "analytics"}, "GET /api/analytics/neighbor-graph": {Summary: "Neighbor graph", Description: "Full neighbor affinity graph for visualization.", Tag: "analytics"}, + "GET /api/analytics/retransmissions": {Summary: "Retransmission pressure over time", Description: "Collision-pressure proxy (#1699): per time bucket, the average number of distinct repeaters in the union of all observed paths of each flood event (route types 0/1, TRACE excluded). A transmission's observations are split into flood events at gaps of more than 5 minutes; each event is bucketed by its first observation, and events before the store retention floor are left out. Hop prefixes are not resolved: a prefix counts once per event, so colliding 1-byte prefixes make this a lower bound. Only repeaters some observer heard are counted, so the value also follows observer coverage; each bucket carries its observer count.", Tag: "analytics", + QueryParams: []paramMeta{ + {Name: "region", Description: "Comma-separated IATA codes; only observations from observers in the region are counted. A region with no known observers is not filtered", Type: "string"}, + {Name: "window", Description: "Relative window: 1h, 24h, 3d, 7d or 30d", Type: "string"}, + {Name: "from", Description: "Absolute window start (RFC3339)", Type: "string"}, + {Name: "to", Description: "Absolute window end (RFC3339)", Type: "string"}, + {Name: "bucket", Description: "Bucket size: 5m, 15m, 1h, 6h or 1d (default 1h)", Type: "string"}, + }}, // Channels - "GET /api/channels": {Summary: "List channels", Description: "Returns known mesh channels with message counts.", Tag: "channels"}, + "GET /api/channels": {Summary: "List channels", Description: "Returns known mesh channels with message counts. With userManagement.channelProposals.enabled the response also carries approvedChannels: the approved hashtag channel names, oldest approval first, at most maxApproved, listed also before they have traffic.", Tag: "channels"}, "GET /api/channels/{hash}/messages": {Summary: "Get channel messages", Description: "Returns messages for a specific channel.", Tag: "channels"}, // Observers @@ -174,6 +252,7 @@ func componentSchemas() map[string]interface{} { "relay_count_24h": map[string]interface{}{"type": "integer", "description": "Repeater/room only: relay-hop appearances in the last 24 hours."}, "unscoped_relay_count_24h": map[string]interface{}{"type": "integer", "description": "Repeater/room only: subset of relay_count_24h that were unscoped floods (route_type FLOOD). A well-configured repeater sets flood.max.unscoped 0, so a non-trivial count flags a base-config problem."}, "last_relayed": str("Repeater/room only: RFC3339 time this node last appeared as a relay hop."), + "declared_regions": map[string]interface{}{"type": "array", "items": map[string]interface{}{"type": "string"}, "description": "Repeater/room only (#1862): named regions from this node's newest declared-regions answer, spelled as GET /api/scope-audit declaredRegions spells them (leading '#' stripped, '*' wildcard excluded). Empty array: it answered and named no region. Absent: it never answered, this database has no declared-regions source, or the declared-regions lookup failed. Absence is not evidence the node lacks a region."}, "relay_window_hours": map[string]interface{}{"type": "integer", "description": "Repeater/room only, /api/nodes/{pubkey} detail endpoint only: width (hours) of the relay-activity window the relay_count_* values cover."}, "traffic_share_score": score01("#672 Traffic axis: share of non-advert traffic relayed through this repeater. Repeater/room only."), "bridge_score": score01("#672 Bridge axis: normalized betweenness centrality (chokepoint importance). Repeater/room only."), @@ -184,14 +263,16 @@ func componentSchemas() map[string]interface{} { "type": "string", "enum": []string{"A", "B", "C", "D", "F"}, "description": "Letter grade derived from usefulness_score. Repeater/room only.", }, + "declared_regions_truncated": map[string]interface{}{"type": "boolean", "description": "Repeater/room only (#1862): present, and true, only when the answer behind declared_regions was flagged as truncated, so that list is partial. GET /api/scope-audit shows the same flag as truncated. Absent otherwise, including for a source that does not record truncation, so absence does not mean the list is complete."}, }, }, "NodeListResponse": map[string]interface{}{ "type": "object", "properties": map[string]interface{}{ - "nodes": map[string]interface{}{"type": "array", "items": schemaRef("Node")}, - "total": map[string]interface{}{"type": "integer", "description": "Total nodes matching the query after filtering."}, - "counts": map[string]interface{}{"type": "object", "additionalProperties": map[string]interface{}{"type": "integer"}, "description": "Per-role node counts."}, + "nodes": map[string]interface{}{"type": "array", "items": schemaRef("Node")}, + "total": map[string]interface{}{"type": "integer", "description": "Total nodes matching the query after filtering."}, + "counts": map[string]interface{}{"type": "object", "additionalProperties": map[string]interface{}{"type": "integer"}, "description": "Per-role node counts."}, + "has_more": map[string]interface{}{"type": "boolean", "description": "True when rows exist past this page. Decided before the blacklist / hidden-prefix / geo-filter / area passes, which drop rows from the page and rewrite total — so neither the page length nor total can be used to stop paginating. Paginate until this is false."}, }, }, "NodeDetailResponse": map[string]interface{}{ @@ -340,10 +421,15 @@ func buildOpenAPISpec(router *mux.Router, version string) map[string]interface{} op["tags"] = []string{meta.Tag} tagSet[meta.Tag] = true } + var security []map[string][]string if meta.Auth { - op["security"] = []map[string]interface{}{ - {"ApiKeyAuth": []string{}}, - } + security = append(security, map[string][]string{"ApiKeyAuth": {}}) + } + if meta.Session { + security = append(security, map[string][]string{"CookieAuth": {}}) + } + if len(security) > 0 { + op["security"] = security } // Add query parameters @@ -389,7 +475,7 @@ func buildOpenAPISpec(router *mux.Router, version string) map[string]interface{} } // Build tags array (sorted) - tagOrder := []string{"admin", "analytics", "channels", "config", "nodes", "observers", "packets"} + tagOrder := []string{"admin", "analytics", "channels", "config", "nodes", "observers", "packets", "users"} tagDescriptions := map[string]string{ "admin": "Server administration and diagnostics", "analytics": "Network analytics and statistics", @@ -398,6 +484,7 @@ func buildOpenAPISpec(router *mux.Router, version string) map[string]interface{} "nodes": "Mesh node operations", "observers": "Packet observer/gateway operations", "packets": "Packet capture and decoding", + "users": "Optional user management (accounts, sessions, admin)", } var tags []interface{} for _, t := range tagOrder { @@ -428,6 +515,12 @@ func buildOpenAPISpec(router *mux.Router, version string) map[string]interface{} "in": "header", "name": "X-API-Key", }, + "CookieAuth": map[string]string{ + "type": "apiKey", + "in": "cookie", + "name": "cs_session", + "description": "User-management session (only when userManagement.enabled). Unsafe methods also need the X-CS-CSRF header from GET /api/auth/me.", + }, }, "schemas": componentSchemas(), }, @@ -479,7 +572,9 @@ const swaggerUIHTML = ` CoreScope API — Swagger UI - + ${shell}`; +} + +let passes = 0; +let failures = 0; + +function check(fn) { + try { fn(); passes++; } catch (err) { console.error(`FAIL ${err.message}`); failures++; } +} + +// ---- Static guards: the markup contract the CSS rule depends on ---------- + +check(() => { + assert.ok( + /\.data-table th\.col-action,\s*\n\s*\.data-table td\.col-action\s*\{[^}]*max-width:\s*none[^}]*\}/.test(CSS), + '.col-action rule in public/style.css must set `max-width: none` (opts out of the global `.data-table td { max-width: 0 }` that starves the column)', + ); + console.log('PASS style.css: .col-action sets max-width: none'); +}); + +check(() => { + const rule = CSS.match(/\.data-table th\.col-action,\s*\n\s*\.data-table td\.col-action\s*\{([^}]*)\}/); + assert.ok(rule, '.col-action rule not found in public/style.css'); + assert.ok(/width:\s*1%/.test(rule[1]), '.col-action must set `width: 1%` (shrink-to-content for auto-layout tables)'); + assert.ok(/overflow:\s*visible/.test(rule[1]), '.col-action must set `overflow: visible` so a miscalculation overhangs visibly instead of cropping silently'); + console.log('PASS style.css: .col-action sets width: 1% and overflow: visible'); +}); + +check(() => { + assert.ok( + NODES_JS.includes('') && NODES_JS.includes("'' + showOnMap"), + 'public/nodes.js renderNeighborTable/renderNeighborRows must tag BOTH the action and with class="col-action"', + ); + console.log('PASS nodes.js: neighbour action and carry .col-action'); +}); + +check(() => { + const ths = (ANALYTICS_JS.match(//g) || []).length; + const tds = (ANALYTICS_JS.match(/\$\{mapBtn\}<\/td>/g) || []).length; + assert.strictEqual(ths, 2, `public/analytics.js must tag 2 action (Top Hops + Top Paths) with .col-action, found ${ths}`); + assert.strictEqual(tds, 2, `public/analytics.js must tag 2 action with .col-action, found ${tds}`); + console.log('PASS analytics.js: both map-button columns carry .col-action'); +}); + +// ---- Geometry: nothing is cropped at any desktop width ------------------- + +async function main() { + let browser; + try { + browser = await chromium.launch({ + headless: true, + executablePath: process.env.CHROMIUM_PATH || undefined, + args: ['--no-sandbox', '--disable-gpu', '--disable-dev-shm-usage'], + }); + } catch (err) { + if (process.env.CHROMIUM_REQUIRE === '1') { + console.error(`test-neighbor-map-btn-clip-e2e.js: FAIL — Chromium required but unavailable: ${err.message}`); + process.exit(1); + } + console.log(`test-neighbor-map-btn-clip-e2e.js: SKIP geometry (Chromium unavailable: ${err.message.split('\n')[0]})`); + console.log(`test-neighbor-map-btn-clip-e2e.js: ${passes} passed, ${failures} failed`); + process.exit(failures > 0 ? 1 : 0); + } + + const ctx = await browser.newContext(); + const p = await ctx.newPage(); + p.setDefaultTimeout(15000); + + for (const [shellName, shell] of Object.entries(SHELLS)) { + for (const w of WIDTHS) { + await p.setViewportSize({ width: w, height: HEIGHT }); + await p.setContent(page(shell), { waitUntil: 'load' }); + + const probe = await p.evaluate(() => { + const rows = Array.from(document.querySelectorAll('#host tbody tr')); + let worst = 0; + let clipped = 0; + let colWidth = 0; + let btnWidth = 0; + for (const tr of rows) { + const td = tr.querySelector('td.col-action'); + const btn = td && td.querySelector('.neighbor-show-map'); + if (!btn) continue; + colWidth = td.getBoundingClientRect().width; + btnWidth = btn.getBoundingClientRect().width; + const over = btn.getBoundingClientRect().right - td.getBoundingClientRect().right; + if (over > 0.5) { clipped++; if (over > worst) worst = over; } + } + return { rows: rows.length, clipped, worst, colWidth, btnWidth }; + }); + + const tag = `${shellName} vw=${w}`; + check(() => { + assert.strictEqual( + probe.clipped, 0, + `${tag}: ${probe.clipped}/${probe.rows} Map buttons cropped by their cell (worst ${probe.worst.toFixed(1)}px; column ${probe.colWidth.toFixed(1)}px vs button ${probe.btnWidth.toFixed(1)}px)`, + ); + console.log(`PASS ${tag}: column ${probe.colWidth.toFixed(1)}px holds ${probe.btnWidth.toFixed(1)}px button, 0/${probe.rows} cropped`); + }); + } + } + + await browser.close(); + console.log(`\ntest-neighbor-map-btn-clip-e2e.js: ${passes} passed, ${failures} failed`); + if (failures > 0) process.exit(1); +} + +main().catch((err) => { + console.error('test-neighbor-map-btn-clip-e2e.js: ERROR', err); + process.exit(1); +}); diff --git a/test-node-reach-coverage-e2e.js b/tests/e2e/test-node-reach-coverage-e2e.js similarity index 100% rename from test-node-reach-coverage-e2e.js rename to tests/e2e/test-node-reach-coverage-e2e.js diff --git a/test-node-reach-e2e.js b/tests/e2e/test-node-reach-e2e.js similarity index 82% rename from test-node-reach-e2e.js rename to tests/e2e/test-node-reach-e2e.js index 639333dd8..b68bc837a 100644 --- a/test-node-reach-e2e.js +++ b/tests/e2e/test-node-reach-e2e.js @@ -65,9 +65,17 @@ async function getJson(page, url) { if (!href || !href.startsWith('#/nodes/')) throw new Error('neighbour link malformed: ' + href); } - // Map must render whenever at least one link has GPS (no swallowed failure). - if (reach.links.some(l => l.lat != null && l.lon != null)) { - await page.waitForSelector('#nqMap .leaflet-container', { timeout: 10000 }); + // Leaflet puts .leaflet-container on the element you hand L.map(), so the + // class lands on #nqMap itself. The old selector had a space in it and so + // waited for a DESCENDANT carrying the class, which never exists: that is + // why this suite timed out here every time it was run. + // + // The condition is the node's own coordinates, not a link's: + // public/node-reach.js calls NodeReachMap.render only when n.lat != null, + // so a node with positioned neighbours but no position of its own has no + // map to wait for. + if (reach.node && reach.node.lat != null && reach.node.lon != null) { + await page.waitForSelector('#nqMap.leaflet-container', { timeout: 10000 }); } } diff --git a/test-nodes-export-e2e.js b/tests/e2e/test-nodes-export-e2e.js similarity index 100% rename from test-nodes-export-e2e.js rename to tests/e2e/test-nodes-export-e2e.js diff --git a/test-observer-iata-1188-e2e.js b/tests/e2e/test-observer-iata-1188-e2e.js similarity index 72% rename from test-observer-iata-1188-e2e.js rename to tests/e2e/test-observer-iata-1188-e2e.js index 7a65835d9..fa1a33511 100644 --- a/test-observer-iata-1188-e2e.js +++ b/tests/e2e/test-observer-iata-1188-e2e.js @@ -40,6 +40,41 @@ async function run() { console.log(`\nRunning observer-IATA E2E tests against ${BASE}\n`); + // #2054: row-dependent tests below opt into a fixture window. Keep the + // actual first-visit default covered independently of those preferences. + for (const width of [1400, 375]) { + await test(`Default 15-minute window renders an empty state at ${width}px`, async () => { + const emptyContext = await browser.newContext({ viewport: { width, height: 900 } }); + try { + const emptyPage = await emptyContext.newPage(); + const queries = []; + await emptyPage.route('**/api/packets?*', async route => { + queries.push(new URL(route.request().url()).searchParams); + await route.fulfill({ json: { packets: [], total: 0 } }); + }); + const started = Date.now(); + await emptyPage.goto(`${BASE}/#/packets`, { waitUntil: 'domcontentloaded' }); + await emptyPage.locator('#pktBody').getByText('No packets found', { exact: true }).waitFor(); + assert(await emptyPage.inputValue('#fTimeWindow') === '15', 'first visit must select 15 minutes'); + assert(await emptyPage.evaluate(() => localStorage.getItem('meshcore-time-window')) === null, + 'fresh context must not inherit a fixture window'); + assert(queries.length > 0, 'empty state must follow an API request'); + for (const query of queries) { + const since = Date.parse(query.get('since')); + assert(since >= started - 15 * 60000 && since <= Date.now() - 15 * 60000, + 'default request must use the 15-minute cutoff'); + assert(!query.has('hash'), 'default request must not bypass the cutoff with a hash'); + } + assert(await emptyPage.locator('#pktBody tr[data-hash]').count() === 0, + 'empty response must not render packet rows'); + assert(await emptyPage.locator('#pktTable thead th').count() > 0, + 'empty table must retain its column headings'); + } finally { + await emptyContext.close(); + } + }); + } + await test('Packets table renders an IATA badge in an observer cell', async () => { await page.goto(`${BASE}/#/packets`, { waitUntil: 'domcontentloaded' }); // Wide time window so fixture rows are in scope @@ -97,9 +132,9 @@ async function run() { const mobile = await browser.newContext({ viewport: { width: 375, height: 812 } }); const mpage = await mobile.newPage(); mpage.setDefaultTimeout(15000); + // #2054: mobile clamps the desktop fixture window (525600) back to 15. + await mpage.addInitScript(() => localStorage.setItem('meshcore-time-window', '180')); await mpage.goto(`${BASE}/#/packets`, { waitUntil: 'domcontentloaded' }); - await mpage.evaluate(() => localStorage.setItem('meshcore-time-window', '525600')); - await mpage.reload({ waitUntil: 'load' }); await mpage.waitForSelector('[data-loaded="true"]', { timeout: 20000 }); await mpage.waitForSelector('table tbody tr:not([id^=vscroll])', { timeout: 15000 }); @@ -112,8 +147,8 @@ async function run() { 'observer column should be hidden in rows at 375px (tier-3, desktop-only per #1415 spec)'); // (b) tap first row → detail panel renders observer + IATA badge - const firstRow = await mpage.$('table tbody tr[data-hash]'); - assert(firstRow, 'no packet row found to tap'); + const firstRow = mpage.locator('#pktBody tr[data-hash]').first(); + assert(await firstRow.count() > 0, 'no packet row found to tap'); await firstRow.click(); await mpage.waitForSelector('.detail-meta', { timeout: 10000 }); const detailIata = await mpage.$('.detail-meta .badge-iata'); diff --git a/tests/e2e/test-packets-compact-columns-e2e.js b/tests/e2e/test-packets-compact-columns-e2e.js new file mode 100644 index 000000000..965d7b7bb --- /dev/null +++ b/tests/e2e/test-packets-compact-columns-e2e.js @@ -0,0 +1,498 @@ +/** + * Playwright E2E — packets table column packing and the Full Names toggle. + * + * The packets table used to store every column width as a percentage of the + * table, measured once on load. Percentages scale with the screen, so on a + * wide monitor "17s ago" sat in a 180px column; and the measurement read the + * virtual-scroll spacer row (one colspan cell as wide as the table) as + * column 0, inflating the expand column too. fitColumnsToContent (app.js) + * now sizes short columns to their content in px and gives the rest to Path + * and Details. + * + * Usage: BASE_URL=http://localhost:13581 node tests/e2e/test-packets-compact-columns-e2e.js + */ +const { chromium } = require('playwright'); + +const BASE = process.env.BASE_URL || 'http://localhost:3000'; +const results = []; + +async function test(name, fn) { + try { + await fn(); + results.push({ name, pass: true }); + console.log(` ✅ ${name}`); + } catch (err) { + results.push({ name, pass: false, error: err.message }); + console.log(` ❌ ${name}: ${err.message}`); + } +} + +function assert(condition, msg) { + if (!condition) throw new Error(msg || 'Assertion failed'); +} + +// Upper bounds for the short columns at any desktop width. Content needs +// ~30-70px; the old percentage layout gave them 100-180px at 1920px. +const MAX_WIDTH = { + 'col-expand': 40, + 'col-time': 100, + 'col-size': 70, + 'col-hashsize': 50, + 'col-scope': 80, + 'col-rpt': 70, +}; + +async function gotoPackets(page, prefs) { + // The window goes in the URL: mobile clamps a stored window above 180min + // back to 15min, and the fixture is freshened at the start of the CI job. + await page.goto(BASE + '/#/packets?timeWindow=10080', { waitUntil: 'domcontentloaded' }); + await page.evaluate((extra) => { + localStorage.removeItem('packets-visible-cols'); + localStorage.removeItem('packets-known-cols'); + localStorage.removeItem('meshcore-pkt-col-px'); + localStorage.removeItem('meshcore-full-names'); + for (const [k, v] of Object.entries(extra || {})) localStorage.setItem(k, v); + }, prefs); + await page.reload({ waitUntil: 'networkidle' }); + await page.waitForSelector('#pktTable tbody tr:not([id^=vscroll]) td.col-type', + { state: 'attached', timeout: 30000 }); + await settle(page); +} + +// Column layout runs synchronously after a render, and container resizes are +// coalesced to the next animation frame: two frames cover both. +function settle(page) { + return page.evaluate(() => new Promise(r => requestAnimationFrame(() => requestAnimationFrame(r)))); +} + +// The "+N" pills are recomputed once hop names settle (up to ~1s after a +// render). Wait for that pass to finish rather than for a fixed delay. +function pillsReady(page) { + return page.waitForFunction(() => { + const tbody = document.getElementById('pktBody'); + return tbody && !tbody._rePathOverflowObserver && + document.querySelector('#pktTable .path-overflow-pill'); + }, null, { timeout: 10000 }); +} + +async function setColumnVisible(page, key, visible) { + // A checkbox click bubbles to the document handler that closes the menu, + // so each toggle needs its own open. + await page.click('#colToggleBtn'); + const box = await page.waitForSelector(`#colToggleMenu input[data-col="${key}"]`); + if ((await box.isChecked()) !== visible) await box.click(); + await page.waitForFunction(([k, v]) => + document.getElementById('pktTable').classList.contains('hide-col-' + k) === !v, [key, visible]); + await settle(page); +} + +function layout(page) { + return page.evaluate(() => { + const t = document.getElementById('pktTable'); + const wrap = t.parentElement; + const widths = {}; + t.querySelectorAll('thead th').forEach(th => { + if (th.offsetParent === null) return; + const key = Array.from(th.classList).find(c => c.startsWith('col-')); + widths[key] = th.offsetWidth; + }); + const row = Array.from(t.querySelectorAll('tbody tr')) + .find(r => r.children.length === t.querySelectorAll('thead th').length); + const cells = row ? Array.from(row.children).filter(td => td.offsetParent !== null) : []; + const lastCell = cells[cells.length - 1]; + const wr = wrap.getBoundingClientRect(); + return { + widths, + wrapRight: wr.left + wrap.clientLeft + wrap.clientWidth, + tableRight: t.getBoundingClientRect().right, + lastCellRight: lastCell ? lastCell.getBoundingClientRect().right : null, + hScroll: wrap.scrollWidth - wrap.clientWidth, + }; + }); +} + +(async () => { + console.log(`\nPackets compact columns — ${BASE}\n`); + const browser = await chromium.launch({ + headless: true, + executablePath: process.env.CHROMIUM_PATH || undefined, + args: ['--no-sandbox', '--disable-gpu', '--disable-dev-shm-usage'] + }); + const page = await browser.newPage({ viewport: { width: 1920, height: 1000 } }); + + await gotoPackets(page); + + await test('short columns stay short on a wide screen', async () => { + const { widths } = await layout(page); + for (const [col, max] of Object.entries(MAX_WIDTH)) { + if (widths[col] == null) continue; // hidden by a preference + assert(widths[col] <= max, `${col} is ${widths[col]}px, expected <= ${max}px (all: ${JSON.stringify(widths)})`); + } + }); + + await test('Path and Details take the freed width', async () => { + const { widths } = await layout(page); + assert(widths['col-path'] >= 400, `col-path only ${widths['col-path']}px`); + assert(widths['col-details'] >= 400, `col-details only ${widths['col-details']}px`); + }); + + await test('no blank strip after the last column, no horizontal scroll', async () => { + const l = await layout(page); + assert(l.lastCellRight != null, 'no data row to measure'); + assert(Math.abs(l.tableRight - l.lastCellRight) <= 2, + `last cell ends at ${l.lastCellRight}, table at ${l.tableRight}: a hidden column's slot is taking width`); + assert(l.hScroll <= 0, `table overflows its wrapper by ${l.hScroll}px`); + assert(Math.abs(l.wrapRight - l.tableRight) <= 2, + `table ends at ${l.tableRight}, its wrapper at ${l.wrapRight}`); + }); + + await test('hiding both Path and Details keeps the table at the wrapper width', async () => { + // Review on #2090: with no flex column the table shrank to the sum of + // the short columns (~400px in a 1900px wrapper). The gap was between + // the table and its wrapper, which the check above did not measure. + let l; + try { + await setColumnVisible(page, 'path', false); + await setColumnVisible(page, 'details', false); + l = await layout(page); + } finally { + // Restore before asserting, so a failure here does not leak two + // hidden columns into every later case. + await setColumnVisible(page, 'path', true); + await setColumnVisible(page, 'details', true); + } + assert(l.widths['col-path'] == null && l.widths['col-details'] == null, 'Path/Details still visible'); + assert(Math.abs(l.wrapRight - l.tableRight) <= 2, + `table ends at ${l.tableRight}, its wrapper at ${l.wrapRight}: ${Math.round(l.wrapRight - l.tableRight)}px empty`); + assert(l.hScroll <= 0, `table overflows its wrapper by ${l.hScroll}px`); + const back = await layout(page); + assert(back.widths['col-details'] >= 400, `Details did not take the width back: ${back.widths['col-details']}px`); + }); + + await test('opening the detail panel re-fits the table without scrolling', async () => { + await page.click('#pktTable tbody tr:not([id^=vscroll]) td.col-time'); + await page.waitForSelector('#pktRight:not(.empty)', { timeout: 10000 }) + .catch(() => { throw new Error('clicking a row did not open the detail panel'); }); + await settle(page); + const l = await layout(page); + assert(l.hScroll <= 0, `table overflows its wrapper by ${l.hScroll}px with the panel open`); + await page.keyboard.press('Escape'); + await page.waitForSelector('#pktRight.empty', { state: 'attached', timeout: 10000 }) + .catch(() => { throw new Error('Escape did not close the detail panel'); }); + await settle(page); + const after = await layout(page); + assert(after.widths['col-details'] >= 400, `Details did not grow back after closing the panel: ${after.widths['col-details']}px`); + }); + + await test('stale percentage widths from the old layout are dropped', async () => { + const stale = JSON.stringify([12.6, 0.4, 8.7, 9.1, 5, 3.4, 12.6, 6.2, 12.6, 12.6, 4, 12.8]); + await gotoPackets(page, { 'meshcore-pkt-col-widths': stale }); + const gone = await page.evaluate(() => localStorage.getItem('meshcore-pkt-col-widths') === null); + assert(gone, 'meshcore-pkt-col-widths still present'); + const { widths } = await layout(page); + assert(widths['col-time'] <= MAX_WIDTH['col-time'], `col-time ${widths['col-time']}px after a stale save`); + }); + + await test('Full Names shows observer names untruncated and lifts the chip cap', async () => { + await gotoPackets(page); + const before = await page.evaluate(() => ({ + cls: document.getElementById('pktTable').classList.contains('pkt-full-names'), + chipMax: (() => { const c = document.querySelector('#pktTable .path-hops .hop-named'); return c ? getComputedStyle(c).maxWidth : null; })(), + })); + assert(!before.cls, 'pkt-full-names set before toggling'); + if (before.chipMax) assert(before.chipMax === '120px', `default chip cap is ${before.chipMax}`); + + await page.click('#fullNamesToggle'); + await page.waitForFunction(() => document.getElementById('pktTable').classList.contains('pkt-full-names')); + await pillsReady(page); + await settle(page); + const after = await page.evaluate(() => { + const t = document.getElementById('pktTable'); + const obs = Array.from(t.querySelectorAll('td.col-observer')).map(td => td.textContent); + const chip = t.querySelector('.path-hops .hop-named'); + return { + cls: t.classList.contains('pkt-full-names'), + active: document.getElementById('fullNamesToggle').classList.contains('active'), + ellipsised: obs.filter(s => s.includes('…')).length, + chipMax: chip ? getComputedStyle(chip).maxWidth : null, + hScroll: t.parentElement.scrollWidth - t.parentElement.clientWidth, + // The virtual scroller positions rows at a fixed height: a path that + // wrapped onto a second line would break it. + tallPaths: Array.from(t.querySelectorAll('.path-hops')).filter(h => h.offsetHeight > 24).length, + // Hops past the edge are reachable through the +N pill, which must + // sit inside the clipped path rather than past its edge. + offscreenPills: Array.from(t.querySelectorAll('.path-overflow-pill')).filter(p => { + const host = p.closest('.path-hops').getBoundingClientRect(); + const r = p.getBoundingClientRect(); + return r.right > host.right + 1 || r.left < host.left; + }).length, + }; + }); + assert(after.cls && after.active, 'toggle did not switch the table to full names'); + assert(after.ellipsised === 0, `${after.ellipsised} observer cells still end in "…"`); + if (after.chipMax) assert(after.chipMax !== '120px', 'path chips still capped at 120px'); + assert(after.hScroll <= 0, `table overflows by ${after.hScroll}px in full-name mode`); + assert(after.tallPaths === 0, `${after.tallPaths} paths wrapped onto a second line`); + assert(after.offscreenPills === 0, `${after.offscreenPills} "+N" overflow pills are outside the visible path`); + }); + + await test('Full Names persists across reloads', async () => { + await page.reload({ waitUntil: 'networkidle' }); + await page.waitForSelector('#pktTable tbody tr:not([id^=vscroll]) td.col-type', + { state: 'attached', timeout: 30000 }); + const on = await page.evaluate(() => + document.getElementById('pktTable').classList.contains('pkt-full-names') && + document.getElementById('fullNamesToggle').classList.contains('active')); + assert(on, 'full-name mode lost on reload'); + await page.click('#fullNamesToggle'); + }); + + await test('hovering the +N pill lists the full path vertically; click pins it', async () => { + await gotoPackets(page, { 'meshcore-full-names': 'true' }); + // At 1920px with full names the fixture's multi-hop paths always overflow. + await pillsReady(page).catch(() => null); + const pillSel = await page.evaluate(() => { + const pills = Array.from(document.querySelectorAll('#pktTable .path-overflow-pill')); + if (!pills.length) return null; + pills[0].closest('tr').setAttribute('data-e2e-pill-row', '1'); + return '#pktTable tr[data-e2e-pill-row] .path-overflow-pill'; + }); + // At 1920px with full names the fixture's multi-hop paths always overflow; + // no pill means the pill logic broke, not that there is nothing to test. + assert(pillSel, 'no "+N" overflow pill rendered under Full Names'); + const hopsInRow = await page.$eval(pillSel, p => + p.closest('.path-hops').querySelectorAll('.arrow').length + 1); + + await page.hover(pillSel); + await page.waitForSelector('#pathPopover', { timeout: 3000 }); + // The pill sits over the last chip: a translucent hover background let + // the chip show through it. + const hoverBg = await page.$eval(pillSel, p => getComputedStyle(p).backgroundColor); + const alpha = /rgba\([^)]*,\s*([\d.]+)\)/.exec(hoverBg); + assert(hoverBg !== 'transparent' && (!alpha || parseFloat(alpha[1]) === 1), + `hovered pill background is not opaque: ${hoverBg}`); + const pop = await page.evaluate(() => { + const el = document.getElementById('pathPopover'); + const lis = Array.from(el.querySelectorAll('li')); + return { + items: lis.length, + vertical: lis.every((li, i) => i === 0 || li.getBoundingClientRect().top > lis[i - 1].getBoundingClientRect().top), + truncated: Array.from(el.querySelectorAll('.hop-named')).filter(c => c.scrollWidth > c.clientWidth + 1).length, + title: el.querySelector('.path-popover-title').textContent, + }; + }); + assert(pop.items === hopsInRow, `popover lists ${pop.items} hops, row has ${hopsInRow}`); + assert(pop.vertical, 'hops are not stacked one per line'); + assert(pop.truncated === 0, `${pop.truncated} names truncated in the popover`); + assert(pop.title.includes(hopsInRow + ' hop'), `title "${pop.title}" should count hops, not arrows`); + + // Moving away closes an unpinned popover (after a 150ms grace period). + await page.mouse.move(5, 5); + await page.waitForSelector('#pathPopover', { state: 'detached', timeout: 3000 }) + .catch(() => { throw new Error('popover still open after the pointer left'); }); + + // Clicking pins it, and must not select the row underneath. + const panelEmptyBefore = await page.$eval('#pktRight', el => el.classList.contains('empty')); + await page.click(pillSel); + await page.waitForSelector('#pathPopover', { timeout: 3000 }); + await page.mouse.move(5, 5); + // Deliberately fixed: proving it stays open means outlasting the 150ms + // close grace an unpinned popover would get. + await page.waitForTimeout(400); + assert(await page.$('#pathPopover'), 'pinned popover closed when the pointer left'); + const panelEmptyAfter = await page.$eval('#pktRight', el => el.classList.contains('empty')); + assert(panelEmptyBefore === panelEmptyAfter, 'clicking the pill also selected the row'); + await page.keyboard.press('Escape'); + await page.waitForSelector('#pathPopover', { state: 'detached', timeout: 3000 }) + .catch(() => { throw new Error('Escape did not close the pinned popover'); }); + + // Keyboard: focusing a pill shows the popover too. Escape above also + // closes the detail panel, which re-renders the rows: query afresh. + await pillsReady(page); + await page.focus('#pktTable .path-overflow-pill'); + await page.waitForSelector('#pathPopover', { timeout: 3000 }); + await page.keyboard.press('Escape'); + + // The popover lives on : leaving the page must take it along. + await page.click('#pktTable .path-overflow-pill'); + await page.waitForSelector('#pathPopover', { timeout: 3000 }); + await page.evaluate(() => { location.hash = '#/home'; }); + await page.waitForSelector('#pathPopover', { state: 'detached', timeout: 5000 }) + .catch(() => { throw new Error('pinned path popover survived navigating away from Packets'); }); + await page.evaluate(() => { localStorage.removeItem('meshcore-full-names'); }); + }); + + await test('Full Names is carried in the URL', async () => { + await gotoPackets(page); + // Don't assume the starting state: the previous test turned Full Names on, + // and the in-page setting (not just localStorage) can reach this page's + // URL before gotoPackets reloads it. Check both directions instead. + const state = () => page.evaluate(() => ({ + on: document.getElementById('pktTable').classList.contains('pkt-full-names'), + inUrl: /[?&]fullNames=1\b/.test(location.hash), + hash: location.hash, + })); + for (let n = 0; n < 2; n++) { + const was = (await state()).on; + await page.click('#fullNamesToggle'); + await page.waitForFunction(w => document.getElementById('pktTable').classList.contains('pkt-full-names') !== w, was); + const s = await state(); + assert(s.on === s.inUrl, `Full Names is ${s.on ? 'on' : 'off'} but the URL says otherwise: ${s.hash}`); + } + // A shared link switches the mode on for a visitor whose own pref is off. + await page.evaluate(() => { localStorage.setItem('meshcore-full-names', 'false'); }); + await page.goto(BASE + '/#/packets?timeWindow=10080&fullNames=1', { waitUntil: 'domcontentloaded' }); + await page.reload({ waitUntil: 'networkidle' }); + await page.waitForSelector('#pktTable tbody tr:not([id^=vscroll]) td.col-type', { state: 'attached', timeout: 30000 }); + const on = await page.$eval('#pktTable', t => t.classList.contains('pkt-full-names')); + assert(on, 'fullNames=1 in the URL did not turn Full Names on'); + // ...for that page only: the visitor's own saved preference is untouched. + const saved = await page.evaluate(() => localStorage.getItem('meshcore-full-names')); + assert(saved === 'false', `opening a fullNames=1 link overwrote the saved preference with ${saved}`); + await page.goto(BASE + '/#/packets?timeWindow=10080', { waitUntil: 'domcontentloaded' }); + await page.reload({ waitUntil: 'networkidle' }); + await page.waitForSelector('#pktTable tbody tr:not([id^=vscroll]) td.col-type', { state: 'attached', timeout: 30000 }); + const offAgain = await page.$eval('#pktTable', t => !t.classList.contains('pkt-full-names')); + assert(offAgain, 'without the parameter the page did not return to the saved preference (off)'); + await page.evaluate(() => { localStorage.removeItem('meshcore-full-names'); }); + }); + + await test('Observer column shrinks back once observer names arrive late', async () => { + // Rows can render before /api/observers resolves (#1692), showing raw + // 64-char pubkeys — ~600px under Full Names. grow() alone would keep the + // column that wide after the names replace them. + await page.route('**/api/observers*', async route => { + await new Promise(r => setTimeout(r, 3000)); + await route.continue(); + }); + try { + await gotoPackets(page, { 'meshcore-full-names': 'true' }); + await page.waitForFunction(() => + !Array.from(document.querySelectorAll('#pktTable td.col-observer')) + .some(td => /[0-9A-F]{40}/.test(td.textContent)), null, { timeout: 15000 }); + await settle(page); + const { th, widest } = await page.evaluate(() => { + const cells = Array.from(document.querySelectorAll('#pktTable td.col-observer')) + .filter(td => td.offsetParent !== null); + return { + th: document.querySelector('#pktTable th.col-observer').offsetWidth, + widest: Math.max(...cells.map(td => cellContentWidth(td))), + }; + }); + assert(th <= widest + 8, `Observer column ${th}px for ${widest}px of content: stuck at the pubkey width`); + } finally { + await page.unroute('**/api/observers*'); + } + }); + + await test('dragging a handle resizes the column; double-click fits it back', async () => { + await gotoPackets(page); + const flexHandles = await page.$$eval('#pktTable th.col-path .col-resize-handle, #pktTable th.col-details .col-resize-handle', h => h.length); + assert(flexHandles === 0, 'Path/Details must not have drag handles: they share the remaining width'); + const before = await page.$eval('#pktTable th.col-hash', th => th.offsetWidth); + // Grab the handle 2px inside the header: its outer half overhangs the + // next header, which paints over it. + const box = await page.$eval('#pktTable th.col-hash', th => { + const r = th.getBoundingClientRect(); return { x: r.right - 2, y: r.top + r.height / 2 }; + }); + await page.mouse.move(box.x, box.y); + await page.mouse.down(); + await page.mouse.move(box.x + 30, box.y, { steps: 3 }); + await page.mouse.move(box.x + 60, box.y, { steps: 3 }); + await page.mouse.up(); + await settle(page); + const dragged = await page.evaluate(() => ({ + w: document.querySelector('#pktTable th.col-hash').offsetWidth, + saved: JSON.parse(localStorage.getItem('meshcore-pkt-col-px') || '{}')['col-hash'], + })); + // The edge follows the pointer: no lag from fixed layout spreading the delta. + assert(Math.abs(dragged.w - (before + 60)) <= 5, `dragged +60px from ${before}px, column is ${dragged.w}px`); + assert(Math.abs(dragged.saved - dragged.w) <= 1, `saved ${dragged.saved}px, column ${dragged.w}px`); + const edge = await page.$eval('#pktTable th.col-hash', th => { + const r = th.getBoundingClientRect(); return { x: r.right - 2, y: r.top + r.height / 2 }; + }); + await page.mouse.dblclick(edge.x, edge.y); + await settle(page); + const reset = await page.evaluate(() => ({ + w: document.querySelector('#pktTable th.col-hash').offsetWidth, + saved: JSON.parse(localStorage.getItem('meshcore-pkt-col-px') || '{}')['col-hash'], + })); + assert(Math.abs(reset.w - before) <= 1, `double-click left the column at ${reset.w}px, expected ${before}px`); + assert(reset.saved === undefined, 'double-click did not forget the saved width'); + }); + + await test('expanding a group leaves the column widths alone', async () => { + // CI seeds one grouped transmission (fae0c9e6d357a814, 3 observations). + await page.goto(BASE + '/#/packets?hash=fae0c9e6d357a814&timeWindow=0', { waitUntil: 'networkidle' }); + // Fail, don't skip: without the seeded row this case would silently stop + // asserting (review on #2090). + await page.waitForSelector('#pktTable tr.group-header', { timeout: 10000 }) + .catch(() => { throw new Error('seeded grouped row fae0c9e6d357a814 not found: check the CI fixture seeding step'); }); + await settle(page); + const widths = () => page.$$eval('#pktTable thead th', ths => Object.fromEntries(ths + .filter(th => th.offsetParent !== null && !th.matches('.col-path, .col-details')) + .map(th => [th.className.split(' ')[0], th.offsetWidth]))); + const before = await widths(); + await page.click('#pktTable tr.group-header td.col-expand'); + await page.waitForSelector('#pktTable tr.group-child', { timeout: 10000 }); + await settle(page); + const after = await widths(); + // Child rows name other observers and may legitimately need a few px + // more; the bug was every column jumping ~14px from the row indent. + const grew = Object.keys(before).filter(k => k !== 'col-time' && after[k] - before[k] > 8); + assert(grew.length === 0, `columns widened on expand: ${grew.map(k => `${k} ${before[k]}→${after[k]}`).join(', ')}`); + await page.keyboard.press('Escape'); + }); + + await test('1100px with the detail panel open: the table still fits', async () => { + // All columns are visible above 1024px, and the panel leaves ~650px. + await page.setViewportSize({ width: 1100, height: 900 }); + await gotoPackets(page); + await page.click('#pktTable tbody tr:not([id^=vscroll]) td.col-time'); + await page.waitForSelector('#pktRight:not(.empty)', { timeout: 10000 }); + await settle(page); + const { hScroll } = await layout(page); + assert(hScroll <= 0, `table overflows its wrapper by ${hScroll}px`); + await page.keyboard.press('Escape'); + }); + + await test('revealing and re-hiding responsive columns re-fits the table', async () => { + // At <=1024px TableResponsive hides Hash, Observer and Rpt behind "+N hidden". + await page.setViewportSize({ width: 1000, height: 900 }); + await gotoPackets(page); + const pill = await page.$('#pktTable .col-hidden-pill:not(.col-rehide-pill)'); + assert(pill, 'no "+N hidden" pill at 1000px'); + const check = async (label) => { + const l = await layout(page); + assert(l.hScroll <= 0, `${label}: table overflows by ${l.hScroll}px`); + assert(Math.abs(l.tableRight - l.lastCellRight) <= 2, + `${label}: last cell ends at ${l.lastCellRight}, table at ${l.tableRight} (blank strip)`); + }; + await pill.click(); + await page.waitForFunction(() => document.querySelector('#pktTable th.col-observer').offsetParent !== null, null, { timeout: 5000 }) + .catch(() => { throw new Error('Observer not revealed'); }); + await settle(page); + await check('after reveal'); + await page.click('#pktTable .col-rehide-pill'); + await page.waitForFunction(() => document.querySelector('#pktTable th.col-observer').offsetParent === null, null, { timeout: 5000 }); + await settle(page); + await check('after re-hide'); + }); + + await test('mobile (375px): table fits without horizontal scroll', async () => { + await page.setViewportSize({ width: 375, height: 800 }); + await gotoPackets(page); + const { hScroll } = await layout(page); + assert(hScroll <= 0, `table overflows by ${hScroll}px at 375px`); + }); + + await browser.close(); + + const passed = results.filter(r => r.pass).length; + const failed = results.length - passed; + console.log(`\n${passed} passed, ${failed} failed`); + process.exit(failed ? 1 : 0); +})().catch(err => { + console.error(err); + process.exit(1); +}); diff --git a/test-packets-scope-column.js b/tests/e2e/test-packets-scope-column.js similarity index 76% rename from test-packets-scope-column.js rename to tests/e2e/test-packets-scope-column.js index 0d9668888..4f0b28d09 100644 --- a/test-packets-scope-column.js +++ b/tests/e2e/test-packets-scope-column.js @@ -33,9 +33,44 @@ async function gotoPackets(page) { await page.evaluate(() => { localStorage.removeItem('packets-visible-cols'); localStorage.removeItem('packets-known-cols'); + // The packets page defaults to a 15-minute window (public/packets.js, + // savedTimeWindowMin). CI freshens the fixture so its newest packet is + // "now" at the START of the job, and the E2E step runs for a quarter of + // an hour or more, so a suite that runs late in the list sees an empty + // table and "No packets found" through no fault of its own. This suite is + // about the Scope column, not about the window, so pin the window wide. + localStorage.setItem('meshcore-time-window', '10080'); }); await page.reload({ waitUntil: 'networkidle' }); - await page.waitForSelector('#pktTable tbody tr:not([id^=vscroll])', { timeout: 30000 }); + // Wait for a row that carries real column cells, not merely for any . + // The table renders a full-width placeholder row while it loads, which + // satisfies a bare `tbody tr` selector: the suite then read an empty table + // and reported "no td.col-scope rendered" / "no packet rows found". That is + // the 4-passed-3-failed signature this suite showed intermittently while it + // was unwired, and it is a race in the wait, not a product fault. + // + // state:'attached', not the default 'visible': every assertion below reads + // the DOM through $$eval/evaluate, which sees a cell whose column is hidden + // by a preference class. Waiting for visibility asks for more than the + // suite needs and times out on a table that is perfectly ready to inspect. + try { + await page.waitForSelector('#pktTable tbody tr:not([id^=vscroll]) td.col-type', + { state: 'attached', timeout: 30000 }); + } catch (e) { + // A bare TimeoutError says nothing about why. Report what the table held. + const state = await page.evaluate(() => { + const tb = document.querySelector('#pktTable tbody'); + if (!tb) return { table: false }; + const trs = Array.from(tb.querySelectorAll('tr')); + return { + table: true, + rows: trs.length, + firstRowHtml: trs.length ? trs[0].innerHTML.slice(0, 200) : null, + tableClass: document.getElementById('pktTable').className, + }; + }).catch(() => null); + throw new Error('packets table never produced a row with td.col-type: ' + JSON.stringify(state)); + } } (async () => { diff --git a/test-path-inspector-coverage-e2e.js b/tests/e2e/test-path-inspector-coverage-e2e.js similarity index 100% rename from test-path-inspector-coverage-e2e.js rename to tests/e2e/test-path-inspector-coverage-e2e.js diff --git a/tests/e2e/test-path-inspector-e2e.js b/tests/e2e/test-path-inspector-e2e.js new file mode 100644 index 000000000..65f606be5 --- /dev/null +++ b/tests/e2e/test-path-inspector-e2e.js @@ -0,0 +1,300 @@ +#!/usr/bin/env node +/* Path Inspector — real fixture API, map route drawing/replacement, side pane, + * legacy trace redirect and tools landing. Seed test-fixtures/path-inspector.sql + * after migrating the fixture and BEFORE starting the server (see #2060). + * Missing candidates are failures, never skips or mocked responses. + */ +'use strict'; + +const assert = require('node:assert/strict'); +const { chromium } = require('playwright'); + +const BASE = process.env.BASE_URL || 'http://localhost:13581'; +// Three-byte prefixes meet the default path-trust policy. The middle prefix +// deliberately matches two repeaters, giving two distinct routes to draw. +const PREFIXES = 'f20601,f20602,f20603'; +const key = (prefix, suffix = '0') => prefix + suffix.repeat(64 - prefix.length); +const POSITIONS = { + [key('f20601')]: [1, 1], + [key('f20602', '1')]: [1.01, 1.01], + [key('f20602', '2')]: [0.99, 1.01], + [key('f20603')]: [1, 1.02], +}; + +let passes = 0, failures = 0; +function pass(msg) { console.log(` ✓ ${msg}`); passes++; } +function fail(msg) { console.error(` ✗ ${msg}`); failures++; } + +// The toggle ships in the page template, but its click handler is attached by +// initMapSidePane(), which public/map.js calls at the END of loadNodes() +// (map.js:1795). So the button exists for seconds before it does anything, and +// a single click on sight is a race that silently no-ops. Measured at ~3s +// against a populated instance. +// +// Clicking for real each attempt rather than dispatching in-page on purpose: a +// click that cannot land (an overlay eating it, as in #2049) has to fail here, +// which an element.click() from evaluate would hide. +async function expandPane(page) { + await page.waitForSelector('#mapPaneToggle', { timeout: 15000 }); + for (let i = 0; i < 20; i++) { + const expanded = await page.evaluate(() => { + const el = document.getElementById('mapSidePane'); + return !!el && /\bexpanded\b/.test(el.className); + }); + if (expanded) return; + try { + await page.click('#mapPaneToggle', { timeout: 1500 }); + } catch { /* not clickable yet; the next round re-checks */ } + await page.waitForTimeout(500); + } + throw new Error('the pane never expanded after 20 clicks over ~20s'); +} + +async function openPaneAndSubmit(page) { + await page.goto(`${BASE}/#/map`, { waitUntil: 'domcontentloaded' }); + await expandPane(page); + await page.fill('#mapPiInput', PREFIXES); + await page.click('#mapPiSubmit'); + await page.waitForSelector('#mapPiResults button[data-idx="1"]', { timeout: 10000 }); +} + +async function assertDesktopLayout(page) { + const layout = await page.evaluate(() => ({ + sidebar: document.querySelector('.mc-rt-sidebar').getBoundingClientRect().toJSON(), + map: document.querySelector('#leaflet-map').getBoundingClientRect().toJSON(), + inspector: document.querySelector('#mapSidePane').getBoundingClientRect().toJSON(), + viewport: innerWidth, + })); + assert.equal(layout.sidebar.left, 0, 'route sidebar stays on the left'); + assert.ok(layout.sidebar.right <= layout.map.left + 1, 'map must reserve the route sidebar width'); + assert.ok(layout.map.right <= layout.inspector.left + 1, 'map must not cover the Path Inspector'); + assert.ok(layout.map.width >= 160, 'map retains at least 160px of usable space'); + assert.ok(layout.inspector.right <= layout.viewport + 1, 'inspector stays within the viewport'); +} + +async function resizeSidebar(page, width) { + const current = await page.locator('.mc-rt-sidebar').boundingBox(); + const handle = await page.locator('.mc-rt-resize-handle').boundingBox(); + // The collapse button overlaps the handle's midpoint; drag its upper half. + await page.mouse.move(handle.x + handle.width / 2, handle.y + handle.height / 4); + await page.mouse.down(); + await page.mouse.move(handle.x + handle.width / 2 + width - current.width, handle.y + handle.height / 4, { steps: 4 }); + await page.mouse.up(); +} + +async function main() { + const requireChromium = process.env.CHROMIUM_REQUIRE === '1'; + let browser; + try { + browser = await chromium.launch({ headless: true, executablePath: process.env.CHROMIUM_PATH || undefined }); + } catch (err) { + if (requireChromium) { + console.error(`HARD FAIL — Chromium unavailable: ${err.message}`); + process.exit(1); + } + console.warn(`SKIP — Chromium unavailable: ${err.message}`); + process.exit(0); + } + + const ctx = await browser.newContext({ viewport: { width: 1280, height: 900 } }); + const page = await ctx.newPage(); + + // (1) The side pane exists and starts collapsed. + await page.goto(`${BASE}/#/map`, { waitUntil: 'domcontentloaded' }); + try { + await page.waitForSelector('#mapSidePane', { timeout: 15000 }); + const cls = await page.getAttribute('#mapSidePane', 'class'); + if (cls && /\bexpanded\b/.test(cls)) fail(`(1) the side pane starts expanded (class="${cls}")`); + else pass('(1) the side pane is present and collapsed by default'); + } catch { + fail('(1) #mapSidePane never appeared within 15s'); + } + + // (2) The toggle expands it. + try { + await expandPane(page); + pass('(2) clicking the toggle expands the pane'); + } catch { + const cls = await page.getAttribute('#mapSidePane', 'class').catch(() => '(missing)'); + fail(`(2) the pane did not gain .expanded (class="${cls}")`); + } + + // (3) Real API candidates must resolve to both seeded routes. An HTTP 200 + // with an empty result used to count as coverage for this round trip. + try { + const response = await page.request.post(BASE + '/api/paths/inspect', { + data: { prefixes: PREFIXES.split(',') }, + }); + assert.equal(response.status(), 200, 'fixture inspector API must be ready'); + const data = await response.json(); + assert.equal(data.candidates.length, 2, 'fixture must return two distinct candidates; apply path-inspector.sql before server startup'); + assert.deepEqual(data.candidates.map(candidate => candidate.path.join(',')).sort(), [ + [key('f20601'), key('f20602', '1'), key('f20603')].join(','), + [key('f20601'), key('f20602', '2'), key('f20603')].join(','), + ].sort(), 'both complete fixture routes must be returned, regardless of score order'); + for (const candidate of data.candidates) { + assert.equal(candidate.path.length, 3); + assert.equal(candidate.speculative, false, 'seeded graph must support both normal routes'); + candidate.path.forEach(pk => assert.ok(POSITIONS[pk], 'candidate must use a seeded GPS repeater')); + assert.equal(candidate.evidence.perHop.length, 3); + assert.ok(candidate.evidence.perHop.every(hop => hop.trusted), 'three-byte hops must satisfy the unchanged trust policy'); + } + await openPaneAndSubmit(page); + assert.equal(await page.locator('#mapPiResults button[data-idx]').count(), 2); + pass('(3) submitting prefixes renders both real, trusted fixture candidates'); + + // (4) Check actual Leaflet polylines, their coordinates and rendered SVG. + // No forced clicks: overlays blocking either candidate must fail the test. + for (let index = 0; index < 2; index++) { + const path = data.candidates[index].path.map(pk => POSITIONS[pk]); + const expected = [path.slice(0, 2), path.slice(1, 3)]; + if (index === 1) { + const receivesClick = await page.locator('#mapPiResults button[data-idx="1"]').evaluate(button => { + const rect = button.getBoundingClientRect(); + return button.contains(document.elementFromPoint(rect.x + rect.width / 2, rect.y + rect.height / 2)); + }); + assert.ok(receivesClick, '#2081: the rendered map must not intercept the second candidate button'); + } + await page.locator(`#mapPiResults button[data-idx="${index}"]`).click(); + await page.waitForFunction(expected => { + const group = window.__mc_routeLayer; + if (!group) return false; + const lines = group.getLayers().filter(layer => layer instanceof L.Polyline && !(layer instanceof L.Polygon)); + const mapRect = window.__mc_map.getContainer().getBoundingClientRect(); + const visible = line => { + const element = line.getElement(); + if (!element || !element.getAttribute('d')) return false; + const style = getComputedStyle(element); + const rect = element.getBoundingClientRect(); + return style.display !== 'none' && style.visibility === 'visible' && Number(style.opacity) > 0 && + rect.width > 0 && rect.height > 0 && rect.right > mapRect.left && rect.left < mapRect.right && + rect.bottom > mapRect.top && rect.top < mapRect.bottom; + }; + return lines.length === 2 && expected.every(points => lines.some(line => + JSON.stringify(line.getLatLngs().map(p => [p.lat, p.lng])) === JSON.stringify(points) && + window.__mc_map.hasLayer(line) && visible(line) + )); + }, expected, { timeout: 10000 }); + if (index === 0) { + await page.evaluate(() => { window.__previousInspectorLayers = window.__mc_routeLayer.getLayers(); }); + pass('(4) Show on Map draws both segments at the first candidate coordinates'); + } else { + const previousRemoved = await page.evaluate(() => window.__previousInspectorLayers.every(layer => + !window.__mc_routeLayer.hasLayer(layer) && !window.__mc_map.hasLayer(layer) + )); + assert.ok(previousRemoved, 'switching candidates must remove every prior route layer from the map'); + pass('(5) switching candidates draws the other route and removes all prior route objects'); + } + } + await assertDesktopLayout(page); + + // The desktop layout repair must preserve the mobile fixed bottom sheet. + await page.setViewportSize({ width: 375, height: 812 }); + const mobile = await page.locator('.mc-rt-sidebar').evaluate(sidebar => ({ + position: getComputedStyle(sidebar).position, + rect: sidebar.getBoundingClientRect().toJSON(), + map: document.querySelector('#leaflet-map').getBoundingClientRect().toJSON(), + })); + assert.equal(mobile.position, 'fixed'); + assert.equal(mobile.rect.left, 0); + assert.equal(mobile.rect.width, 375); + assert.equal(mobile.map.left, 0); + assert.equal(mobile.map.width, 375); + assert.ok(mobile.rect.bottom <= 812 && mobile.rect.top > 600, 'mobile route details remain a bottom sheet'); + await page.locator('.mc-rt-mobile-handle').click(); + assert.equal(await page.locator('.mc-rt-mobile-handle').getAttribute('aria-expanded'), 'true'); + await page.waitForFunction(() => document.querySelector('.mc-rt-sidebar').getBoundingClientRect().height >= innerHeight * 0.75 - 1); + const expanded = await page.locator('.mc-rt-sidebar').evaluate(sidebar => ({ + rect: sidebar.getBoundingClientRect().toJSON(), + map: document.querySelector('#leaflet-map').getBoundingClientRect().toJSON(), + })); + assert.equal(expanded.rect.left, 0); + assert.equal(expanded.rect.width, 375); + assert.ok(Math.abs(expanded.rect.bottom - mobile.rect.bottom) < 1, 'expanded sheet keeps its bottom anchor'); + assert.ok(expanded.map.height > 0 && expanded.map.height < mobile.map.height, 'expanded sheet leaves a smaller usable map'); + assert.ok(expanded.map.bottom <= expanded.rect.top + 1, 'map ends above the expanded sheet'); + await page.locator('.mc-rt-mobile-handle').click(); + assert.equal(await page.locator('.mc-rt-mobile-handle').getAttribute('aria-expanded'), 'false'); + await page.waitForFunction(initial => { + const sidebar = document.querySelector('.mc-rt-sidebar').getBoundingClientRect(); + const map = document.querySelector('#leaflet-map').getBoundingClientRect(); + return Math.abs(sidebar.height - initial.rect.height) < 1 && Math.abs(sidebar.top - initial.rect.top) < 1 && + Math.abs(map.height - initial.map.height) < 1 && Math.abs(map.bottom - initial.map.bottom) < 1; + }, mobile); + pass('(6) mobile sheet expansion reserves map space and collapse restores both geometries'); + + await page.setViewportSize({ width: 1280, height: 900 }); + await page.locator('.mc-rt-collapse-btn').click(); + await assertDesktopLayout(page); + assert.equal(await page.locator('.mc-rt-sidebar').evaluate(el => el.getBoundingClientRect().width), 36); + await page.locator('.mc-rt-collapse-btn').click(); + await assertDesktopLayout(page); + pass('(7) collapsing and restoring the route sidebar keeps both map and inspector accessible'); + + await resizeSidebar(page, 400); + assert.equal(await page.locator('.mc-rt-sidebar').evaluate(el => el.getBoundingClientRect().width), 400); + await assertDesktopLayout(page); + pass('(8) dragging the route sidebar to 400px preserves space for the inspector'); + await resizeSidebar(page, 700); + assert.equal(await page.evaluate(() => localStorage.getItem('mc-rt-sidebar-width')), '700'); + for (const width of [900, 768]) { + await page.setViewportSize({ width, height: 900 }); + await assertDesktopLayout(page); + await page.locator('.mc-rt-collapse-btn').click(); + await assertDesktopLayout(page); + assert.equal(await page.locator('.mc-rt-sidebar').evaluate(el => el.getBoundingClientRect().width), 36); + await page.locator('.mc-rt-collapse-btn').click(); + await assertDesktopLayout(page); + } + // A saved desktop preference must be constrained on a new narrow page too. + // The previous route's sidebar survives the hash-only goto to #/map, and + // the new one replaces it only after /api/resolve-hops answers. Waiting + // for any .mc-rt-sidebar matched the old one, so the checks below could + // measure it, or measure it just after its removal: a detached element + // has a 0px rect, which is how this failed intermittently in CI. + await openPaneAndSubmit(page); + await page.evaluate(() => { window.__previousRouteSidebar = document.querySelector('.mc-rt-sidebar'); }); + await page.locator('#mapPiResults button[data-idx="0"]').click(); + await page.waitForFunction(() => { + const sidebar = document.querySelector('.mc-rt-sidebar'); + return !!sidebar && sidebar !== window.__previousRouteSidebar; + }, null, { timeout: 10000 }); + assert.equal(await page.locator('.mc-rt-sidebar').evaluate(el => el.style.width), '700px'); + await assertDesktopLayout(page); + await page.setViewportSize({ width: 1280, height: 900 }); + assert.equal(await page.locator('.mc-rt-sidebar').evaluate(el => el.getBoundingClientRect().width), 700); + await assertDesktopLayout(page); + pass('(9) dragged and saved 700px widths adapt at 900/768px and restore on a wider viewport'); + } catch (err) { + fail('(3-9) fixture route coverage: ' + err.message); + } + + // (10) The legacy trace URL still redirects. + await page.goto(`${BASE}/#/traces/abc123`, { waitUntil: 'domcontentloaded' }); + try { + await page.waitForFunction(() => location.hash.indexOf('#/tools/trace/abc123') === 0, null, { timeout: 5000 }); + pass('(10) /#/traces/ redirects to /#/tools/trace/'); + } catch { + fail(`(10) no redirect; the URL is ${JSON.stringify(page.url())}`); + } + + // (11) The tools landing lists both tools. + await page.goto(`${BASE}/#/tools`, { waitUntil: 'domcontentloaded' }); + try { + await page.waitForSelector('.tools-landing', { timeout: 8000 }); + const links = await page.evaluate(() => ({ + pi: !!document.querySelector('a[href="#/tools/path-inspector"]'), + trace: !!document.querySelector('a[href*="#/tools/trace"]'), + })); + if (links.pi && links.trace) pass('(11) the tools landing links to both tools'); + else fail(`(11) the tools landing is missing a link (path-inspector: ${links.pi}, trace: ${links.trace})`); + } catch { + fail('(11) .tools-landing never rendered within 8s'); + } + + await browser.close(); + console.log(`\ntest-path-inspector-e2e: ${passes} passed, ${failures} failed`); + process.exit(failures ? 1 : 0); +} + +main().catch((e) => { console.error(e); process.exit(1); }); diff --git a/tests/e2e/test-postgres-backup-e2e.js b/tests/e2e/test-postgres-backup-e2e.js new file mode 100644 index 000000000..694437f18 --- /dev/null +++ b/tests/e2e/test-postgres-backup-e2e.js @@ -0,0 +1,128 @@ +/** + * Native SQLite/PostgreSQL backup and restored-browser-session proof. + * Run test-user-management-e2e.js first to create its synthetic accounts. + * + * BASE_URL=http://localhost:13582 E2E_BACKUP_DIR= + * node tests/e2e/test-postgres-backup-e2e.js capture + * Restore telemetry.dump and accounts.dump into fresh PostgreSQL databases, + * grant the restricted runtime roles, and restart the fixture server against + * those databases. Then use the SAME directory and local browser origin: + * node tests/e2e/test-postgres-backup-e2e.js verify + * + * The directory contains account snapshots and session cookies. Keep it private + * and outside the checkout; never publish it as a CI artifact. + */ +'use strict'; +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const { isDeepStrictEqual } = require('node:util'); +const { chromium } = require('playwright'); + +const BASE = process.env.BASE_URL || 'http://localhost:13582'; +const DIR = process.env.E2E_BACKUP_DIR || ''; +const MODE = process.argv[2]; +const BACKEND = process.env.CORESCOPE_TEST_BACKEND || 'postgres'; +assert(['sqlite','postgres'].includes(BACKEND), 'explicit supported backup backend required'); +const EXTENSION = BACKEND === 'sqlite' ? 'db' : 'dump'; +const HASH = 'fae0c9e6d357a814'; +const ACCOUNTS = ['admin@e2e.test', 'sync@e2e.test']; +let passed = 0; + +function pass(name) { passed++; console.log(' ✓ ' + name); } +function save(name, value) { + fs.writeFileSync(path.join(DIR, name), JSON.stringify(value), { mode: 0o600, flag: 'wx' }); +} +async function json(page, route) { + const response = await page.request.get(BASE + route); + assert.equal(response.status(), 200, route + ' status'); + return response.json(); +} +async function telemetry(page) { + const stats = await json(page, '/api/stats'); + const counts = {}; + for (const key of ['totalTransmissions', 'totalObservations', 'totalNodes', 'totalObservers']) { + assert(Number.isInteger(stats[key]) && stats[key] > 0, key + ' must contain fixture data'); + counts[key] = stats[key]; + } + const detail = await json(page, '/api/packets/' + HASH); + assert.equal(detail.packet.hash, HASH); + assert.equal(detail.observations.length, 3, 'seeded observation count'); + return { counts, hash: detail.packet.hash, observations: detail.observations.map(o => ({ + id: o.id, raw_hex: o.raw_hex, path_json: o.path_json, + })).sort((a, b) => a.id - b.id) }; +} + +(async () => { + assert(['capture', 'verify'].includes(MODE), 'use capture or verify'); + assert(['localhost', '127.0.0.1', '[::1]'].includes(new URL(BASE).hostname), 'local fixture server required'); + assert(path.isAbsolute(DIR), 'E2E_BACKUP_DIR must be a private absolute directory'); + const repo = path.resolve(__dirname, '../..'); + const relative = path.relative(repo, DIR); + assert(relative.startsWith('..' + path.sep) || path.isAbsolute(relative), 'backup directory must be outside the checkout'); + fs.mkdirSync(DIR, { recursive: true, mode: 0o700 }); + const expected = MODE === 'verify' ? JSON.parse(fs.readFileSync(path.join(DIR, 'expected.json'), 'utf8')) : { accounts: [] }; + const browser = await chromium.launch({ headless: true, + executablePath: process.env.CHROMIUM_PATH || undefined, + args: ['--no-sandbox', '--disable-gpu', '--disable-dev-shm-usage'] }); + try { + const pages = []; + for (let i = 0; i < ACCOUNTS.length; i++) { + const statePath = path.join(DIR, 'browser-' + i + '.json'); + const context = await browser.newContext(MODE === 'verify' ? { storageState: statePath } : {}); + const page = await context.newPage(); + page.setDefaultTimeout(10000); + if (MODE === 'capture') { + await page.goto(BASE + '/#/account/login', { waitUntil: 'domcontentloaded' }); + await page.fill('#loginEmail', ACCOUNTS[i]); + await page.fill('#loginPassword', 'correct horse battery'); + await page.click('#loginForm button[type="submit"]'); + } else { + // No login request: the pre-backup cookie must resolve the restored session. + await page.goto(BASE + '/#/account', { waitUntil: 'domcontentloaded' }); + } + await page.waitForSelector('#profileForm'); + await page.evaluate(() => window.CSAuth.ready()); + if (MODE === 'capture') await page.evaluate(() => window.CSSettingsSync.syncNow()); + const identity = await json(page, '/api/auth/me'); + assert.equal(identity.email, ACCOUNTS[i]); + const settings = (await json(page, '/api/account/settings')).doc; + if (MODE === 'capture') { + expected.accounts.push({ identity, settings }); + save('browser-' + i + '.json', await context.storageState()); + } else { + // Keep session tokens and account contents out of failure output. + assert(isDeepStrictEqual(identity, expected.accounts[i].identity), 'restored identity and CSRF token'); + assert(isDeepStrictEqual(settings, expected.accounts[i].settings), 'restored account settings'); + } + pages.push(page); + pass(MODE + ': ' + (i === 0 ? 'admin' : 'user') + ' session and settings'); + } + assert.equal(expected.accounts[1].settings.keys['meshcore-time-window'], '180', 'synced fixture setting'); + const actualTelemetry = await telemetry(pages[0]); + if (MODE === 'capture') expected.telemetry = actualTelemetry; + else assert(isDeepStrictEqual(actualTelemetry, expected.telemetry), 'restored telemetry API identity and observation bytes'); + pass(MODE + ': telemetry counts and distinct observation bytes'); + + const anonymous = await (await browser.newContext()).newPage(); + // Telemetry falls back to the API-key gate; account backups use withAdmin. + for (const [route, filename, anonymousStatus] of [['/api/backup', 'telemetry.' + EXTENSION, 403], ['/api/admin/users-backup', 'accounts.' + EXTENSION, 401]]) { + assert.equal((await anonymous.request.get(BASE + route)).status(), anonymousStatus, route + ' anonymous denial'); + assert.equal((await pages[1].request.get(BASE + route)).status(), 403, route + ' non-admin denial'); + const response = await pages[0].request.get(BASE + route); + assert.equal(response.status(), 200, route + ' admin download'); + assert.match(response.headers()['content-disposition'], new RegExp('^attachment; filename="corescope-[^"]+\\.' + EXTENSION + '"$')); + assert.equal(response.headers()['content-type'], 'application/octet-stream'); + assert.equal(response.headers()['cache-control'], 'no-store'); + const body = await response.body(); + const magic = BACKEND === 'sqlite' ? 'SQLite format 3\0' : 'PGDMP'; + assert.equal(body.subarray(0, Buffer.byteLength(magic)).toString(), magic, route + ' native archive magic'); + if (MODE === 'capture') fs.writeFileSync(path.join(DIR, filename), body, { mode: 0o600, flag: 'wx' }); + pass(MODE + ': protected native ' + filename + ' download'); + } + if (MODE === 'capture') save('expected.json', expected); + } finally { + await browser.close(); + } + console.log('\n' + passed + '/' + passed + ' tests passed'); +})().catch(err => { console.error('PostgreSQL backup E2E failed: ' + err.message); process.exitCode = 1; }); diff --git a/tests/e2e/test-pr-1490-live-map-gpu-animations-e2e.js b/tests/e2e/test-pr-1490-live-map-gpu-animations-e2e.js new file mode 100644 index 000000000..fc7249bb8 --- /dev/null +++ b/tests/e2e/test-pr-1490-live-map-gpu-animations-e2e.js @@ -0,0 +1,132 @@ +#!/usr/bin/env node +/* PR #1490 — the live map's canvas animation engine. + * + * The engine draws in-flight packets on its own canvas, drains the queue + * when each animation finishes, then goes back to sleep. Three things can + * regress without any visible error: the queue never drains (the map keeps + * repainting forever), the engine never sleeps (a permanent rAF loop), or + * the fading trails pile up past the cap of 5. + * + * Also pinned here (#1514 M2): the animation canvas and the fading polylines + * live on animationsPane (z=625), not on Leaflet's default overlayPane + * (z=400), where they would be drawn underneath the markers. + * + * Ported from the @playwright/test version of this file, which was written + * against a runner this project does not install and so had never run + * (#2037). The polling the original expressed with expect.poll is done here + * with waitForFunction. CHROMIUM_REQUIRE=1 makes Chromium-launch failure a + * HARD FAIL. + */ +'use strict'; + +const { chromium } = require('playwright'); + +const BASE = process.env.BASE_URL || 'http://localhost:13581'; + +// #1514 S5: 20, not 5, so the `while (recentPaths.length > 5)` prune at +// public/live.js:4285 actually runs and the cap assertion means something. +const PACKET_COUNT = 20; +const RECENT_PATHS_CAP = 5; +// One animation steps at scaledDt/660 (live.js:4154), so 660ms at 1x. The +// @playwright/test original allowed 1500ms, but that figure was never tested: +// the file had no runner. Measured against a populated instance, 20 queued +// animations do not all finish inside it. What this asserts is that the queue +// drains at all and the engine then sleeps, not that it does so within twice +// one animation's duration, so the budget is generous on purpose. +const DRAIN_TIMEOUT_MS = 8000; + +let passes = 0, failures = 0; +function pass(msg) { console.log(` ✓ ${msg}`); passes++; } +function fail(msg) { console.error(` ✗ ${msg}`); failures++; } + +async function main() { + const requireChromium = process.env.CHROMIUM_REQUIRE === '1'; + let browser; + try { + browser = await chromium.launch({ headless: true }); + } catch (err) { + if (requireChromium) { + console.error(`HARD FAIL — Chromium unavailable: ${err.message}`); + process.exit(1); + } + console.warn(`SKIP — Chromium unavailable: ${err.message}`); + process.exit(0); + } + + const ctx = await browser.newContext({ viewport: { width: 1280, height: 900 } }); + const page = await ctx.newPage(); + await page.goto(`${BASE}/#/live`, { waitUntil: 'domcontentloaded' }); + + try { + await page.waitForSelector('#liveMap', { timeout: 15000 }); + await page.waitForFunction( + () => !!window._liveDrawAnimatedLine && !!window._liveTestSeams, + null, + { timeout: 15000 }, + ); + } catch { + fail('the live map never exposed its animation seams within 15s'); + await browser.close(); + console.log(`\ntest-pr-1490-live-map-gpu-animations-e2e: ${passes} passed, ${failures} failed`); + process.exit(1); + } + + // (1) The animation canvas is on animationsPane, not on overlayPane. + // Leaflet's own preferCanvas:true renderer also creates a canvas, on + // overlayPane, so an unqualified `canvas` selector would match the wrong one. + const canvasOnPane = await page.evaluate(() => { + const pane = document.querySelector('.leaflet-pane.leaflet-animations-pane'); + if (!pane) return { pane: false, canvases: 0 }; + return { pane: true, canvases: pane.querySelectorAll('canvas').length }; + }); + if (canvasOnPane.pane && canvasOnPane.canvases >= 1) { + pass('(1) the animation canvas is attached to animationsPane'); + } else { + fail(`(1) animationsPane ${canvasOnPane.pane ? 'has no canvas' : 'is missing'} (canvases: ${canvasOnPane.canvases})`); + } + + // (2) Firing packets queues exactly that many animations and wakes the engine. + const afterFire = await page.evaluate((count) => { + if (window._liveVcrSetMode) window._liveVcrSetMode('LIVE'); + for (let i = 0; i < count; i++) { + window._liveDrawAnimatedLine([37.4, -122.0], [37.5, -122.1], '#00ff00', null, null, '00AA', 'test-hash-' + i); + } + return { count: window._liveTestSeams.getAnimCount(), animating: window._liveTestSeams.isAnimating() }; + }, PACKET_COUNT); + + if (afterFire.count === PACKET_COUNT) pass(`(2) all ${PACKET_COUNT} animations were queued`); + else fail(`(2) queued ${afterFire.count} animations, expected ${PACKET_COUNT}`); + + if (afterFire.animating) pass('(3) the engine woke up'); + else fail('(3) the engine did not wake up, so nothing would be drawn'); + + // (4) The queue drains. + try { + await page.waitForFunction(() => window._liveTestSeams.getAnimCount() === 0, null, { timeout: DRAIN_TIMEOUT_MS, polling: 50 }); + pass(`(4) the queue drained to 0 within ${DRAIN_TIMEOUT_MS}ms`); + } catch { + const left = await page.evaluate(() => window._liveTestSeams.getAnimCount()); + fail(`(4) ${left} animation(s) still queued after ${DRAIN_TIMEOUT_MS}ms`); + } + + // (5) The engine goes back to sleep. There is one rAF tick between the + // queue emptying and renderAnimations flipping isAnimating to false, so + // poll rather than read once (#1514). + try { + await page.waitForFunction(() => window._liveTestSeams.isAnimating() === false, null, { timeout: 1000, polling: 50 }); + pass('(5) the engine went back to sleep'); + } catch { + fail('(5) isAnimating stayed true with an empty queue, so the rAF loop never stops'); + } + + // (6) The fading trails respect the cap. + const paths = await page.evaluate(() => window._liveTestSeams.getPathCount()); + if (paths <= RECENT_PATHS_CAP) pass(`(6) recentPaths held at ${paths}, within the cap of ${RECENT_PATHS_CAP}`); + else fail(`(6) recentPaths grew to ${paths}, past the cap of ${RECENT_PATHS_CAP}`); + + await browser.close(); + console.log(`\ntest-pr-1490-live-map-gpu-animations-e2e: ${passes} passed, ${failures} failed`); + process.exit(failures ? 1 : 0); +} + +main().catch((e) => { console.error(e); process.exit(1); }); diff --git a/test-rx-coverage-mobile-nav-e2e.js b/tests/e2e/test-rx-coverage-mobile-nav-e2e.js similarity index 100% rename from test-rx-coverage-mobile-nav-e2e.js rename to tests/e2e/test-rx-coverage-mobile-nav-e2e.js diff --git a/tests/e2e/test-rx-coverage-noise-e2e.js b/tests/e2e/test-rx-coverage-noise-e2e.js new file mode 100644 index 000000000..e0a02a53d --- /dev/null +++ b/tests/e2e/test-rx-coverage-noise-e2e.js @@ -0,0 +1,91 @@ +#!/usr/bin/env node +'use strict'; +// RF noise-floor layer on the Mobile RX coverage page: the Signal/Noise toggle +// exists only when clientRfSamples is on, switching draws /api/rf-noise cells +// with the noise legend, the layer is deep-linkable, and an empty answer says +// so instead of leaving a blank map. +const assert = require('assert'); +const { chromium } = require('playwright'); +const BASE = process.env.BASE_URL || 'http://localhost:3000'; + +const cell = (lon, lat, median) => ({ + type: 'Feature', + geometry: { type: 'Polygon', coordinates: [[[lon, lat], [lon + 0.01, lat], [lon + 0.01, lat + 0.01], [lon, lat + 0.01], [lon, lat]]] }, + properties: { cell: 'c' + median, count: 3, median_noise_floor: median, quietest_noise_floor: median - 2, noisiest_noise_floor: median + 2 }, +}); + +let passed = 0, failed = 0; +async function step(name, fn) { + try { await fn(); passed++; console.log(' ✓ ' + name); } + catch (e) { failed++; console.log(' ✗ ' + name + ': ' + e.message); } +} + +async function openPage(browser, { rfSamples, noiseFeatures, hash }) { + const page = await browser.newPage(); + const errors = []; + page.on('pageerror', e => errors.push(e.message)); + const noiseRequests = []; + await page.route('**/api/config/client', async route => { + const response = await route.fetch(); + await route.fulfill({ json: { ...await response.json(), clientRxCoverage: true, clientRfSamples: rfSamples } }); + }); + await page.route('**/api/config/map', route => route.fulfill({ json: { center: [51, 4], zoom: 9 } })); + await page.route('**/api/rx-leaderboard?*', route => route.fulfill({ json: { observers: [] } })); + await page.route('**/api/rx-coverage?*', route => route.fulfill({ json: { type: 'FeatureCollection', features: [] } })); + await page.route('**/api/rf-noise?*', route => { + noiseRequests.push(route.request().url()); + return route.fulfill({ json: { type: 'FeatureCollection', features: noiseFeatures, truncated: false } }); + }); + await page.goto(BASE + '/#/rx-coverage' + (hash || '')); + await page.waitForSelector('#rxMap.leaflet-container', { timeout: 15000 }); + return { page, errors, noiseRequests }; +} + +(async () => { + const browser = await chromium.launch({ headless: true, executablePath: process.env.CHROMIUM_PATH || undefined }); + console.log('\n=== RF noise layer E2E against ' + BASE + ' ==='); + try { + await step('no Signal/Noise toggle when clientRfSamples is off', async () => { + const { page, errors } = await openPage(browser, { rfSamples: false, noiseFeatures: [] }); + assert.strictEqual(await page.locator('#rxLayerBar').count(), 0, 'toggle must not render when the feature is off'); + assert.deepStrictEqual(errors, []); + await page.close(); + }); + + await step('Noise draws /api/rf-noise cells with the noise legend and deep link', async () => { + const features = [cell(4.40, 51.20, -118), cell(4.50, 51.20, -111), cell(4.60, 51.20, -100)]; + const { page, errors, noiseRequests } = await openPage(browser, { rfSamples: true, noiseFeatures: features }); + await page.waitForSelector('#rxLayerBar button[data-layer="noise"]'); + await page.click('#rxLayerBar button[data-layer="noise"]'); + await page.waitForFunction(() => document.querySelectorAll('#rxMap path.leaflet-interactive').length === 3, null, { timeout: 10000 }); + assert.ok(noiseRequests.length >= 1, 'the noise layer must fetch /api/rf-noise'); + const q = new URL(noiseRequests[noiseRequests.length - 1]).searchParams; + assert.ok(q.get('bbox') && q.get('z') && q.get('days'), 'bbox, z and days are sent: ' + q.toString()); + const legend = await page.locator('#rxLegend').innerText(); + assert.ok(/quiet/.test(legend) && /busy/.test(legend), 'legend switches to the noise tiers: ' + legend); + assert.ok(/noise floor/i.test(await page.locator('#rxSubtitle').innerText()), 'subtitle names the noise floor'); + assert.ok(/[?&]layer=noise/.test(await page.evaluate(() => location.hash)), 'layer is in the URL hash'); + const pressed = await page.locator('#rxLayerBar button[data-layer="noise"]').getAttribute('class'); + assert.ok(/active/.test(pressed || ''), 'Noise button is marked active'); + assert.ok(!(await page.locator('#rxNoiseEmpty').isVisible()), 'no empty-state message when cells exist'); + assert.deepStrictEqual(errors, []); + await page.close(); + }); + + await step('a deep link opens on the noise layer, and an empty answer is labelled', async () => { + const { page, errors, noiseRequests } = await openPage(browser, { rfSamples: true, noiseFeatures: [], hash: '?layer=noise' }); + await page.waitForFunction(() => { + const el = document.getElementById('rxNoiseEmpty'); + return el && getComputedStyle(el).display !== 'none'; + }, null, { timeout: 10000 }); + assert.ok(noiseRequests.length >= 1, 'a ?layer=noise link must fetch the noise layer without a click'); + assert.ok(/No RF samples/.test(await page.locator('#rxNoiseEmpty').innerText())); + assert.deepStrictEqual(errors, []); + await page.close(); + }); + } finally { + await browser.close(); + } + console.log(`\ntest-rx-coverage-noise-e2e: ${passed} passed, ${failed} failed`); + process.exit(failed ? 1 : 0); +})(); diff --git a/tests/e2e/test-rx-coverage-viewport-e2e.js b/tests/e2e/test-rx-coverage-viewport-e2e.js new file mode 100644 index 000000000..c334dc666 --- /dev/null +++ b/tests/e2e/test-rx-coverage-viewport-e2e.js @@ -0,0 +1,88 @@ +#!/usr/bin/env node +'use strict'; +// #2032: real Leaflet viewport initialization, persistence and observer links. +const assert = require('assert'); +const { chromium } = require('playwright'); +const BASE = process.env.BASE_URL || 'http://localhost:3000'; +(async () => { + const browser = await chromium.launch({ headless: true, executablePath: process.env.CHROMIUM_PATH || undefined }); + try { + const page = await browser.newPage(); + const errors = []; + page.on('pageerror', error => errors.push(error.message)); + await page.addInitScript(() => { + let leaflet; + Object.defineProperty(window, 'L', { + configurable: true, + get: () => leaflet, + set(value) { + leaflet = value; + value.Map.addInitHook(function () { window.__rxTestMap = this; }); + } + }); + }); + await page.route('**/api/config/client', async route => { + const response = await route.fetch(); + await route.fulfill({ json: { ...await response.json(), clientRxCoverage: true } }); + }); + await page.route('**/api/config/map', route => route.fulfill({ json: { center: [12, 34], zoom: 6 } })); + await page.route('**/api/rx-leaderboard?*', route => route.fulfill({ json: { observers: [] } })); + let worldRequests = 0; + const observerCell = { type: 'Feature', geometry: { type: 'Polygon', coordinates: [[[40, 10], [41, 10], [41, 11], [40, 11], [40, 10]]] }, properties: { count: 1, has_sig: false, nodes: [] } }; + await page.route('**/api/rx-coverage?*', route => { + if (new URL(route.request().url()).searchParams.get('bbox') === '-90,-180,90,180') worldRequests++; + return route.fulfill({ json: { type: 'FeatureCollection', features: [observerCell] } }); + }); + async function waitMap() { + await page.waitForFunction(() => window.__rxTestMap && window.__rxTestMap._loaded && document.querySelector('#rxMap.leaflet-container')); + } + async function viewport() { + return page.evaluate(() => { const m = window.__rxTestMap, c = m.getCenter(); return { lat: c.lat, lng: c.lng, zoom: m.getZoom() }; }); + } + // Leaflet keeps the centre as a pixel coordinate, so a centre read back is + // only ever accurate to the pixel it landed on. One pixel spans + // 360/(256*2^zoom) degrees of longitude: 0.022° at zoom 6, 0.0014° at + // zoom 10. The budget below used to be a flat 0.001°, which at zoom 6 + // demanded 1/20 of a pixel, so any change to the map container's size + // moved the pixel centre enough to fail an assertion about the *view*. + // For latitude the Mercator scale is this figure times cos(lat), so using + // the longitude figure is the looser bound there, deliberately. + const degPerPixel = (zoom) => 360 / (256 * Math.pow(2, zoom)); + + function assertViewport(actual, expected) { + const budget = degPerPixel(expected.zoom); + const off = (axis) => axis + ' ' + actual[axis] + ' is more than one pixel (' + + budget.toFixed(5) + '°) from ' + expected[axis] + ': ' + JSON.stringify(actual); + assert(Math.abs(actual.lat - expected.lat) < budget, off('lat')); + assert(Math.abs(actual.lng - expected.lng) < budget, off('lng')); + assert.equal(actual.zoom, expected.zoom); + } + await page.goto(BASE + '/#/rx-coverage'); await waitMap(); + assertViewport(await viewport(), { lat: 12, lng: 34, zoom: 6 }); + await page.evaluate(() => window.__rxTestMap.setView([22, 44], 10, { animate: false })); + await page.waitForFunction((budget) => Math.abs(Number(new URLSearchParams(location.hash.split('?')[1]).get('lat')) - 22) < budget, degPerPixel(10)); + assertViewport(await page.evaluate(() => JSON.parse(localStorage.getItem('rx-coverage-view'))), { lat: 22, lng: 44, zoom: 10 }); + assert.equal(await page.evaluate(() => localStorage.getItem('map-view')), null, 'coverage must not write the main map\'s saved viewport'); + await page.reload(); await waitMap(); assertViewport(await viewport(), { lat: 22, lng: 44, zoom: 10 }); + // Remove the shareable URL to independently verify this page's saved state. + await page.goto(BASE + '/#/rx-coverage'); await page.reload(); await waitMap(); + assertViewport(await viewport(), { lat: 22, lng: 44, zoom: 10 }); + await page.goto(BASE + '/#/rx-coverage?days=14&rx=abcd&lat=0&lon=0&zoom=5'); await page.reload(); await waitMap(); + await page.waitForTimeout(500); + assertViewport(await viewport(), { lat: 0, lng: 0, zoom: 5 }); + assert.equal(worldRequests, 0, 'explicit observer viewport must not auto-fit'); + await page.click('#rxDays button[data-days="30"]'); + const params = new URLSearchParams(new URL(page.url()).hash.split('?')[1]); + assert.equal(params.get('days'), '30'); assert.equal(params.get('rx'), 'abcd'); assert.equal(params.get('lat'), '0.00000'); + // Observer-only links still fit the observer's complete extent. + await page.goto(BASE + '/#/rx-coverage?rx=abcd'); await page.reload(); await waitMap(); + await page.waitForFunction(() => { + const c = window.__rxTestMap.getCenter(); + return c.lat > 10 && c.lat < 11 && c.lng > 40 && c.lng < 41; + }); + assert(worldRequests > 0, 'observer-only links must request full extent'); + if (process.env.SCREENSHOT_PATH) await page.screenshot({ path: process.env.SCREENSHOT_PATH, fullPage: true }); + assert.deepStrictEqual(errors, []); + console.log('RX coverage viewport browser regressions OK'); + } finally { await browser.close(); } +})().catch(error => { console.error(error); process.exit(1); }); diff --git a/test-show-neighbors.js b/tests/e2e/test-show-neighbors.js similarity index 100% rename from test-show-neighbors.js rename to tests/e2e/test-show-neighbors.js diff --git a/test-slideover-1056-e2e.js b/tests/e2e/test-slideover-1056-e2e.js similarity index 100% rename from test-slideover-1056-e2e.js rename to tests/e2e/test-slideover-1056-e2e.js diff --git a/test-slideover-1168-munger-e2e.js b/tests/e2e/test-slideover-1168-munger-e2e.js similarity index 100% rename from test-slideover-1168-munger-e2e.js rename to tests/e2e/test-slideover-1168-munger-e2e.js diff --git a/test-table-fluid-e2e.js b/tests/e2e/test-table-fluid-e2e.js similarity index 69% rename from test-table-fluid-e2e.js rename to tests/e2e/test-table-fluid-e2e.js index 59f0dd9de..6795d0300 100644 --- a/test-table-fluid-e2e.js +++ b/tests/e2e/test-table-fluid-e2e.js @@ -55,6 +55,52 @@ function expectHidden(pageName, w) { console.log(`\n=== #1056 fluid tables E2E against ${BASE} ===`); + await step('#2079: responsive columns preserve full-width status and spacer rows', async () => { + const ctx = await browser.newContext({ viewport: { width: 375, height: 900 } }); + try { + const page = await ctx.newPage(); + await page.goto(BASE + '/#/home', { waitUntil: 'domcontentloaded' }); + await page.waitForFunction(() => !!window.TableResponsive); + const result = await page.evaluate(() => { + const table = document.createElement('table'); + table.id = 'responsiveSpanFixture'; + table.className = 'data-table'; + table.innerHTML = 'KeptHidden' + + 'HiddenKept' + + 'ABCD' + + 'No packets found' + + 'Failed to load packets' + + '' + + 'APartial spanD'; + document.getElementById('app').appendChild(table); + window.TableResponsive.apply(table); + return { + normalHidden: Array.from(table.querySelector('[data-normal]').children) + .map(cell => getComputedStyle(cell).display === 'none'), + partialHidden: getComputedStyle(table.querySelector('[data-partial] td[colspan]')).display === 'none', + status: Array.from(table.querySelectorAll('[data-status]')).map(row => ({ + name: row.dataset.status, + visible: getComputedStyle(row.firstElementChild).display !== 'none', + height: row.firstElementChild.getBoundingClientRect().height, + })), + }; + }); + assert(JSON.stringify(result.normalHidden) === '[false,true,true,false]', + 'ordinary priority cells must still hide: ' + JSON.stringify(result)); + assert(result.partialHidden, 'a partial span must retain column-index mapping'); + for (const row of result.status) { + assert(row.visible && row.height > 0, row.name + ' full-width row must remain visible'); + } + assert(result.status.find(row => row.name === 'spacer').height >= 48, + 'virtual-scroll spacer must retain its height'); + await page.locator('#responsiveSpanFixture .col-hidden-pill').click(); + assert(await page.locator('#responsiveSpanFixture .col-hidden').count() === 0, + 'reveal must still restore ordinary and partial-span cells'); + } finally { + await ctx.close(); + } + }); + for (const vp of VIEWPORTS) { const ctx = await browser.newContext({ viewport: { width: vp.w, height: vp.h } }); const page = await ctx.newPage(); diff --git a/test-table-sort.js b/tests/e2e/test-table-sort.js similarity index 89% rename from test-table-sort.js rename to tests/e2e/test-table-sort.js index a6255b911..27144335e 100644 --- a/test-table-sort.js +++ b/tests/e2e/test-table-sort.js @@ -1,5 +1,6 @@ /* test-table-sort.js — Unit tests for TableSort utility */ 'use strict'; +const REPO_ROOT = require('path').resolve(__dirname, '..', '..'); const { JSDOM } = require('jsdom'); const fs = require('fs'); @@ -27,7 +28,7 @@ function createDOM(html) { runScripts: 'dangerously' }); // Load TableSort into this DOM - const script = fs.readFileSync(path.join(__dirname, 'public', 'table-sort.js'), 'utf8'); + const script = fs.readFileSync(path.join(REPO_ROOT, 'public', 'table-sort.js'), 'utf8'); const el = dom.window.document.createElement('script'); el.textContent = script; dom.window.document.head.appendChild(el); @@ -233,6 +234,17 @@ test('aria-sort updates when switching columns', () => { console.log('\nTableSort — visual indicator'); +// The sort indicator is a Phosphor sprite since #1648 M2: the direction is +// carried by the , not by a text glyph. +// Reading it through the href keeps this test honest if the icon set changes +// again, because it then fails rather than passing on an empty textContent. +function sortArrowDirection(arrow) { + const use = arrow && arrow.querySelector('use'); + const href = use && (use.getAttribute('href') || use.getAttribute('xlink:href')); + const m = href && href.match(/#ph-caret-(up|down)$/); + return m ? m[1] : null; +} + test('sort arrow shows on active column', () => { const html = makeTable( [{key: 'a'}], @@ -244,7 +256,11 @@ test('sort arrow shows on active column', () => { const arrow = dom.window.document.querySelector('.sort-arrow'); assert.ok(arrow, 'sort arrow should exist'); - assert.ok(arrow.textContent.includes('▲'), 'ascending should show ▲'); + // #1648 M2 replaced the up/down glyphs with Phosphor sprite refs, so the + // direction now lives in the , not in textContent. + assert.strictEqual(sortArrowDirection(arrow), 'up', 'ascending should point the caret up'); + const activeTh = dom.window.document.querySelector('th[data-sort-key="a"]'); + assert.strictEqual(activeTh.getAttribute('aria-sort'), 'ascending', 'ascending must be announced'); }); test('sort arrow changes on direction toggle', () => { @@ -259,7 +275,8 @@ test('sort arrow changes on direction toggle', () => { const th = dom.window.document.querySelector('th[data-sort-key="a"]'); th.click(); // desc const arrow = dom.window.document.querySelector('.sort-arrow'); - assert.ok(arrow.textContent.includes('▼'), 'descending should show ▼'); + assert.strictEqual(sortArrowDirection(arrow), 'down', 'descending should point the caret down'); + assert.strictEqual(th.getAttribute('aria-sort'), 'descending', 'descending must be announced'); }); console.log('\nTableSort — onSort callback'); diff --git a/test-touch-gestures-coverage-e2e.js b/tests/e2e/test-touch-gestures-coverage-e2e.js similarity index 96% rename from test-touch-gestures-coverage-e2e.js rename to tests/e2e/test-touch-gestures-coverage-e2e.js index 763d8660d..75495210a 100644 --- a/test-touch-gestures-coverage-e2e.js +++ b/tests/e2e/test-touch-gestures-coverage-e2e.js @@ -174,6 +174,10 @@ async function main() { // Stub clipboard so cov3 can observe writes without a real permission. await page.addInitScript(() => { + // #2054: mobile clamps larger saved windows to 15; keep fixture rows for 3 hours. + localStorage.setItem('meshcore-time-window', '180'); + // First-visit hints have their own suite; the drawer hint covers this swipe target. + localStorage.setItem('meshcore-gesture-hints-edge-drawer', 'seen'); window.__clipboardWrites = []; if (!navigator.clipboard) { Object.defineProperty(navigator, 'clipboard', { @@ -249,7 +253,7 @@ async function main() { hash: location.hash, overlay: !!document.querySelector('.row-action-overlay.row-action-overlay-open'), })); - const expected = `#/packets?hash=${encodeURIComponent(r2.hash)}`; + const expected = `#/packets?timeWindow=180&hash=${encodeURIComponent(r2.hash)}`; if (state.hash === expected && !state.overlay) { pass(`(cov2) filter button navigated to ${state.hash} and dismissed overlay`); } else { @@ -328,8 +332,8 @@ async function main() { await synthSwipe(page, cx - 80, cy, cx + 80, cy); await page.waitForTimeout(250); const hash = await page.evaluate(() => location.hash); - if (hash === '#/packets') pass('(cov5) LTR bottom-nav swipe on #/live navigated back to #/packets'); - else fail(`(cov5) expected #/packets, got ${hash}`); + if (hash === '#/packets?timeWindow=180') pass('(cov5) LTR bottom-nav swipe returned to packets with the saved window'); + else fail(`(cov5) expected #/packets?timeWindow=180, got ${hash}`); } // ── (cov6) bottom-nav boundary — LTR swipe on first tab (#/home) no-op ── @@ -363,6 +367,7 @@ async function main() { { const ctxD = await browser.newContext({ viewport: { width: 1200, height: 900 }, hasTouch: true }); const pD = await ctxD.newPage(); + await pD.addInitScript(() => localStorage.setItem('meshcore-time-window', '1440')); pD.setDefaultTimeout(15000); pD.on('pageerror', (e) => console.error('[pageerror-desktop]', e.message)); await pD.goto(`${BASE}/#/packets`, { waitUntil: 'domcontentloaded' }); @@ -397,6 +402,10 @@ async function main() { viewport: { width: 375, height: 812 }, hasTouch: true, isMobile: true, }); const pP = await ctxP.newPage(); + await pP.addInitScript(() => { + localStorage.setItem('meshcore-time-window', '180'); + localStorage.setItem('meshcore-gesture-hints-edge-drawer', 'seen'); + }); pP.setDefaultTimeout(15000); pP.on('pageerror', (e) => console.error('[pageerror-phone2]', e.message)); await pP.goto(`${BASE}/#/packets`, { waitUntil: 'domcontentloaded' }); diff --git a/test-touch-targets.js b/tests/e2e/test-touch-targets.js similarity index 86% rename from test-touch-targets.js rename to tests/e2e/test-touch-targets.js index c509c1f37..de551e302 100644 --- a/test-touch-targets.js +++ b/tests/e2e/test-touch-targets.js @@ -17,18 +17,33 @@ * in CSS (:focus / :focus-within rule in the Touch Targets section). */ 'use strict'; +const REPO_ROOT = require('path').resolve(__dirname, '..', '..'); const fs = require('fs'); const path = require('path'); const assert = require('assert'); const { chromium, devices } = require('playwright'); -const REPO = __dirname; +const REPO = REPO_ROOT; const CSS = fs.readFileSync(path.join(REPO, 'public/style.css'), 'utf8'); -// Selectors we claim to make 48x48. Each entry: [selector, tag, classes, -// optional inner-html]. Tag matters because some rules are scoped to -// `button.ch-item` and some only apply to specific input[type=...]. +// All listed button surfaces use the shared 48px house minimum (#2052). +const DEFAULT_MIN = 48; + +// Each entry: [selector, tag, classes, optional inner-html]. +// Tag matters because some rules are scoped to `button.ch-item` and some only +// apply to specific input[type=...]. +// +// Not listed, and why: +// .compare-btn the Compare CTA was removed in #1646 (see style.css) +// .ch-back-btn display:none outside the mobile channels layout, so a +// standalone element in this harness measures 0x0 +// .filter-toggle-btn hidden on mobile since #1461 (style.css:1161, +// display:none !important); the control actually shown is +// the navbar mirror, which mobile-page-actions.js:70 +// builds with class "nav-btn filter-toggle-btn-mirror +// mpa-btn-pill", so the .nav-btn entry above already +// measures it const BUTTON_SELECTORS = [ ['.btn', 'button', 'btn'], ['.btn-icon', 'button', 'btn-icon'], @@ -42,14 +57,11 @@ const BUTTON_SELECTORS = [ ['button.ch-item', 'button', 'ch-item'], ['.btn-link', 'button', 'btn-link'], ['.col-toggle-btn', 'button', 'col-toggle-btn'], - ['.filter-toggle-btn', 'button', 'filter-toggle-btn'], ['.ch-add-channel-btn', 'button', 'ch-add-channel-btn'], - ['.ch-back-btn', 'button', 'ch-back-btn'], ['.ch-modal-btn-secondary','button', 'ch-modal-btn-secondary'], ['.ch-scroll-btn', 'button', 'ch-scroll-btn'], ['.chooser-btn', 'button', 'chooser-btn'], ['.clock-filter-btn', 'button', 'clock-filter-btn'], - ['.compare-btn', 'button', 'compare-btn'], ['.copy-link-btn', 'button', 'copy-link-btn'], ['.alab-btn', 'button', 'alab-btn'], ]; @@ -111,9 +123,9 @@ async function run() { } catch (err) { // Allow the test to be skipped on hosts where Chromium cannot launch // (e.g. some musl-libc dev boxes). CI uses standard glibc Ubuntu runners - // where this path is never taken. Set TOUCH_TARGETS_REQUIRE=1 to force + // where this path is never taken. Set TOUCH_TARGETS_REQUIRE=1 or CHROMIUM_REQUIRE=1 to force // a hard failure even when Chromium is unavailable. - if (process.env.TOUCH_TARGETS_REQUIRE === '1') throw err; + if (process.env.TOUCH_TARGETS_REQUIRE === '1' || process.env.CHROMIUM_REQUIRE === '1') throw err; console.log(`test-touch-targets.js: SKIP (Chromium unavailable: ${err.message.split('\n')[0]})`); process.exit(0); } @@ -149,11 +161,12 @@ async function run() { const cs = getComputedStyle(el); return { w: r.width, h: r.height, mh: cs.minHeight, mw: cs.minWidth }; }); - const okH = dim.h >= 48; - const okW = dim.w >= 48; - record(`${selector}: rendered ${dim.w.toFixed(1)}x${dim.h.toFixed(1)} (min ${dim.mw}/${dim.mh})`, + const min = DEFAULT_MIN; + const okH = dim.h >= min; + const okW = dim.w >= min; + record(`${selector}: rendered ${dim.w.toFixed(1)}x${dim.h.toFixed(1)} (min ${dim.mw}/${dim.mh}, required ${min})`, okH && okW, - `expected >=48x48, got ${dim.w}x${dim.h}`); + `expected >=${min}x${min}, got ${dim.w}x${dim.h}`); } // --- Form controls: rendered height must be at least 48 CSS px. diff --git a/tests/e2e/test-user-management-e2e.js b/tests/e2e/test-user-management-e2e.js new file mode 100644 index 000000000..a9b6985ec --- /dev/null +++ b/tests/e2e/test-user-management-e2e.js @@ -0,0 +1,448 @@ +/** + * E2E: optional user management (docs/specs/2026-10-06-user-management-design.md). + * BASE_URL server with userManagement on + fake mailer (-tags e2etest build) + * BASE_URL_OFF the regular fixture server (feature off) + * + * Local run: prepare/freshen a disposable copy of the tracked SQLite fixture, + * apply the CI synthetic seeds and migrate-fixture-hashes.go, then import it + * with corescope-migrate -offline -from-sqlite . Use owner URLs only for + * migration; grant a read-only telemetry role and a separate account writer. + * (cd cmd/server && go build -o ../../corescope-server . && go build -tags e2etest -o ../../corescope-server-e2e .) + * # config.json for the on-server (in $CFGDIR): port 13582, userManagement {enabled: true, + * # adminEmails ["admin@e2e.test"], publicBaseUrl "http://localhost:13582", + * # mail {provider "fake", fromEmail "noreply@e2e.test"}, channelProposals {enabled: true}, notifications {enabled: true}} + * # Set CORESCOPE_READER_DATABASE_URL for both processes and + * # CORESCOPE_USERS_DATABASE_URL for the on-server only (fresh account DB). + * corescope-server -port 13581 -state-dir "$TMP/off-state" -public public & + * (cd "$CFGDIR" && corescope-server-e2e -config-dir . -port 13582 -state-dir "$TMP/on-state" -public /public) & + * BASE_URL=http://localhost:13582 BASE_URL_OFF=http://localhost:13581 node tests/e2e/test-user-management-e2e.js + * Set CHROMIUM_PATH to a Chrome/Chromium binary if Playwright's own is not installed. + */ +'use strict'; +const { chromium } = require('playwright'); +const { AxeBuilder } = require('@axe-core/playwright'); +const BASE = process.env.BASE_URL || 'http://localhost:13582'; +const BASE_OFF = process.env.BASE_URL_OFF || 'http://localhost:13581'; +const PW = 'correct horse battery'; + +let passed = 0, failed = 0; +async function step(name, fn) { + try { await fn(); passed++; console.log(' ✓ ' + name); } + catch (e) { failed++; console.error(' ✗ ' + name + ': ' + e.message); } +} +function assert(c, m) { if (!c) throw new Error(m || 'assertion failed'); } + +// Resolves once auth.js has finished its first /api/auth/me round trip. +async function authReady(page) { + await page.waitForFunction(() => !!window.CSAuth); + await page.evaluate(() => window.CSAuth.ready()); +} + +// Resolves once the full node page has rendered its paths section, which +// needs a network round trip started after CSNotify.mount ran: a positive +// signal that the page settled before asserting that something is absent. +async function nodePageRendered(page) { + await page.waitForFunction(() => { + const el = document.getElementById('fullPathsContent'); + return !!el && !el.querySelector('.spinner'); + }); +} + +async function lastMailToken(page) { + const r = await page.request.get(BASE + '/__e2e/last-mail'); + assert(r.ok(), 'last-mail HTTP ' + r.status()); + const m = await r.json(); + const sm = /token=([A-Za-z0-9_%-]+)/.exec(m.text); + assert(sm, 'no token in mail text: ' + m.text); + return { to: m.to, token: decodeURIComponent(sm[1]) }; +} + +async function registerAndActivate(page, email, name) { + await page.goto(BASE + '/#/account/register', { waitUntil: 'domcontentloaded' }); + await page.waitForSelector('#registerForm'); + await page.fill('#regEmail', email); + await page.fill('#regName', name); + await page.fill('#regPassword', PW); + await page.click('#registerForm button[type="submit"]'); + await page.waitForSelector('#mailSentHeading'); + const { to, token } = await lastMailToken(page); + assert(to === email, 'activation mail went to ' + to); + await page.goto(BASE + '/#/account/activate?token=' + encodeURIComponent(token)); + await page.waitForSelector('#activateForm'); + await page.fill('#actPassword', PW); + await page.click('#activateForm button[type="submit"]'); + await page.waitForSelector('#accountToggle .nav-account-label:has-text("' + name + '")'); +} + +// axeClean fails on serious or critical WCAG 2 A/AA violations inside sel. +async function axeClean(pg, sel) { + // A finite animation in flight (the 150 ms .page-enter fade) blends colours + // and fails color-contrast at random; wait until those have finished. + await pg.waitForFunction(() => document.getAnimations().every((a) => + a.playState !== 'running' || !a.effect || a.effect.getComputedTiming().endTime === Infinity)); + const res = await new AxeBuilder({ page: pg }).include(sel).withTags(['wcag2a', 'wcag2aa']).analyze(); + const bad = res.violations.filter((v) => v.impact === 'serious' || v.impact === 'critical'); + assert(bad.length === 0, sel + ': ' + bad.map((v) => v.id + ' ' + v.nodes.map((n) => n.target.join(' ') + ' ' + ((n.any[0] || {}).message || '')).join(' | ')).join(', ')); +} + +// The account's synced keys, read through the page's own session. +async function accountKeys(pg) { + return pg.evaluate(() => window.CSAuth.request('GET', '/api/account/settings').then((r) => (r.data.doc && r.data.doc.keys) || {})); +} + +async function until(fn, label) { + const end = Date.now() + 8000; + for (;;) { + if (await fn()) return; + if (Date.now() > end) throw new Error('timed out: ' + label); + await new Promise((r) => setTimeout(r, 200)); + } +} + +(async () => { + const browser = await chromium.launch({ + headless: true, + executablePath: process.env.CHROMIUM_PATH || undefined, + args: ['--no-sandbox', '--disable-gpu', '--disable-dev-shm-usage'], + }); + console.log(`\n=== user management E2E against ${BASE} (off: ${BASE_OFF}) ===`); + + const off = await (await browser.newContext()).newPage(); + off.setDefaultTimeout(8000); + await step('feature off: no account control and no auth API', async () => { + await off.goto(BASE_OFF + '/', { waitUntil: 'domcontentloaded' }); + await authReady(off); + assert(await off.locator('#accountToggle').count() === 0, 'account control rendered while off'); + assert(await off.evaluate(() => !window.CSAuth.isEnabled()), 'CSAuth enabled while off'); + // Unknown /api paths fall through to the SPA page (200 HTML): only JSON with a csrfToken would be a leak. + const r = await off.request.get(BASE_OFF + '/api/auth/me'); + let body = null; + try { body = await r.json(); } catch (_) { /* HTML, expected */ } + assert(!(body && body.csrfToken), '/api/auth/me answered a session while off'); + }); + + const admin = await (await browser.newContext()).newPage(); + admin.setDefaultTimeout(8000); + admin.on('dialog', (d) => d.accept()); + admin.on('pageerror', (e) => console.error('[pageerror admin]', e.message)); + await step('config admin registers, activates with a password, sees the Admin entry', async () => { + await registerAndActivate(admin, 'admin@e2e.test', 'E2E Admin'); + await admin.click('#accountToggle'); + assert(await admin.locator('#accountMenu a[href="#/admin"]').isVisible(), 'no Admin menu entry'); + }); + + const user = await (await browser.newContext()).newPage(); + user.setDefaultTimeout(8000); + user.on('pageerror', (e) => console.error('[pageerror user]', e.message)); + await step('second user registers and activates; no Admin entry for a non-admin', async () => { + await registerAndActivate(user, 'user@e2e.test', 'E2E User'); + await user.click('#accountToggle'); + assert(await user.locator('#accountMenu').isVisible(), 'account menu did not open'); + assert(await user.locator('#accountMenu a[href="#/admin"]').count() === 0, 'non-admin sees Admin'); + }); + + await step('user logs out from the account page (phone width) and logs in again', async () => { + // At phone width the header control is hidden: the page button is the only way out. + await user.setViewportSize({ width: 375, height: 800 }); + await user.goto(BASE + '/#/account'); + await user.waitForSelector('#profileForm'); + await user.click('#accountPageLogout'); + // Settings sync is active for a logged-in user: logout asks keep or remove. + await user.click('.cs-dialog [data-choice="keep"]'); + await user.waitForSelector('#loginForm'); + assert(await user.evaluate(() => location.hash) === '#/account/login', 'not on the login view after logout'); + assert(await user.evaluate(() => window.CS_USER === null), 'client still holds the user'); + await user.setViewportSize({ width: 1280, height: 720 }); + await user.waitForSelector('#accountToggle .nav-account-label:has-text("Log in")'); + await user.fill('#loginEmail', 'user@e2e.test'); + await user.fill('#loginPassword', PW); + await user.click('#loginForm button[type="submit"]'); + await user.waitForSelector('#profileForm'); + await user.waitForSelector('#accountToggle .nav-account-label:has-text("E2E User")'); + }); + + // Settings sync: two browser contexts are two devices on one account. + const SYNC_FAV = 'e2e5e7c0000000000000000000000000000000000000000000000000000000a1'; + const d1 = await (await browser.newContext()).newPage(); + const d2 = await (await browser.newContext()).newPage(); + for (const [pg, tag] of [[d1, 'd1'], [d2, 'd2']]) { + pg.setDefaultTimeout(8000); + pg.on('pageerror', (e) => console.error('[pageerror ' + tag + ']', e.message)); + } + + await step('settings sync: device 1 saves a packet time window and a favorite to the account', async () => { + await registerAndActivate(d1, 'sync@e2e.test', 'E2E Sync'); + await d1.goto(BASE + '/#/packets'); + await d1.waitForSelector('#fTimeWindow'); + await d1.selectOption('#fTimeWindow', '180'); + // No favorite star without node rows in view: write the key as nodes.js does. + await d1.evaluate((pk) => localStorage.setItem('meshcore-favorites', JSON.stringify([pk])), SYNC_FAV); + await until(async () => { + const k = await accountKeys(d1); + return k['meshcore-time-window'] === '180' && (k['meshcore-favorites'] || '').includes(SYNC_FAV); + }, 'account holds the time window and the favorite'); + }); + + await step('settings sync: device 2 logs in and gets both; its channel key stays local', async () => { + await d2.goto(BASE + '/#/account/login', { waitUntil: 'domcontentloaded' }); + await d2.waitForSelector('#loginForm'); + await d2.evaluate(() => localStorage.setItem('corescope_channel_keys', JSON.stringify({ '#e2e': '00112233445566778899aabbccddeeff' }))); + await d2.fill('#loginEmail', 'sync@e2e.test'); + await d2.fill('#loginPassword', PW); + await d2.click('#loginForm button[type="submit"]'); + await d2.waitForSelector('#profileForm'); + await d2.waitForFunction((pk) => (localStorage.getItem('meshcore-favorites') || '').includes(pk) && + localStorage.getItem('meshcore-time-window') === '180', SYNC_FAV); + // Check after a full round trip from device 2 (pull, merge, push), not + // just after its first pull. + await d2.evaluate(() => window.CSSettingsSync.syncNow()); + const k = await accountKeys(d2); + assert(!('corescope_channel_keys' in k), 'channel key reached the account'); + }); + + await step('settings sync: a favorite removed on device 1 is gone on device 2', async () => { + await d1.evaluate(() => localStorage.setItem('meshcore-favorites', '[]')); + await until(async () => !((await accountKeys(d1))['meshcore-favorites'] || '').includes(SYNC_FAV), 'removal reached the account'); + await d2.evaluate(() => window.CSSettingsSync.syncNow()); + await d2.waitForFunction((pk) => !(localStorage.getItem('meshcore-favorites') || '').includes(pk), SYNC_FAV); + }); + + await step('settings sync: axe on the section and the logout dialog; Remove keeps the channel key', async () => { + await d2.waitForSelector('#syncStatus'); + await axeClean(d2, '#syncSection'); + await d2.click('#accountPageLogout'); + await d2.waitForSelector('.cs-dialog'); + assert(await d2.evaluate(() => document.activeElement && document.activeElement.getAttribute('data-choice') === 'keep'), 'Keep is not focused'); + await axeClean(d2, '.cs-dialog'); + await d2.click('.cs-dialog [data-choice="remove"]'); + await d2.waitForSelector('#loginForm'); + const left = await d2.evaluate(() => ({ + fav: localStorage.getItem('meshcore-favorites'), tw: localStorage.getItem('meshcore-time-window'), + base: localStorage.getItem('cs-settings-sync-base'), ch: localStorage.getItem('corescope_channel_keys'), + })); + assert(left.fav === null && left.tw === null && left.base === null, 'synced keys left: ' + JSON.stringify(left)); + assert(left.ch && left.ch.includes('#e2e'), 'channel key removed'); + }); + + await step('admin disables the user; the live session is logged out without a reload', async () => { + await admin.goto(BASE + '/#/admin/users'); + const row = admin.locator('tr[data-email="user@e2e.test"]'); + await row.waitFor(); + await row.locator('button[data-act="disable"]').click(); + await admin.waitForSelector('tr[data-email="user@e2e.test"] .um-status-disabled'); + // No reload: leave and re-enter the account page; its sessions call answers 401. + await user.goto(BASE + '/#/home'); + await user.goto(BASE + '/#/account'); + // The 60 s settings pull may log the user out first, so assert the end state only. + await user.waitForSelector('#accountToggle .nav-account-label:has-text("Log in")'); + assert(await user.evaluate(() => window.CS_USER === null), 'CS_USER is not null'); + }); + + await step('admin overview: user figures and a failed-login burst under Needs attention, followed to the audit tab', async () => { + // Five wrong passwords for an existing account (user@e2e.test: 1 earlier login + 5 stays under the 10-per-address limit). + const guess = await (await browser.newContext()).newPage(); + guess.setDefaultTimeout(8000); + await guess.goto(BASE + '/#/home', { waitUntil: 'domcontentloaded' }); + await authReady(guess); + const codes = await guess.evaluate(async () => { + const out = []; + for (let i = 0; i < 5; i++) { + const r = await fetch('/api/auth/login', { method: 'POST', headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ email: 'user@e2e.test', password: 'not the password' }) }); + out.push(r.status); + } + return out; + }); + assert(codes.every((c) => c === 401), 'failed logins answered ' + codes.join(',')); + await guess.context().close(); + await admin.goto(BASE + '/#/admin'); + await admin.waitForSelector('#aoUsers [data-stat="total"]'); + // admin, user and sync are the three accounts at this point. + assert((await admin.textContent('#aoUsers [data-stat="total"]')) === '3', 'total accounts'); + const link = admin.locator('#adminAttention a[href*="action=user.login.failed"]'); + await link.waitFor(); + await link.click(); + await admin.waitForSelector('#auditBody tr[data-action="user.login.failed"]'); + assert((await admin.evaluate(() => location.hash)).indexOf('tab=audit') !== -1, 'not on the audit tab'); + }); + + await step('the old #/admin/users link lands on the Users tab with its filter', async () => { + await admin.goto(BASE + '/#/admin/users?status=disabled'); + await admin.waitForSelector('tr[data-email="user@e2e.test"]'); + assert((await admin.evaluate(() => location.hash)) === '#/admin?tab=users&status=disabled', 'hash not rewritten'); + }); + + await step('axe: no serious or critical violations on the three admin tabs', async () => { + for (const [route, sel] of [['/#/admin', '#aoUsers [data-stat="total"]'], ['/#/admin?tab=users', '.um-table'], ['/#/admin?tab=audit', '#auditBody tr']]) { + await admin.goto(BASE + route); + await admin.waitForSelector(sel); + await admin.mouse.move(0, 0); // parked to avoid hover noise; status text on the hover tint is AA (contrast unit test) + await admin.waitForTimeout(1500); + await axeClean(admin, '#app'); + } + }); + + await step('axe: no serious or critical violations on the new views', async () => { + for (const [pg, route, sel] of [[user, '/#/account/login', '#loginForm'], [admin, '/#/admin/users', '.um-table']]) { + await pg.goto(BASE + route); + await pg.waitForSelector(sel); + await authReady(pg); + await pg.mouse.move(0, 0); + await pg.waitForTimeout(1500); + await axeClean(pg, '#app'); + } + }); + + const proposer = await (await browser.newContext()).newPage(); + proposer.setDefaultTimeout(8000); + proposer.on('pageerror', (e) => console.error('[pageerror proposer]', e.message)); + await step('proposals: no Propose button in the Add Channel dialog while logged out', async () => { + await proposer.goto(BASE + '/#/channels', { waitUntil: 'domcontentloaded' }); + await authReady(proposer); + await proposer.click('#chAddChannelBtn'); + await proposer.waitForSelector('#chHashtagName'); + assert(await proposer.locator('#chHashtagProposeBtn').isHidden(), 'Propose button visible while logged out'); + }); + + await step('proposals: a user proposes #e2e-test from the Add Channel dialog', async () => { + await registerAndActivate(proposer, 'proposer@e2e.test', 'E2E Proposer'); + await proposer.goto(BASE + '/#/channels'); + await proposer.click('#chAddChannelBtn'); + await proposer.waitForSelector('#chHashtagProposeBtn:not([hidden])'); + await proposer.fill('#chHashtagName', 'public'); + await until(async () => (await proposer.textContent('#chHashtagProposeMsg')).indexOf('built-in') !== -1, 'live validation'); + await proposer.fill('#chHashtagName', 'e2e-test'); + await proposer.click('#chHashtagProposeBtn'); + await until(async () => (await proposer.textContent('#chHashtagProposeMsg')) === 'Proposed #e2e-test, an admin will review it.', 'propose result'); + }); + + await step('proposals: My proposals on the account page shows it waiting', async () => { + await proposer.goto(BASE + '/#/account'); + await proposer.waitForSelector('#propList li[data-subject="#e2e-test"]'); + assert((await proposer.textContent('#propList')).indexOf('Waiting for review') !== -1, 'status not shown'); + }); + + await step('proposals: the admin approves it on the Proposals tab after the warning', async () => { + const warnings = []; + const onDialog = (d) => warnings.push(d.message()); + admin.on('dialog', onDialog); + try { + await admin.goto(BASE + '/#/admin?tab=proposals&status=pending'); + await admin.waitForSelector('#propAdminBody tr[data-subject="#e2e-test"]'); + await admin.click('#propAdminBody tr[data-subject="#e2e-test"] button[data-act="approve"]'); + await until(async () => (await admin.textContent('#propAdminMsg')) === '#e2e-test: approved', 'approve result'); + } finally { + admin.off('dialog', onDialog); + } + assert(warnings.some((m) => /every visitor/.test(m)), 'no readability warning: ' + warnings.join(' | ')); + }); + + await step('proposals: the approved channel, without traffic, is listed on the Channels page with its marker', async () => { + const body = await (await proposer.request.get(BASE + '/api/channels')).json(); + assert(Array.isArray(body.approvedChannels) && body.approvedChannels.indexOf('#e2e-test') !== -1, JSON.stringify(body.approvedChannels)); + assert(!(body.channels || []).some((c) => c && c.name === '#e2e-test'), '#e2e-test has traffic in the fixture'); + await proposer.goto(BASE + '/#/channels'); + // The client caches /api/channels for 15 s: reload to fetch the list after the approval. + await proposer.reload(); + await proposer.waitForSelector('#chList .ch-item[data-hash="#e2e-test"] .ch-approved-badge'); + }); + + await step('proposals: revoke removes it from the approved list', async () => { + await admin.goto(BASE + '/#/admin?tab=proposals&status=approved'); + await admin.waitForSelector('#propAdminBody tr[data-subject="#e2e-test"]'); + await admin.click('#propAdminBody tr[data-subject="#e2e-test"] button[data-act="revoke"]'); + await until(async () => (await admin.textContent('#propAdminMsg')) === '#e2e-test: revoked', 'revoke result'); + const body = await (await admin.request.get(BASE + '/api/channels')).json(); + assert(Array.isArray(body.approvedChannels) && body.approvedChannels.indexOf('#e2e-test') === -1, JSON.stringify(body.approvedChannels)); + }); + + await step('axe: no serious or critical violations on the Proposals tab', async () => { + await admin.goto(BASE + '/#/admin?tab=proposals&status=all'); + await admin.waitForSelector('#propAdminBody tr[data-subject="#e2e-test"]'); + await admin.mouse.move(0, 0); + await admin.waitForTimeout(1500); + await axeClean(admin, '#app'); + }); + + await step('feature off: /api/channels carries no approvedChannels', async () => { + const body = await (await off.request.get(BASE_OFF + '/api/channels')).json(); + assert(!('approvedChannels' in body), 'approvedChannels present while off'); + }); + + let watchKey = null; + await step('notifications: no Notify me toggle on a node page while logged out', async () => { + const body = await (await proposer.request.get(BASE + '/api/nodes?limit=1')).json(); + watchKey = body.nodes[0].public_key; + const anon = await (await browser.newContext()).newPage(); + anon.setDefaultTimeout(8000); + await anon.goto(BASE + '/#/nodes/' + encodeURIComponent(watchKey)); + await anon.waitForSelector('#nodeNotifySlotFull', { state: 'attached' }); + await authReady(anon); + await nodePageRendered(anon); + assert((await anon.locator('[data-notify-toggle]').count()) === 0, 'toggle shown while logged out'); + await anon.context().close(); + }); + + await step('notifications: a user turns on Notify me on a node page', async () => { + await proposer.goto(BASE + '/#/nodes/' + encodeURIComponent(watchKey)); + await proposer.waitForSelector('#nodeNotifySlotFull [data-notify-toggle][aria-pressed="false"]'); + await proposer.click('#nodeNotifySlotFull [data-notify-toggle]'); + await proposer.waitForSelector('#nodeNotifySlotFull [data-notify-toggle][aria-pressed="true"]'); + }); + + await step('notifications: the #/account?section=notifications deep link shows the watched node', async () => { + await proposer.goto(BASE + '/#/account?section=notifications'); + await proposer.waitForSelector('#notifySection li[data-pubkey="' + watchKey + '"]'); + assert(await proposer.isChecked('#notifyEnabled'), 'notifications are not on by default'); + await until(() => proposer.evaluate(() => { + const r = document.getElementById('notifications').getBoundingClientRect(); + return r.top >= -1 && r.top < window.innerHeight; + }), 'deep link scrolls to the Notifications heading'); + await axeClean(proposer, '#notifySection'); + }); + + await step('notifications: the unsubscribe link turns notifications off', async () => { + const r = await proposer.request.get(BASE + '/__e2e/unsubscribe-link?email=' + encodeURIComponent('proposer@e2e.test')); + assert(r.ok(), 'unsubscribe-link HTTP ' + r.status()); + const { link } = await r.json(); + const anon = await (await browser.newContext()).newPage(); + anon.setDefaultTimeout(8000); + await anon.goto(link); + await anon.waitForSelector('#unsubBtn:not([disabled])'); + await axeClean(anon, '#app'); + await anon.click('#unsubBtn'); + await until(async () => (await anon.textContent('#accountMsg')).indexOf('Notifications are off') !== -1, 'unsubscribe result'); + await anon.context().close(); + await proposer.goto(BASE + '/#/account?section=notifications'); + await proposer.reload(); + await proposer.waitForSelector('#notifyEnabled'); + assert(!(await proposer.isChecked('#notifyEnabled')), 'still on after the unsubscribe link'); + }); + + await step('data export: the account page links to the export, which returns the user\'s own data', async () => { + await proposer.goto(BASE + '/#/account'); + await proposer.waitForSelector('#accountExport'); + assert(await proposer.getAttribute('#accountExport', 'href') === '/api/account/export', 'export link href'); + const r = await proposer.request.get(BASE + '/api/account/export'); + assert(r.ok(), 'export HTTP ' + r.status()); + const cd = r.headers()['content-disposition'] || ''; + assert(/^attachment; filename="corescope-account-\d{4}-\d{2}-\d{2}\.json"$/.test(cd), 'content-disposition ' + cd); + const body = await r.json(); + assert(body.formatVersion === 1, 'formatVersion ' + body.formatVersion); + assert(body.profile && body.profile.email === 'proposer@e2e.test', 'export profile ' + JSON.stringify(body.profile)); + }); + + await step('feature off: no userManagement block, so no notifications flag, and no toggle on the node page', async () => { + const body = await (await off.request.get(BASE_OFF + '/api/config/client')).json(); + assert(!body.userManagement, 'userManagement present while off'); + await off.goto(BASE_OFF + '/#/nodes/' + encodeURIComponent(watchKey)); + await off.waitForSelector('#nodeNotifySlotFull', { state: 'attached' }); + await authReady(off); + await nodePageRendered(off); + assert((await off.locator('[data-notify-toggle]').count()) === 0, 'toggle shown while the feature is off'); + }); + + await browser.close(); + console.log('\n' + passed + '/' + (passed + failed) + ' tests passed'); + process.exit(failed > 0 ? 1 : 0); +})(); diff --git a/test-1108-region-hide-nodes.js b/tests/unit/test-1108-region-hide-nodes.js similarity index 96% rename from test-1108-region-hide-nodes.js rename to tests/unit/test-1108-region-hide-nodes.js index c693bfee9..2a4b8d52c 100644 --- a/test-1108-region-hide-nodes.js +++ b/tests/unit/test-1108-region-hide-nodes.js @@ -10,6 +10,7 @@ * (no DOM, no fetch). Mirrors the pattern from test-area-filter.js. */ 'use strict'; +const REPO_ROOT = require('path').resolve(__dirname, '..', '..'); const vm = require('vm'); const fs = require('fs'); const assert = require('assert'); @@ -37,7 +38,7 @@ function buildCtx(initialStorage) { }; ctx.window = ctx; vm.createContext(ctx); - const src = fs.readFileSync(__dirname + '/public/region-filter.js', 'utf8'); + const src = fs.readFileSync(REPO_ROOT + '/public/region-filter.js', 'utf8'); vm.runInContext(src, ctx); return ctx; } diff --git a/test-1659-analytics-warmup.js b/tests/unit/test-1659-analytics-warmup.js similarity index 100% rename from test-1659-analytics-warmup.js rename to tests/unit/test-1659-analytics-warmup.js diff --git a/test-a11y-1705-subpath-hop-prefix-e2e.js b/tests/unit/test-a11y-1705-subpath-hop-prefix-e2e.js similarity index 98% rename from test-a11y-1705-subpath-hop-prefix-e2e.js rename to tests/unit/test-a11y-1705-subpath-hop-prefix-e2e.js index 98dfbefae..959ef6c45 100644 --- a/test-a11y-1705-subpath-hop-prefix-e2e.js +++ b/tests/unit/test-a11y-1705-subpath-hop-prefix-e2e.js @@ -45,11 +45,12 @@ * serves from). */ 'use strict'; +const REPO_ROOT = require('path').resolve(__dirname, '..', '..'); const fs = require('fs'); const path = require('path'); -const STYLE_CSS_PATH = process.env.STYLE_CSS_PATH || path.join(__dirname, 'public', 'style.css'); +const STYLE_CSS_PATH = process.env.STYLE_CSS_PATH || path.join(REPO_ROOT, 'public', 'style.css'); const THEMES = ['dark', 'light']; // -------------------- Pure helpers (unit-testable) -------------------- diff --git a/test-a11y-1715-dark-role-swatches.js b/tests/unit/test-a11y-1715-dark-role-swatches.js similarity index 98% rename from test-a11y-1715-dark-role-swatches.js rename to tests/unit/test-a11y-1715-dark-role-swatches.js index e60b32fac..0b9ed1820 100644 --- a/test-a11y-1715-dark-role-swatches.js +++ b/tests/unit/test-a11y-1715-dark-role-swatches.js @@ -32,12 +32,13 @@ * Usage: node test-a11y-1715-dark-role-swatches.js */ 'use strict'; +const REPO_ROOT = require('path').resolve(__dirname, '..', '..'); const fs = require('fs'); const path = require('path'); const assert = require('assert'); -const ROOT = __dirname; +const ROOT = REPO_ROOT; const STYLE_CSS = process.env.STYLE_CSS_PATH || path.join(ROOT, 'public', 'style.css'); const ROLES = ['repeater', 'companion', 'room', 'sensor', 'observer']; diff --git a/test-a11y-1716-rf-range-btn-active.js b/tests/unit/test-a11y-1716-rf-range-btn-active.js similarity index 98% rename from test-a11y-1716-rf-range-btn-active.js rename to tests/unit/test-a11y-1716-rf-range-btn-active.js index db4b9afd2..9cbbe6413 100644 --- a/test-a11y-1716-rf-range-btn-active.js +++ b/tests/unit/test-a11y-1716-rf-range-btn-active.js @@ -31,11 +31,12 @@ * Usage: node test-a11y-1716-rf-range-btn-active.js */ 'use strict'; +const REPO_ROOT = require('path').resolve(__dirname, '..', '..'); const fs = require('fs'); const path = require('path'); -const ROOT = __dirname; +const ROOT = REPO_ROOT; const STYLE_CSS = process.env.STYLE_CSS_PATH || path.join(ROOT, 'public', 'style.css'); // ---- WCAG helpers (copied small, no shared import) ---- diff --git a/test-a11y-1719-contrast-root-causes-e2e.js b/tests/unit/test-a11y-1719-contrast-root-causes-e2e.js similarity index 99% rename from test-a11y-1719-contrast-root-causes-e2e.js rename to tests/unit/test-a11y-1719-contrast-root-causes-e2e.js index d112555ee..8517fb9db 100644 --- a/test-a11y-1719-contrast-root-causes-e2e.js +++ b/tests/unit/test-a11y-1719-contrast-root-causes-e2e.js @@ -37,11 +37,12 @@ * Usage: node test-a11y-1719-contrast-root-causes-e2e.js */ 'use strict'; +const REPO_ROOT = require('path').resolve(__dirname, '..', '..'); const fs = require('fs'); const path = require('path'); -const ROOT = __dirname; +const ROOT = REPO_ROOT; const STYLE_CSS = process.env.STYLE_CSS_PATH || path.join(ROOT, 'public', 'style.css'); const CUSTOMIZE_JS = path.join(ROOT, 'public', 'customize.js'); diff --git a/tests/unit/test-a11y-1996-scope-audit-chips.js b/tests/unit/test-a11y-1996-scope-audit-chips.js new file mode 100644 index 000000000..e26a415f4 --- /dev/null +++ b/tests/unit/test-a11y-1996-scope-audit-chips.js @@ -0,0 +1,146 @@ +/* test-a11y-1996-scope-audit-chips.js + * + * Issue #1996: the Scope Audit observed/verified chips failed light-theme + * contrast. They tint their OWN background with + * `color-mix(in srgb, var(--status-green) 16%, transparent)`, which darkens + * whatever surface is behind them, and the text was --status-green-text + * (green-700, #15803d). Measured: 4.38:1 on white and 4.04:1 on --surface-0, + * against the 4.5:1 the #1719 gate requires for normal text. The reporter + * measured 4.04:1 on the populated page; this reproduces that number exactly, + * which is how we know the model here matches the browser's. + * + * Why this is a separate suite rather than a case in + * test-a11y-1719-contrast-root-causes-e2e.js: that suite's parseColor handles + * hex and rgb() only. Teaching it color-mix() is a larger change than the fix, + * and these chips are the only current user of the function on a contrast- + * critical surface. If a second one appears, move this in there. + * + * The default Scope Audit fixture renders no chips at all, which is why the + * existing browser coverage missed this. Deriving the values from the + * stylesheets rather than from a rendered page keeps that gap from mattering. + */ +'use strict'; + +const REPO_ROOT = require('path').resolve(__dirname, '..', '..'); +const fs = require('fs'); +const path = require('path'); +const assert = require('assert'); + +let passed = 0, failed = 0; +function test(name, fn) { + try { fn(); passed++; console.log(' ✓ ' + name); } + catch (e) { failed++; console.error(' ✗ ' + name + ': ' + e.message); } +} + +const STYLE = fs.readFileSync(path.join(REPO_ROOT, 'public/style.css'), 'utf8'); +const AUDIT = fs.readFileSync(path.join(REPO_ROOT, 'public/scope-audit.css'), 'utf8'); + +const REQUIRED = 4.5; + +// --- colour maths ---------------------------------------------------------- + +function hex(s) { + const m = String(s).trim().match(/^#([0-9a-f]{6})$/i); + if (!m) return null; + const n = parseInt(m[1], 16); + return [(n >> 16) & 255, (n >> 8) & 255, n & 255]; +} +function lum(rgb) { + const f = (c) => { c /= 255; return c <= 0.03928 ? c / 12.92 : Math.pow((c + 0.055) / 1.055, 2.4); }; + return 0.2126 * f(rgb[0]) + 0.7152 * f(rgb[1]) + 0.0722 * f(rgb[2]); +} +function contrast(fg, bg) { + const a = lum(fg), b = lum(bg); + return (Math.max(a, b) + 0.05) / (Math.min(a, b) + 0.05); +} +// color-mix(in srgb, C p%, transparent) over a surface is a plain alpha +// composite of C at p% onto that surface. +function mixOver(colour, pct, surface) { + return [0, 1, 2].map((i) => Math.round(colour[i] * pct + surface[i] * (1 - pct))); +} + +// --- reading the stylesheets ---------------------------------------------- + +// Variable values per theme. The light values come from the first :root block, +// the dark ones from the prefers-color-scheme block, which is what a visitor on +// the default setting gets. +function varValue(name, theme) { + const scope = theme === 'dark' + ? STYLE.slice(STYLE.indexOf('prefers-color-scheme: dark')) + : STYLE; + // Follow one level of var(--x, #fallback) indirection, which is how the + // palette is referenced. + const re = new RegExp('--' + name + '\\s*:\\s*([^;]+);'); + const m = scope.match(re); + if (!m) return null; + const raw = m[1].trim(); + const direct = hex(raw); + if (direct) return direct; + const via = raw.match(/var\(\s*--([\w-]+)\s*(?:,\s*(#[0-9a-f]{6})\s*)?\)/i); + if (!via) return null; + return varValue(via[1], theme) || (via[2] ? hex(via[2]) : null); +} + +function chipTintPct() { + const m = AUDIT.match(/\.sa-chip-observed\s*\{[^}]*color-mix\(in srgb,\s*var\(--status-green\)\s*(\d+)%/); + assert.ok(m, '.sa-chip-observed must still tint with color-mix on --status-green'); + return Number(m[1]) / 100; +} + +function chipFgVar() { + const m = AUDIT.match(/\.sa-chip-observed\s*\{[^}]*[^-\w]color\s*:\s*var\(\s*--([\w-]+)/); + assert.ok(m, '.sa-chip-observed must set its colour from a variable, never a literal'); + return m[1]; +} + +console.log('\n=== #1996: Scope Audit chip contrast ==='); + +test('the chip still tints its own background, so this test is testing something', () => { + const pct = chipTintPct(); + assert.ok(pct > 0 && pct < 1, 'tint percentage: ' + pct); +}); + +test('the chip takes its colour from a variable, not a literal', () => { + // A literal would not follow the customizer and would sidestep every theme + // check in this repo. + const name = chipFgVar(); + assert.ok(name.length > 0); +}); + +// The surfaces a chip can sit on. --surface-0 is the worse of the two in light +// and is what the report measured, so both are checked rather than the kinder one. +for (const theme of ['light', 'dark']) { + for (const surfaceVar of ['surface-0', 'card-bg']) { + test(`${theme}: chip meets ${REQUIRED}:1 on --${surfaceVar}`, () => { + const green = varValue('status-green', theme); + const surface = varValue(surfaceVar, theme); + const fg = varValue(chipFgVar(), theme); + assert.ok(green, 'could not resolve --status-green for ' + theme); + assert.ok(surface, 'could not resolve --' + surfaceVar + ' for ' + theme); + assert.ok(fg, 'could not resolve --' + chipFgVar() + ' for ' + theme); + + const composited = mixOver(green, chipTintPct(), surface); + const ratio = contrast(fg, composited); + assert.ok( + ratio >= REQUIRED, + `got ${ratio.toFixed(2)}:1, need ${REQUIRED}:1 ` + + `(text rgb(${fg}) on composited rgb(${composited}))` + ); + }); + } +} + +test('the regression itself: green-700 on the light chip would still fail', () => { + // Guards the fix from being quietly reverted to --status-green-text. If this + // ever passes, either the tint changed or the palette did, and the fix above + // is no longer doing the work this issue needed. + const green = varValue('status-green', 'light'); + const surface = varValue('surface-0', 'light'); + const old = varValue('palette-green-700', 'light'); + const ratio = contrast(old, mixOver(green, chipTintPct(), surface)); + assert.ok(ratio < REQUIRED, + `green-700 now reaches ${ratio.toFixed(2)}:1 — re-check whether this suite still guards anything`); +}); + +console.log(`\nTotal: ${passed} passed, ${failed} failed`); +if (failed) process.exit(1); diff --git a/test-a11y-axe-1668-selftest.js b/tests/unit/test-a11y-axe-1668-selftest.js similarity index 98% rename from test-a11y-axe-1668-selftest.js rename to tests/unit/test-a11y-axe-1668-selftest.js index 86bb8361f..77f40f528 100644 --- a/test-a11y-axe-1668-selftest.js +++ b/tests/unit/test-a11y-axe-1668-selftest.js @@ -11,13 +11,14 @@ */ 'use strict'; +const REPO_ROOT = require('path').resolve(__dirname, '..', '..'); const assert = require('assert'); const fs = require('fs'); const os = require('os'); const path = require('path'); -const mod = require('./test-a11y-axe-1668.js'); +const mod = require('../e2e/test-a11y-axe-1668.js'); // ---- routes / themes -------------------------------------------------------- assert.ok(Array.isArray(mod.ROUTES), 'ROUTES must be an array'); @@ -103,7 +104,7 @@ for (const r of mod.ROUTES) { // Cross-check REGISTERED_* against actual source code so the constant cannot // drift silently from `registerPage()` calls or analytics tab `case` arms. -const repoRoot = __dirname; +const repoRoot = REPO_ROOT; const publicDir = path.join(repoRoot, 'public'); if (fs.existsSync(publicDir)) { const files = fs.readdirSync(publicDir).filter(f => f.endsWith('.js')); @@ -184,7 +185,7 @@ assert.throws( // And loadAllowlist must propagate the throw end-to-end via a tmpfile. const tmpMalformed = path.join(os.tmpdir(), `a11y-malformed-${process.pid}.yaml`); fs.writeFileSync(tmpMalformed, 'this :: is :: garbage\n!!! no\n'); -const ALLOWLIST_PATH_ORIG = require.resolve('./test-a11y-axe-1668.js'); +const ALLOWLIST_PATH_ORIG = require.resolve('../e2e/test-a11y-axe-1668.js'); // We can't easily redirect ALLOWLIST_PATH without re-requiring; instead call // parseAllowlistYaml directly on the tmpfile content for end-to-end coverage. assert.throws( @@ -382,7 +383,7 @@ assert.throws( ); // ---- repo allowlist file: shape sanity -------------------------------------- -const allowPath = path.join(__dirname, 'tests', 'a11y-allowlist.yaml'); +const allowPath = path.join(REPO_ROOT, 'tests', 'a11y-allowlist.yaml'); assert.ok(fs.existsSync(allowPath), `tests/a11y-allowlist.yaml missing at ${allowPath}`); const entries = mod.loadAllowlist(); for (const e of entries) { diff --git a/test-a11y-axe-routes-coverage.js b/tests/unit/test-a11y-axe-routes-coverage.js similarity index 93% rename from test-a11y-axe-routes-coverage.js rename to tests/unit/test-a11y-axe-routes-coverage.js index b2846281e..66b25dc64 100644 --- a/test-a11y-axe-routes-coverage.js +++ b/tests/unit/test-a11y-axe-routes-coverage.js @@ -15,15 +15,16 @@ */ 'use strict'; +const REPO_ROOT = require('path').resolve(__dirname, '..', '..'); const assert = require('assert'); const fs = require('fs'); const path = require('path'); -const mod = require('./test-a11y-axe-1668.js'); +const mod = require('../e2e/test-a11y-axe-1668.js'); const analyticsSrc = fs.readFileSync( - path.join(__dirname, 'public', 'analytics.js'), + path.join(REPO_ROOT, 'public', 'analytics.js'), 'utf8' ); diff --git a/tests/unit/test-admin-dashboard-ui.js b/tests/unit/test-admin-dashboard-ui.js new file mode 100644 index 000000000..71c4d20a6 --- /dev/null +++ b/tests/unit/test-admin-dashboard-ui.js @@ -0,0 +1,695 @@ +/* Unit tests for the admin area (docs/specs/2026-10-07-admin-dashboard-design.md): + * public/admin-audit.js, public/admin-overview.js and public/admin.js, each + * loaded from disk into a vm sandbox (same pattern as + * test-user-management-ui.js). */ +'use strict'; +const vm = require('vm'); +const fs = require('fs'); +const path = require('path'); +const assert = require('assert'); + +const ROOT = path.resolve(__dirname, '..', '..'); +let passed = 0, failed = 0; +const pending = []; +function test(name, fn) { + const p = Promise.resolve().then(fn).then( + () => { passed++; console.log(' ok ' + name); }, + (e) => { failed++; console.log(' FAIL ' + name + ': ' + e.message); }); + pending.push(p); + return p; +} +const tick = () => new Promise((r) => setTimeout(r, 5)); +const src = (f) => fs.readFileSync(path.join(ROOT, f), 'utf8'); +const XSS = ''; + +// escapeHtml comes from the real app.js, not a copy. +function loadEscapeHtml() { + const m = src('public/app.js').match(/function escapeHtml\(s\) \{[\s\S]*?\n\}/); + assert(m, 'escapeHtml not found in app.js'); + return vm.runInNewContext('(' + m[0].replace(/^function escapeHtml/, 'function') + ')'); +} + +// Elements are created on first lookup and remember handlers and content. +function makeDom() { + const els = {}; + const mk = (id) => { + const el = { id, value: '', checked: false, textContent: '', innerHTML: '', hidden: false, handlers: {} }; + el.addEventListener = (t, fn) => { el.handlers[t] = fn; }; + el.insertAdjacentHTML = (pos, html) => { el.innerHTML += html; }; + el.querySelector = (sel) => els[id + ' ' + sel] || (els[id + ' ' + sel] = mk(id + ' ' + sel)); + return el; + }; + const document = { + visibilityState: 'visible', + getElementById(id) { return els[id] || (els[id] = mk(id)); }, + querySelector(sel) { return els[sel] || (els[sel] = mk(sel)); }, + }; + return { els, mk, document }; +} + +// loadTab runs the given public files with window === the sandbox global, +// a fake CSAuth whose request() answers from routes(path), a recording +// history.replaceState and recordable intervals. +function loadTab(files, hash, routes, extra) { + const dom = makeDom(); + const loc = { hash }; + const replaced = []; + const calls = []; + const bodies = []; + const timers = []; + const CSAuth = { + request(method, p, body) { calls.push(p); bodies.push(body); return Promise.resolve().then(() => routes(p, body)); }, + say(id, text, ok) { const el = dom.document.getElementById(id); el.textContent = text; el.ok = !!ok; }, + errText(r) { return (r.data && r.data.error) || ('Request failed (HTTP ' + r.status + ')'); }, + }; + const ctx = Object.assign({ + document: dom.document, location: loc, URLSearchParams, Promise, String, Number, Object, Math, Date, JSON, console, CSAuth, + history: { replaceState(a, b, h) { replaced.push(h); loc.hash = h; } }, + escapeHtml: loadEscapeHtml(), + setInterval(fn, ms) { timers.push({ fn, ms, cleared: false }); return timers.length; }, + clearInterval(id) { if (timers[id - 1]) timers[id - 1].cleared = true; }, + }, extra || {}); + ctx.window = ctx; + vm.createContext(ctx); + [].concat(files).forEach((f) => vm.runInContext(src(f), ctx)); + return { ctx, dom, els: dom.els, loc, replaced, calls, bodies, timers }; +} + +const NOW = Date.UTC(2026, 9, 7, 12, 0, 0); +const OK = (data) => ({ ok: true, status: 200, data }); + +console.log('admin-audit.js'); + +const E = (o) => Object.assign({ id: 10, at: '2026-10-07T10:00:00Z', action: 'user.login.failed', actor: null, + target: { id: 7, displayName: 'Eve', email: 'eve@example.org' }, detail: { reason: 'wrong_password' } }, o); +const auditEnv = (hash, routes) => loadTab('public/admin-audit.js', hash, routes); +const auditT = () => auditEnv('', () => OK({ entries: [], next: null })).ctx.CSAdminAudit._test; +const rows = (html) => (html.match(/ { + const t = auditT(); + const rh = (h) => JSON.parse(JSON.stringify(t.readHash(h))); + assert.deepStrictEqual(rh('#/admin?tab=audit&action=user.login.failed&user=12&period=7d'), + { action: 'user.login.failed', user: '12', period: '7d' }); + assert.deepStrictEqual(rh('#/admin?tab=audit&action=drop&user=1x&period=1y'), { action: '', user: '', period: '' }); + assert.strictEqual(rh('#/admin?tab=audit&user=0').user, '', 'user=0 is not an account id'); + assert.strictEqual(rh('#/admin?tab=audit&user=007').user, '', 'leading zeros are rejected'); + assert.strictEqual(rh('#/admin?tab=audit&action=user.login.*').action, 'user.login.*'); +}); + +test('the audit action filter offers channel proposals', () => { + assert.strictEqual(auditT().readHash('#/admin?tab=audit&action=proposal.*').action, 'proposal.*'); +}); + +test('the audit action filter offers data export and users.db backup', () => { + const t = auditT(); + assert.strictEqual(t.readHash('#/admin?tab=audit&action=user.export').action, 'user.export'); + assert.strictEqual(t.readHash('#/admin?tab=audit&action=user.backup').action, 'user.backup'); +}); + +test('hashFor writes tab=audit and only the set filters', () => { + const t = auditT(); + assert.strictEqual(t.hashFor({ action: 'user.login.*', user: '12', period: '' }), '#/admin?tab=audit&action=user.login.*&user=12'); + assert.strictEqual(t.hashFor({ action: '', user: '', period: '30d' }), '#/admin?tab=audit&period=30d'); +}); + +test('apiPath turns the period into from, adds before and the page size', () => { + const t = auditT(); + const from = encodeURIComponent(new Date(NOW - 864e5).toISOString()); + assert.strictEqual(t.apiPath({ action: 'user.login.failed', user: '12', period: '24h' }, null, NOW), + '/api/admin/audit?action=user.login.failed&user=12&from=' + from + '&limit=100'); + assert.strictEqual(t.apiPath({ action: '', user: '', period: '' }, 55, NOW), '/api/admin/audit?before=55&limit=100'); +}); + +test('rows escape actions, names, emails and details; deleted and system refs', () => { + const t = auditT(); + const html = t.rowHtml(E({ action: XSS, actor: { id: 3, displayName: XSS, email: XSS }, detail: { [XSS]: XSS } })); + assert(html.indexOf('