diff --git a/.env.example b/.env.example index 548e3c0ba..860d868fe 100644 --- a/.env.example +++ b/.env.example @@ -1,15 +1,36 @@ # MeshCore Analyzer — Environment Configuration -# Copy to .env and customize. All values have sensible defaults. +# New installation: copy to .env and customize. Existing installation: merge +# missing keys into your private .env; do not replace paths or credentials. # # This file is read by BOTH docker compose AND manage.sh — one source of truth. # manage.sh setup negotiates and updates only these production managed keys: # PROD_DATA_DIR, PROD_HTTP_PORT, PROD_HTTPS_PORT, PROD_MQTT_PORT, DISABLE_MOSQUITTO # Each environment keeps config + data together in one directory: -# ~/meshcore-data/config.json, meshcore.db, Caddyfile, theme.json -# ~/meshcore-staging-data/config.json, meshcore.db, Caddyfile +# ~/meshcore-data/config.json, Caddyfile, theme.json, queues and backups +# ~/meshcore-staging-data/config.json, Caddyfile, queues and backups +# With PostgreSQL selected, its separate persistent directory is not a logical backup. + +# --- Storage choice --- +# SQLite is the default and needs no database service or PostgreSQL credentials. +# setup offers PostgreSQL explicitly. Once installed, storage-selection.json in +# the shared state directory is authoritative; ordinary updates keep it. +# To change engines, use the verified offline storage switch in docs/storage.md. +# Optional PostgreSQL credentials/directories: merge .env.postgres.example. +# Do not replace an existing .env or regenerate existing database passwords. +# Optional container state subdirectory, shared by setup/server/ingestor: +# CORESCOPE_STATE_DIR=/app/data + +# --- Example deployment (docker-compose.example.yml only) --- +# For upgrades, point DATA_DIR at the existing state directory BEFORE bootstrap. +# PROD_DATA_DIR and STAGING_DATA_DIR below do not control the example variant. +DATA_DIR=./data +HTTP_PORT=80 +# CORESCOPE_IMAGE= +# Bundled Caddy forwards port 80 to the Go server on 3000 by default. If you +# explicitly set DISABLE_CADDY=true, your proxy/port mapping must use port 3000. # --- Production --- -# Data directory (database, theme, etc.) +# State directory (config, theme, queues and backups) # Default: ~/meshcore-data # Used by: docker compose, manage.sh PROD_DATA_DIR=~/meshcore-data diff --git a/.env.postgres.example b/.env.postgres.example new file mode 100644 index 000000000..d0bbe08a2 --- /dev/null +++ b/.env.postgres.example @@ -0,0 +1,21 @@ +# OPTIONAL PostgreSQL settings — merge into your existing private .env. +# Base Compose files need none of these. Add the matching .postgres.yml override. +# Use a different persistent directory for each production/staging instance. +# Generate a DIFFERENT value for each blank field: openssl rand -hex 32 +# Keep .env private. Hexadecimal passwords can be safely embedded in the +# generated container URLs; the initializer enforces at least 32 hex digits. +# These roles are created only when PostgreSQL's data directory is empty. +# Editing these values afterward does not change existing database passwords. +POSTGRES_ADMIN_PASSWORD= +CORESCOPE_OWNER_PASSWORD= +CORESCOPE_READER_PASSWORD= +CORESCOPE_WRITER_PASSWORD= +CORESCOPE_ACCOUNTS_PASSWORD= +CORESCOPE_CHANNELS_PASSWORD= +PROD_POSTGRES_DATA_DIR=~/corescope-postgres +STAGING_POSTGRES_DATA_DIR=~/corescope-staging-postgres +# docker-compose.example.yml uses this local default instead: +POSTGRES_DATA_DIR=./postgres-data +# Existing installations keep their recorded backend on update. Changing engines +# requires the offline storage switch in docs/storage.md. Never mount two database +# servers on one persistent directory. diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 79f594ebc..b2028a3ee 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -23,10 +23,17 @@ on: description: 'Release fast path already published the tag images' type: boolean default: false + postgres_benchmark: + description: Run only the isolated PostgreSQL benchmark (no publish or deploy) + type: boolean + default: false + candidate_sha: + description: Full candidate commit for postgres_benchmark + type: string + default: '' permissions: contents: read - packages: write concurrency: group: ci-${{ github.event.pull_request.number || github.ref }} @@ -47,12 +54,26 @@ env: # PRs stop after build-and-publish (no GHCR push). Master continues to deploy + badges. jobs: + postgres-benchmark: + name: Isolated PostgreSQL comparison + needs: [changes] + if: | + (github.event_name == 'workflow_dispatch' && inputs.postgres_benchmark) + || (github.event_name == 'pull_request' + && needs.changes.outputs.code == 'true' + && contains(github.event.pull_request.body, '[x] Run PostgreSQL comparison')) + permissions: + contents: read + uses: ./.github/workflows/postgres-benchmark.yml + with: + candidate_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || inputs.candidate_sha }} + # ─────────────────────────────────────────────────────────────── # 0. Change scope — decides whether the expensive jobs below run. # ─────────────────────────────────────────────────────────────── changes: name: "🔎 Change scope" - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 outputs: code: ${{ steps.scope.outputs.code }} ingestor: ${{ steps.scope.outputs.ingestor }} @@ -66,6 +87,12 @@ jobs: id: scope run: | set -euo pipefail + if [ "${{ inputs.postgres_benchmark }}" = "true" ]; then + echo "code=false" >> "$GITHUB_OUTPUT" + echo "ingestor=false" >> "$GITHUB_OUTPUT" + echo "benchmark-only dispatch" + exit 0 + fi # Only pull requests are scoped. Pushes and dispatches always count as # code, so master keeps producing an :edge image and tag builds are # never skipped. @@ -115,19 +142,46 @@ jobs: # The server has -race in its own step already; this closes the same gap for # the ingestor, which carries an atomic.Pointer snapshot (the region key # set) whose safety is an argument until something checks it. - race-test: - name: "🏁 Race detector (ingestor)" - runs-on: ubuntu-latest + race-backend: + name: "Race detector (ingestor, ${{ matrix.backend }})" + strategy: + fail-fast: false + matrix: + backend: [sqlite, postgres] + runs-on: ubuntu-24.04 needs: [changes] if: needs.changes.outputs.ingestor == 'true' + services: &postgres_test_services + postgres: + image: ${{ matrix.backend == 'postgres' && 'postgres:18.6-alpine3.24' || '' }} + env: + POSTGRES_PASSWORD: "ci-${{ github.run_id }}-${{ github.run_attempt }}" + POSTGRES_INITDB_ARGS: "--encoding=UTF8 --locale=C" + ports: ["127.0.0.1:5432:5432"] + options: >- + --health-cmd "pg_isready -h 127.0.0.1 -U postgres" + --health-interval 5s --health-timeout 5s --health-retries 20 + env: + CORESCOPE_TEST_BACKEND: ${{ matrix.backend }} + CORESCOPE_DB_BACKEND: ${{ matrix.backend }} + CORESCOPE_TEST_POSTGRES_URL: ${{ matrix.backend == 'postgres' && format('postgresql://postgres:ci-{0}-{1}@127.0.0.1:5432/postgres?sslmode=disable', github.run_id, github.run_attempt) || '' }} + PGHOST: 127.0.0.1 + PGPORT: 5432 + PGUSER: postgres + PGPASSWORD: "ci-${{ github.run_id }}-${{ github.run_attempt }}" + PGSSLMODE: disable steps: - name: Checkout code uses: actions/checkout@v5 - - name: Set up Go 1.27 + - name: Install pinned PostgreSQL 18.6 clients + if: matrix.backend == 'postgres' + run: &install_postgres_clients bash scripts/install-postgres-ci.sh client + + - name: Set up Go 1.27.2 uses: actions/setup-go@v6 with: - go-version: '1.27' + go-version: '1.27.2' # All modules: a -race build of a cgo package is the slowest thing in # this pipeline, and setup-go caches ~/.cache/go-build as well as the # module cache, so a key that misses on an unrelated module hurts most @@ -140,14 +194,50 @@ jobs: cd cmd/ingestor go test -timeout 30m -race ./... + race-test: + name: "🏁 Race detector (ingestor)" + runs-on: ubuntu-24.04 + needs: [race-backend, changes] + if: ${{ !cancelled() && needs.changes.outputs.ingestor == 'true' }} + steps: + - name: Require both native race matrices + run: | + result="${{ needs['race-backend'].result }}" + [ "$result" = success ] || { echo "::error::Race matrices did not all pass ($result)"; exit 1; } + + go-test: + name: "✅ Go Build & Test" + runs-on: ubuntu-24.04 + needs: [go-backend, changes] + if: ${{ !cancelled() && needs.changes.outputs.code == 'true' }} + steps: + - name: Require both native storage matrices + run: | + result="${{ needs['go-backend'].result }}" + [ "$result" = success ] || { echo "::error::Storage matrices did not all pass ($result)"; exit 1; } + # ─────────────────────────────────────────────────────────────── # 1. Go Build & Test # ─────────────────────────────────────────────────────────────── - go-test: - name: "✅ Go Build & Test" - runs-on: ubuntu-latest + go-backend: + name: "Go Build & Test (${{ matrix.backend }})" + strategy: + fail-fast: false + matrix: + backend: [sqlite, postgres] + runs-on: ubuntu-24.04 needs: [changes] if: needs.changes.outputs.code == 'true' + services: *postgres_test_services + env: + CORESCOPE_TEST_BACKEND: ${{ matrix.backend }} + CORESCOPE_DB_BACKEND: ${{ matrix.backend }} + CORESCOPE_TEST_POSTGRES_URL: ${{ matrix.backend == 'postgres' && format('postgresql://postgres:ci-{0}-{1}@127.0.0.1:5432/postgres?sslmode=disable', github.run_id, github.run_attempt) || '' }} + PGHOST: 127.0.0.1 + PGPORT: 5432 + PGUSER: postgres + PGPASSWORD: "ci-${{ github.run_id }}-${{ github.run_attempt }}" + PGSSLMODE: disable steps: - name: Checkout code uses: actions/checkout@v5 @@ -157,21 +247,28 @@ jobs: - name: Clean Go module cache run: rm -rf ~/go/pkg/mod 2>/dev/null || true - - name: Set up Go 1.27 + - name: Install pinned PostgreSQL 18.6 clients + if: matrix.backend == 'postgres' + run: *install_postgres_clients + + + - name: Set up Go 1.27.2 uses: actions/setup-go@v6 with: - go-version: '1.27' - # All modules, not just two: the cgo SQLite build is expensive to + go-version: '1.27.2' + # All modules: the offline SQLite importer is expensive to # redo, and setup-go caches ~/.cache/go-build as well as the module # cache, so a key that misses on an unrelated module hurts. cache-dependency-path: "**/go.sum" + - name: Test PostgreSQL benchmark controls + run: python3 -m unittest discover -s scripts/postgres-benchmark -p 'test_*.py' + - name: Enforce gofmt + go vet (issue #1859) run: | set -e - # gofmt: every tracked *.go file except the misnamed Dockerfile.go, - # which is a Dockerfile (FROM ...) that gofmt cannot parse. - unformatted=$(gofmt -l $(git ls-files '*.go' | grep -vi 'Dockerfile.go')) + # Every tracked Go source file must be formatted. + unformatted=$(gofmt -l $(git ls-files '*.go')) if [ -n "$unformatted" ]; then echo "::error::gofmt required on these files:" echo "$unformatted" @@ -188,7 +285,7 @@ jobs: run: | set -e -o pipefail cd cmd/server - go build . + CGO_ENABLED=1 go build . # -race gates PR #1208's atomic.Pointer migration: the race-detector # is what makes path_inspect_atomic_race_test.go actually assert. go test -timeout 20m -race -coverprofile=server-coverage.out ./... 2>&1 | tee server-test.log @@ -201,7 +298,7 @@ jobs: run: | set -e -o pipefail cd cmd/ingestor - go build . + CGO_ENABLED=1 go build . go test -timeout 20m -coverprofile=ingestor-coverage.out ./... 2>&1 | tee ingestor-test.log echo "--- Go Ingestor Coverage ---" go tool cover -func=ingestor-coverage.out | tail -1 @@ -213,7 +310,7 @@ jobs: go test ./... echo "--- Channel library tests passed ---" cd ../../cmd/decrypt - go build -ldflags="-s -w" -o corescope-decrypt . + CGO_ENABLED=1 go build -ldflags="-s -w" -o corescope-decrypt . go test ./... echo "--- Decrypt CLI tests passed ---" @@ -224,18 +321,32 @@ jobs: cd ../mailer && go test -race ./... echo "--- users + mailer tests passed ---" + - name: Test shared selection and native SQLite snapshot modules + if: matrix.backend == 'sqlite' + run: | + set -euo pipefail + cd internal/dbconfig && go test -race ./... + cd ../sqliteutil && go test -race ./... + - name: Test migrate CLI + dbschema run: | set -e -o pipefail # Neither module had CI test execution before the mattn/go-sqlite3 # migration; both open the database, and dbschema owns the migrations. + if [ "$CORESCOPE_TEST_BACKEND" = postgres ]; then + (cd internal/pgutil && go test ./...) + fi cd internal/dbschema go test ./... cd ../../cmd/migrate - go build -o /dev/null . + CGO_ENABLED=1 go build -o /dev/null . go test ./... echo "--- migrate + dbschema tests passed ---" + - name: Validate default and optional Compose models + if: matrix.backend == 'sqlite' + run: node scripts/test-storage-compose.js + - name: Verify Dockerfile COPY invariants (issue #1316) run: bash scripts/check-dockerfile-internal-pkgs.sh @@ -246,7 +357,12 @@ jobs: run: bash docker/test-select-service-user.sh - name: QA SQL parameter-binding unit tests (issue #1977) - run: bash qa/scripts/test-blacklist-sql.sh + run: | + if [ "$CORESCOPE_TEST_BACKEND" = sqlite ] && ! command -v sqlite3 >/dev/null; then + sudo apt-get update -qq + sudo apt-get install -y sqlite3 + fi + bash qa/scripts/test-blacklist-sql.sh - name: Lint CSS variables (issue #1128) run: | @@ -295,7 +411,7 @@ jobs: echo "✅ All .proto files are syntactically valid" - name: Generate Go coverage badges - if: success() + if: success() && matrix.backend == 'sqlite' run: | mkdir -p .badges @@ -324,7 +440,7 @@ jobs: echo "| Ingestor | ${INGESTOR_COV}% |" >> $GITHUB_STEP_SUMMARY - name: Upload Go coverage badges - if: success() + if: success() && matrix.backend == 'sqlite' uses: actions/upload-artifact@v6 with: name: go-badges @@ -345,19 +461,30 @@ jobs: # uses nothing go-test produces. build-and-publish still needs both, so # nothing reaches GHCR unless every test job passed. e2e-shard: - name: "🎭 Playwright E2E shard ${{ matrix.shard }}/3" + name: "🎭 Playwright E2E (${{ matrix.backend }}) shard ${{ matrix.shard }}/3" needs: [changes] - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 strategy: + fail-fast: false matrix: shard: [1, 2, 3] + backend: [sqlite, postgres] env: + CORESCOPE_TEST_BACKEND: ${{ matrix.backend }} + CORESCOPE_DB_BACKEND: ${{ matrix.backend }} + CORESCOPE_TEST_POSTGRES_URL: ${{ matrix.backend == 'postgres' && format('postgresql://postgres:ci-{0}-{1}@127.0.0.1:5432/postgres?sslmode=disable', github.run_id, github.run_attempt) || '' }} + PGHOST: 127.0.0.1 + PGPORT: 5432 + PGUSER: postgres + PGPASSWORD: "ci-${{ github.run_id }}-${{ github.run_attempt }}" + PGSSLMODE: disable SHARD: ${{ matrix.shard }} E2E_SHARDS: 3 defaults: run: shell: bash if: needs.changes.outputs.code == 'true' && !(startsWith(github.ref, 'refs/tags/v') && inputs.images_published) + services: *postgres_test_services steps: - name: Checkout code uses: actions/checkout@v5 @@ -372,16 +499,21 @@ jobs: - name: Clean Go module cache run: rm -rf ~/go/pkg/mod 2>/dev/null || true - - name: Set up Go 1.27 + - name: Install pinned PostgreSQL 18.6 clients + if: matrix.backend == 'postgres' + run: *install_postgres_clients + + + - name: Set up Go 1.27.2 uses: actions/setup-go@v6 with: - go-version: '1.27' + go-version: '1.27.2' cache-dependency-path: "**/go.sum" - name: Build Go server run: | cd cmd/server - go build -o ../../corescope-server . + CGO_ENABLED=1 go build -o ../../corescope-server . echo "Go server built successfully" # Only test-user-management-e2e.js uses the e2etest build and its server, @@ -391,12 +523,12 @@ jobs: if: matrix.shard == 3 run: | cd cmd/server - go build -tags e2etest -o ../../corescope-server-e2e . + CGO_ENABLED=1 go build -tags e2etest -o ../../corescope-server-e2e . - name: Build Go migrate tool run: | cd cmd/migrate - go build -o ../../corescope-migrate . + CGO_ENABLED=1 go build -o ../../corescope-migrate . echo "Go migrate tool built successfully" - name: Install npm dependencies @@ -410,8 +542,17 @@ jobs: - name: Instrument frontend JS for coverage run: sh scripts/instrument-frontend.sh + - name: Prepare disposable legacy fixture copy + run: | + mkdir -p "$RUNNER_TEMP/corescope-fixture" + cd cmd/migrate + go run ../../scripts/prepare-postgres-fixture.go \ + -source ../../test-fixtures/e2e-fixture.db \ + -destination "$RUNNER_TEMP/corescope-fixture/fixture.sqlite" + echo "E2E_SQLITE=$RUNNER_TEMP/corescope-fixture/fixture.sqlite" >> "$GITHUB_ENV" + - name: Freshen fixture timestamps - run: bash tools/freshen-fixture.sh test-fixtures/e2e-fixture.db + run: bash tools/freshen-fixture.sh "$E2E_SQLITE" - name: Seed grouped-packet row for #1486 collapse test # The committed fixture has 499 packets, each with exactly ONE @@ -420,11 +561,9 @@ jobs: # (toggle-select) row. Insert a NEW transmission with 3 # observations. # - # The server's async hash-migrate (cmd/server/hash_migrate.go) - # recomputes `transmissions.hash` from `raw_hex` via - # ComputeContentHash(), so the inserted hash MUST equal that - # function's output for the chosen raw_hex — otherwise the row - # gets relabelled and the E2E can't find it. + # Fixture hashes are normalized offline before PostgreSQL import. + # The read-only server verifies them and never rewrites telemetry. + # This seeded hash must equal the shared runtime ContentHash output. # # raw_hex 15000102030405060708090a0b0c0d0e0f # → header=0x15 (route_type=1, payload_type=5) @@ -435,7 +574,7 @@ jobs: # The #1486 and #2104 tests reach it via an explicit hash + # ?timeWindow=0 deep-link. run: | - sqlite3 test-fixtures/e2e-fixture.db <<'SQL' + sqlite3 "$E2E_SQLITE" <<'SQL' -- Sort the seeded row LAST in BOTH default packets views: -- • flat view sorts by transmissions.id DESC → id=0 puts it last -- • grouped view (#default for the packets page) sorts by @@ -473,20 +612,11 @@ jobs: (-1000000,1,'rx',7.0,-88,0,'[]',CAST(strftime('%s','now') AS INTEGER),'[]'); SQL - - name: Migrate fixture DB to current schema (#1287) - # Server now ASSERTs schema is migrated and refuses to start - # otherwise (cmd/server/main.go: dbschema.AssertReady). In prod - # the ingestor owns dbschema.Apply, but CI starts only the - # server against the committed e2e fixture — so we run the - # standalone migrate tool here to bring the fixture up to the - # required shape before the server boots. - run: ./corescope-migrate -db test-fixtures/e2e-fixture.db - - name: 'Seed distinct observation bytes for #2104' # raw_hex exists only after migration. Keep the #1486 payload/hash # while encoding each observation's existing one-hop path. run: | - sqlite3 test-fixtures/e2e-fixture.db <<'SQL' + sqlite3 "$E2E_SQLITE" <<'SQL' UPDATE observations SET raw_hex = CASE observer_idx WHEN 1 THEN '1501aa0102030405060708090a0b0c0d0e0f' WHEN 2 THEN '1501bb0102030405060708090a0b0c0d0e0f' @@ -495,13 +625,61 @@ jobs: SQL - name: Seed deterministic Path Inspector routes (#2060) - run: sqlite3 test-fixtures/e2e-fixture.db < test-fixtures/path-inspector.sql + run: sqlite3 "$E2E_SQLITE" < test-fixtures/path-inspector.sql + + - name: Normalize fixture hashes using the runtime parser + run: | + cd cmd/migrate + go run ../../scripts/migrate-fixture-hashes.go "$E2E_SQLITE" + + - name: Import fixture and grant restricted runtime roles + if: matrix.backend == 'postgres' + run: | + set -euo pipefail + for key in CORESCOPE_OWNER_PASSWORD CORESCOPE_READER_PASSWORD CORESCOPE_WRITER_PASSWORD CORESCOPE_ACCOUNTS_PASSWORD CORESCOPE_CHANNELS_PASSWORD; do + printf -v "$key" '%s' "$(openssl rand -hex 32)" + export "$key" + echo "::add-mask::$(printenv "$key")" + done + sh docker/postgres-init.sh + owner_url="postgresql://corescope_owner:$CORESCOPE_OWNER_PASSWORD@127.0.0.1:5432/corescope_telemetry?sslmode=disable" + owner_accounts="postgresql://corescope_owner:$CORESCOPE_OWNER_PASSWORD@127.0.0.1:5432/corescope_accounts?sslmode=disable" + CORESCOPE_DATABASE_URL="$owner_url" CORESCOPE_USERS_DATABASE_URL="$owner_accounts" \ + ./corescope-migrate -offline -from-sqlite "$E2E_SQLITE" -state-dir "$RUNNER_TEMP/corescope-import" + for database in corescope_telemetry corescope_accounts; do + PGUSER=corescope_owner PGPASSWORD="$CORESCOPE_OWNER_PASSWORD" PGDATABASE="$database" \ + psql -X -v ON_ERROR_STOP=1 -f docker/postgres-grants.sql + done + export CORESCOPE_DATABASE_URL="$owner_url" CORESCOPE_USERS_OWNER_DATABASE_URL="$owner_accounts" + export CORESCOPE_READER_DATABASE_URL="postgresql://corescope_reader:$CORESCOPE_READER_PASSWORD@127.0.0.1:5432/corescope_telemetry?sslmode=disable&search_path=public" + export CORESCOPE_WRITER_DATABASE_URL="postgresql://corescope_writer:$CORESCOPE_WRITER_PASSWORD@127.0.0.1:5432/corescope_telemetry?sslmode=disable&search_path=public" + export CORESCOPE_USERS_DATABASE_URL="postgresql://corescope_accounts:$CORESCOPE_ACCOUNTS_PASSWORD@127.0.0.1:5432/corescope_accounts?sslmode=disable&search_path=public" + export CORESCOPE_APPROVED_CHANNELS_DATABASE_URL="postgresql://corescope_channels:$CORESCOPE_CHANNELS_PASSWORD@127.0.0.1:5432/corescope_accounts?sslmode=disable&search_path=public" + for key in CORESCOPE_DATABASE_URL CORESCOPE_USERS_OWNER_DATABASE_URL CORESCOPE_READER_DATABASE_URL CORESCOPE_WRITER_DATABASE_URL CORESCOPE_USERS_DATABASE_URL CORESCOPE_APPROVED_CHANNELS_DATABASE_URL; do + printf '%s=%s\n' "$key" "${!key}" >> "$GITHUB_ENV" + done + mkdir -p "$RUNNER_TEMP/corescope-state" + ./corescope-migrate -storage-action=adopt -backend=postgres -offline \ + -selection-file "$RUNNER_TEMP/corescope-state/storage-selection.json" + # Retain the masked owner password for the later restore step only. + umask 077 + printf '%s' "$CORESCOPE_OWNER_PASSWORD" > "$RUNNER_TEMP/corescope-e2e-owner-password" + + - name: Adopt prepared SQLite fixture without a PostgreSQL service + if: matrix.backend == 'sqlite' + run: | + set -euo pipefail + mkdir -p "$RUNNER_TEMP/corescope-state" + ./corescope-migrate -storage-action=setup -backend=sqlite -offline \ + -selection-file "$RUNNER_TEMP/corescope-state/storage-selection.json" \ + -sqlite-path "$E2E_SQLITE" -users-sqlite-path "$RUNNER_TEMP/corescope-state/users.db" + printf 'DB_PATH=%s\n' "$E2E_SQLITE" >> "$GITHUB_ENV" - name: Start Go server with fixture DB run: | fuser -k 13581/tcp 2>/dev/null || true sleep 1 - ./corescope-server -port 13581 -db test-fixtures/e2e-fixture.db -public public-instrumented & + env -u PGPASSWORD -u PGUSER -u CORESCOPE_TEST_POSTGRES_URL -u CORESCOPE_DATABASE_URL -u CORESCOPE_USERS_OWNER_DATABASE_URL ./corescope-server -port 13581 -state-dir "$RUNNER_TEMP/corescope-state" -public public-instrumented & echo $! > .server.pid for i in $(seq 1 30); do if curl -sf http://localhost:13581/api/healthz > /dev/null 2>&1; then @@ -521,14 +699,12 @@ jobs: fuser -k 13582/tcp 2>/dev/null || true rm -rf /tmp/cs-um mkdir -p /tmp/cs-um - # Own copy of the (already migrated and seeded) fixture; fresh users.db every run. - cp test-fixtures/e2e-fixture.db /tmp/cs-um/on.db + # Shared read-only telemetry; the account writer has its own logical database. cat > /tmp/cs-um/config.json <<'JSON' { "port": 13582, "userManagement": { "enabled": true, - "dbPath": "/tmp/cs-um/users.db", "adminEmails": ["admin@e2e.test"], "publicBaseUrl": "http://localhost:13582", "mail": { "provider": "fake", "fromEmail": "noreply@e2e.test" }, @@ -537,7 +713,18 @@ jobs: } } JSON - ./corescope-server-e2e -config-dir /tmp/cs-um -port 13582 -db /tmp/cs-um/on.db -public public-instrumented & + if [ "$CORESCOPE_TEST_BACKEND" = sqlite ]; then + # Existing telemetry is adopted with no account target; initialize + # first accounts explicitly before starting the strict runtime. + ./corescope-migrate -storage-action=adopt -backend=sqlite -offline -config-dir /tmp/cs-um \ + -selection-file /tmp/cs-um/storage-selection.json -sqlite-path "$E2E_SQLITE" -users-sqlite-path /tmp/cs-um/users.db + ./corescope-migrate -storage-action=setup -backend=sqlite -offline -config-dir /tmp/cs-um \ + -selection-file /tmp/cs-um/storage-selection.json -sqlite-path "$E2E_SQLITE" -users-sqlite-path /tmp/cs-um/users.db + else + ./corescope-migrate -storage-action=adopt -backend=postgres -offline -config-dir /tmp/cs-um \ + -selection-file /tmp/cs-um/storage-selection.json + fi + env -u PGPASSWORD -u PGUSER -u CORESCOPE_TEST_POSTGRES_URL -u CORESCOPE_DATABASE_URL -u CORESCOPE_USERS_OWNER_DATABASE_URL ./corescope-server-e2e -config-dir /tmp/cs-um -port 13582 -state-dir /tmp/cs-um -public public-instrumented & echo $! > .server-um.pid for i in $(seq 1 30); do if curl -sf http://localhost:13582/api/healthz > /dev/null 2>&1; then echo "UM server ready after ${i}s"; break; fi @@ -711,6 +898,104 @@ jobs: suite 2 test-show-neighbors.js BASE_URL=$B suite 3 test-user-management-e2e.js BASE_URL=http://localhost:13582 BASE_URL_OFF=$B + - name: Restore native HTTP backups and verify browser sessions + if: matrix.shard == 3 + run: | + set -euo pipefail + export E2E_BACKUP_DIR="$RUNNER_TEMP/corescope-http-backup" + export BASE_URL=http://localhost:13582 + echo "=== E2E SUITE: test-postgres-backup-e2e.js capture ===" | tee -a e2e-output.txt + node tests/e2e/test-postgres-backup-e2e.js capture 2>&1 | tee -a e2e-output.txt + for pidfile in .server.pid .server-um.pid; do + pid=$(cat "$pidfile") + kill "$pid" + for attempt in $(seq 1 30); do + kill -0 "$pid" 2>/dev/null || break + sleep 1 + done + ! kill -0 "$pid" 2>/dev/null || { echo 'Fixture server did not stop' >&2; exit 1; } + done + if [ "$CORESCOPE_TEST_BACKEND" = postgres ]; then + # Fresh databases, same deployment roles, no overwrite or role widening. + psql -X -v ON_ERROR_STOP=1 -d postgres <<'SQL' + CREATE DATABASE corescope_restore_telemetry OWNER corescope_owner TEMPLATE template0 ENCODING 'UTF8' LOCALE 'C'; + CREATE DATABASE corescope_restore_accounts OWNER corescope_owner TEMPLATE template0 ENCODING 'UTF8' LOCALE 'C'; + REVOKE ALL ON DATABASE corescope_restore_telemetry FROM PUBLIC; + REVOKE ALL ON DATABASE corescope_restore_accounts FROM PUBLIC; + GRANT CONNECT ON DATABASE corescope_restore_telemetry TO corescope_reader, corescope_writer; + GRANT CONNECT ON DATABASE corescope_restore_accounts TO corescope_accounts, corescope_channels; + SQL + for kind in telemetry accounts; do + PGUSER=corescope_owner PGPASSWORD="$(cat "$RUNNER_TEMP/corescope-e2e-owner-password")" \ + pg_restore --no-owner --no-privileges --exit-on-error --single-transaction \ + --dbname="corescope_restore_$kind" "$E2E_BACKUP_DIR/$kind.dump" + PGUSER=corescope_owner PGPASSWORD="$(cat "$RUNNER_TEMP/corescope-e2e-owner-password")" \ + psql -X -v ON_ERROR_STOP=1 -d "corescope_restore_$kind" -f docker/postgres-grants.sql + PGUSER=corescope_owner PGPASSWORD="$(cat "$RUNNER_TEMP/corescope-e2e-owner-password")" \ + psql -X -v ON_ERROR_STOP=1 -d "corescope_restore_$kind" -c 'ANALYZE' + done + # Switch restored database URLs. + restore_urls=$(python3 - <<'PY' + import os + from urllib.parse import urlsplit, urlunsplit + for key, database in ( + ("CORESCOPE_READER_DATABASE_URL", "corescope_restore_telemetry"), + ("CORESCOPE_USERS_DATABASE_URL", "corescope_restore_accounts"), + ("CORESCOPE_WRITER_DATABASE_URL", "corescope_restore_telemetry"), + ("CORESCOPE_APPROVED_CHANNELS_DATABASE_URL", "corescope_restore_accounts"), + ("CORESCOPE_DATABASE_URL", "corescope_restore_telemetry"), + ("CORESCOPE_USERS_OWNER_DATABASE_URL", "corescope_restore_accounts"), + ): + raw = os.environ[key] + try: + parsed = urlsplit(raw) + except ValueError: + raise SystemExit("Invalid restore database URL") from None + if parsed.scheme not in ("postgres", "postgresql") or not parsed.netloc or "\n" in raw or "\r" in raw: + raise SystemExit("Invalid restore database URL") + print(key + "=" + urlunsplit(parsed._replace(path="/" + database))) + PY + ) + while IFS= read -r assignment; do export "$assignment"; done <<< "$restore_urls" + unset restore_urls assignment + # End restored database URLs. + else + mkdir -m 700 "$RUNNER_TEMP/corescope-restored-sqlite" + for kind in telemetry accounts; do + cp "$E2E_BACKUP_DIR/$kind.db" "$RUNNER_TEMP/corescope-restored-sqlite/$kind.db" + test "$(sqlite3 -readonly "$RUNNER_TEMP/corescope-restored-sqlite/$kind.db" 'PRAGMA quick_check')" = ok + done + export DB_PATH="$RUNNER_TEMP/corescope-restored-sqlite/telemetry.db" + fi + # A restore uses new verified selection records; changing only role URLs + # must never silently replace the recorded old targets. + restored_state="$RUNNER_TEMP/corescope-restored-state" + restored_accounts_state="$RUNNER_TEMP/corescope-restored-accounts-state" + mkdir -m 700 "$restored_state" "$restored_accounts_state" + for selection in "$restored_state/storage-selection.json" "$restored_accounts_state/storage-selection.json"; do + if [ "$CORESCOPE_TEST_BACKEND" = sqlite ]; then + ./corescope-migrate -storage-action=adopt -backend=sqlite -offline \ + -selection-file "$selection" -sqlite-path "$DB_PATH" \ + -users-sqlite-path "$RUNNER_TEMP/corescope-restored-sqlite/accounts.db" + else + ./corescope-migrate -storage-action=adopt -backend=postgres -offline -selection-file "$selection" + fi + done + # Restart restored servers. + env -u PGPASSWORD -u PGUSER -u CORESCOPE_TEST_POSTGRES_URL -u CORESCOPE_DATABASE_URL -u CORESCOPE_USERS_OWNER_DATABASE_URL ./corescope-server -port 13581 -state-dir "$restored_state" -public public-instrumented & + echo $! > .server.pid + env -u PGPASSWORD -u PGUSER -u CORESCOPE_TEST_POSTGRES_URL -u CORESCOPE_DATABASE_URL -u CORESCOPE_USERS_OWNER_DATABASE_URL ./corescope-server-e2e -config-dir /tmp/cs-um -port 13582 -state-dir "$restored_accounts_state" -public public-instrumented & + echo $! > .server-um.pid + for port in 13581 13582; do + for attempt in $(seq 1 30); do + curl -sf "http://localhost:$port/api/healthz" >/dev/null && break + [ "$attempt" -lt 30 ] || { echo 'Restored server did not become ready' >&2; exit 1; } + sleep 1 + done + done + echo "=== E2E SUITE: test-postgres-backup-e2e.js verify ===" | tee -a e2e-output.txt + node tests/e2e/test-postgres-backup-e2e.js verify 2>&1 | tee -a e2e-output.txt + # #1616: slide-over focus-restore flake-gate. Runs the slide-over # E2E 3 more times against the SAME backend instance (it already # ran once in the list above) so the Chromium-headless focus race @@ -773,7 +1058,7 @@ jobs: if: success() uses: actions/upload-artifact@v6 with: - name: e2e-shard-${{ matrix.shard }} + name: e2e-shard-${{ matrix.backend }}-${{ matrix.shard }} path: | e2e-output.txt coverage-shard.json @@ -791,7 +1076,7 @@ jobs: e2e-test: name: "🎭 Playwright E2E Tests" needs: [e2e-shard, changes] - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 if: ${{ !cancelled() && needs.changes.outputs.code == 'true' && !(startsWith(github.ref, 'refs/tags/v') && inputs.images_published) }} steps: - name: Require every shard to have passed @@ -813,7 +1098,7 @@ jobs: run: | set -e ls shards - [ "$(ls -d shards/e2e-shard-* | wc -l)" -eq 3 ] || { echo "::error::expected results from 3 shards"; exit 1; } + [ "$(ls -d shards/e2e-shard-* | wc -l)" -eq 6 ] || { echo "::error::expected results from 3 shards for each backend"; exit 1; } cat shards/e2e-shard-*/e2e-output.txt > e2e-output.txt # Start empty: the checkout holds master's old .badges/ files, # which would otherwise go out in e2e-badges (#2136). @@ -881,7 +1166,7 @@ jobs: # ─────────────────────────────────────────────────────────────── # 3a. Two-arch image build — beside the tests, never publishes # ─────────────────────────────────────────────────────────────── - # Since the SQLite driver became cgo, the container build depends on a zig + # Native SQLite support uses cgo, so the container build needs a zig # cross-toolchain and static musl linking. Builds both architectures and # actually runs the arm64 binaries under QEMU, on every event: for a PR it is # the only image check (the GHCR push below is push/tag-only, so without it @@ -896,7 +1181,7 @@ jobs: image-check: name: "🧱 Two-arch image build" needs: [changes] - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 if: needs.changes.outputs.code == 'true' && !(startsWith(github.ref, 'refs/tags/v') && inputs.images_published) outputs: build_time: ${{ steps.meta.outputs.build_time }} @@ -926,7 +1211,11 @@ jobs: # GHCR on its own runner. It rebuilt the same Dockerfile as the step # below for two minutes. Validating the compose file is what is left. - name: Validate staging compose file - run: docker compose -f "$STAGING_COMPOSE_FILE" -p corescope-staging config --quiet + run: | + for key in POSTGRES_ADMIN_PASSWORD CORESCOPE_OWNER_PASSWORD CORESCOPE_READER_PASSWORD CORESCOPE_WRITER_PASSWORD CORESCOPE_ACCOUNTS_PASSWORD CORESCOPE_CHANNELS_PASSWORD; do + export "$key=$(openssl rand -hex 32)" + done + docker compose -f "$STAGING_COMPOSE_FILE" -p corescope-staging config --quiet - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -948,26 +1237,52 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max + - name: Load amd64 image for packaged Compose smoke (no publish) + uses: docker/build-push-action@v6 + with: + context: . + push: false + load: true + platforms: linux/amd64 + tags: corescope-pg-smoke:${{ github.sha }} + build-args: | + APP_VERSION=${{ steps.meta.outputs.app_version }} + GIT_COMMIT=${{ steps.meta.outputs.git_commit }} + BUILD_TIME=${{ steps.meta.outputs.build_time }} + cache-from: type=gha + - name: Smoke the arm64 binaries under QEMU + env: + APP_VERSION: ${{ steps.meta.outputs.app_version }} + GIT_COMMIT: ${{ steps.meta.outputs.git_commit }} + BUILD_TIME: ${{ steps.meta.outputs.build_time }} run: | set -e # Export the arm64 binaries so the runner's own `file` can confirm # they are static, then execute one under QEMU. Cheap: the build above # already populated the layer cache. docker buildx build . --platform linux/arm64 \ - --build-arg APP_VERSION=smoke --cache-from type=gha \ + --build-arg APP_VERSION="$APP_VERSION" \ + --build-arg GIT_COMMIT="$GIT_COMMIT" \ + --build-arg BUILD_TIME="$BUILD_TIME" --cache-from type=gha \ --output type=local,dest=./arm64-image - for b in corescope-server corescope-ingestor corescope-decrypt; do + for b in corescope-server corescope-ingestor corescope-decrypt corescope-migrate; do file "./arm64-image/app/$b" | tee /dev/stderr | grep -q 'statically linked' \ || { echo "::error::$b is not statically linked"; exit 1; } file "./arm64-image/app/$b" | grep -q 'ARM aarch64' \ || { echo "::error::$b is not an arm64 binary"; exit 1; } done - docker run --rm --platform linux/arm64 \ - -v "$PWD/arm64-image/app:/bin-under-test:ro" alpine:3.20 \ + docker run --rm --platform linux/arm64 --network none \ + -v "$PWD/arm64-image/app:/bin-under-test:ro" alpine:3.24 \ /bin-under-test/corescope-decrypt --version rm -rf ./arm64-image - echo "arm64 static binaries run under QEMU ✅" + echo "All four arm64 binaries are static; decrypt --version ran under QEMU." + + - name: Exercise packaged PostgreSQL Compose startup + timeout-minutes: 10 + run: | + timeout --kill-after=150s 10m bash scripts/test-postgres-compose.sh "corescope-pg-smoke:$GITHUB_SHA" sqlite + timeout --kill-after=150s 10m bash scripts/test-postgres-compose.sh "corescope-pg-smoke:$GITHUB_SHA" postgres # ─────────────────────────────────────────────────────────────── # 3b. Publish Docker Image — only after every test job passed @@ -996,10 +1311,13 @@ jobs: # a skipped required check as passing. race-test is skipped when no ingestor # Go file changed, so for it skipped is accepted; a master push always runs it. build-and-publish: + permissions: + contents: read + packages: write name: "🏗️ Build & Publish Docker Image" needs: [go-test, race-test, e2e-test, image-check, changes] - runs-on: ubuntu-latest - if: ${{ !cancelled() && needs.changes.outputs.code == 'true' && !(startsWith(github.ref, 'refs/tags/v') && inputs.images_published) }} + runs-on: ubuntu-24.04 + if: ${{ !cancelled() && needs.changes.outputs.code == 'true' && !(startsWith(github.ref, 'refs/tags/v') && inputs.images_published) && !inputs.postgres_benchmark }} steps: - name: Require every test job and the image check to have passed run: | @@ -1073,49 +1391,58 @@ jobs: release-artifacts: name: "📦 Release Artifacts" needs: [go-test, changes] - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 permissions: contents: write - if: startsWith(github.ref, 'refs/tags/v') && needs.changes.outputs.code == 'true' + if: (startsWith(github.ref, 'refs/tags/v') && needs.changes.outputs.code == 'true') && !inputs.postgres_benchmark steps: - name: Checkout code uses: actions/checkout@v5 - - name: Set up Go 1.27 + - name: Set up Go 1.27.2 uses: actions/setup-go@v6 with: - go-version: '1.27' + go-version: '1.27.2' cache-dependency-path: | cmd/decrypt/go.sum - internal/channel/go.sum + cmd/migrate/go.sum + internal/**/go.sum - # The SQLite driver is cgo now (github.com/mattn/go-sqlite3), so the Go - # toolchain alone cannot cross-compile these. zig cc is the C compiler - # that can target both architectures; musl makes the result static. + # Both release tools support native SQLite and need a cross C compiler. - name: Set up Zig uses: mlugg/setup-zig@v2 with: version: 0.16.0 - - name: Build corescope-decrypt (static, linux/amd64) - run: | - cd cmd/decrypt - CGO_ENABLED=1 GOOS=linux GOARCH=amd64 CC="zig cc -target x86_64-linux-musl" go build -trimpath -tags netgo,osusergo,sqlite_omit_load_extension -ldflags="-s -w -extldflags '-static -Wl,-s' -X main.version=${{ github.ref_name }}" -o ../../corescope-decrypt-linux-amd64 . - - - name: Build corescope-decrypt (static, linux/arm64) + - name: Build static release tools for both architectures run: | - cd cmd/decrypt - CGO_ENABLED=1 GOOS=linux GOARCH=arm64 CC="zig cc -target aarch64-linux-musl" go build -trimpath -tags netgo,osusergo,sqlite_omit_load_extension -ldflags="-s -w -extldflags '-static -Wl,-s' -X main.version=${{ github.ref_name }}" -o ../../corescope-decrypt-linux-arm64 . + set -euo pipefail + for arch in amd64 arm64; do + case "$arch" in amd64) target=x86_64-linux-musl ;; arm64) target=aarch64-linux-musl ;; esac + ( + cd cmd/decrypt + CGO_ENABLED=1 GOOS=linux GOARCH="$arch" CC="zig cc -target $target" \ + go build -trimpath -tags netgo,osusergo,sqlite_omit_load_extension \ + -ldflags="-s -w -extldflags '-static -Wl,-s' -X main.version=$GITHUB_REF_NAME" -o "../../corescope-decrypt-linux-$arch" . + ) + ( + cd cmd/migrate + CGO_ENABLED=1 GOOS=linux GOARCH="$arch" CC="zig cc -target $target" \ + go build -trimpath -tags netgo,osusergo,sqlite_omit_load_extension \ + -ldflags="-s -w -extldflags '-static -Wl,-s'" -o "../../corescope-migrate-linux-$arch" . + ) + done - name: Verify release binaries are static and runnable run: | set -e - for f in corescope-decrypt-linux-amd64 corescope-decrypt-linux-arm64; do + for f in corescope-{decrypt,migrate}-linux-{amd64,arm64}; do file "$f" | grep -q 'statically linked' || { echo "::error::$f is not statically linked"; exit 1; } done # amd64 runs natively on the runner; arm64 is only checked structurally # here (the container image job exercises arm64 under QEMU). ./corescope-decrypt-linux-amd64 --version + ./corescope-migrate-linux-amd64 -help - name: Locate the release notes for this tag id: notes @@ -1152,6 +1479,8 @@ jobs: files: | corescope-decrypt-linux-amd64 corescope-decrypt-linux-arm64 + corescope-migrate-linux-amd64 + corescope-migrate-linux-arm64 # ─────────────────────────────────────────────────────────────── # 4b. Deploy Staging (master only) @@ -1176,7 +1505,8 @@ jobs: # variable is set while the runner is still missing the job fails in ten # minutes rather than blocking the branch again. if: | - vars.ENABLE_STAGING_DEPLOY == 'true' + !inputs.postgres_benchmark + && vars.ENABLE_STAGING_DEPLOY == 'true' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref == 'refs/heads/master' needs: [build-and-publish] @@ -1267,18 +1597,15 @@ jobs: # ─────────────────────────────────────────────────────────────── publish: name: "📝 Publish Badges & Summary" - if: github.event_name == 'push' + if: github.event_name == 'push' && !inputs.postgres_benchmark # Was needs: [deploy]. The badges are built from the go-test and e2e-test # artifacts and never needed the deploy step; depending on it meant the # coverage badges stopped updating whenever the self-hosted runner was # unavailable. build-and-publish already transitively requires both test # jobs, so ordering is unchanged. needs: [build-and-publish] - runs-on: ubuntu-latest - # The badges go to their own `badges` branch (#2136). master is protected, - # so writing there took a PAT (BADGE_PUSH_TOKEN), which stopped working. - # The badges branch is unprotected, so the job's own GITHUB_TOKEN can - # write it. + runs-on: ubuntu-24.04 + # The unprotected badges branch uses this job's GITHUB_TOKEN (#2136). permissions: contents: write steps: diff --git a/.github/workflows/postgres-benchmark.yml b/.github/workflows/postgres-benchmark.yml new file mode 100644 index 000000000..1ccb47eb5 --- /dev/null +++ b/.github/workflows/postgres-benchmark.yml @@ -0,0 +1,136 @@ +name: PostgreSQL paired benchmark + +on: + workflow_dispatch: + inputs: + candidate_sha: + description: Exact 40-character candidate commit + type: string + required: true + workflow_call: + inputs: + candidate_sha: + type: string + required: true + +permissions: + contents: read + +concurrency: + group: postgres-benchmark-${{ github.run_id }} + cancel-in-progress: false + +jobs: + benchmark: + name: Five supplemental pairs (50 events/s, 128K transmissions / 2.048M observations) + runs-on: ubuntu-24.04 + timeout-minutes: 240 + env: + CANDIDATE_SHA: ${{ inputs.candidate_sha }} + CGO_ENABLED: '1' + GOMAXPROCS: '3' + steps: + - name: Validate exact candidate and runner capacity + run: | + set -euo pipefail + [[ "$CANDIDATE_SHA" =~ ^[0-9a-f]{40}$ ]] || { echo 'candidate_sha must be a full lowercase commit SHA' >&2; exit 1; } + [[ $(nproc) -ge 4 ]] || { echo 'At least four CPUs are required' >&2; exit 1; } + awk '/MemTotal:/ {exit !($2 >= 8*1024*1024)}' /proc/meminfo || { echo 'At least 8 GiB RAM is required' >&2; exit 1; } + test -f /sys/fs/cgroup/cgroup.controllers + echo "BENCH_OUTPUT=$RUNNER_TEMP/corescope-bench-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" >> "$GITHUB_ENV" + echo "SMOKE_OUTPUT=$RUNNER_TEMP/corescope-bench-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT-smoke" >> "$GITHUB_ENV" + + - name: Checkout the exact candidate + uses: actions/checkout@v5 + with: + repository: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name || github.repository }} + ref: ${{ inputs.candidate_sha }} + fetch-depth: 0 + persist-credentials: false + + - name: Verify candidate and workflow snapshot + env: + EXPECTED_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + WORKFLOW_SHA: ${{ github.workflow_sha }} + run: | + set -euo pipefail + test "$(git rev-parse HEAD)" = "$CANDIDATE_SHA" + test "$EXPECTED_SHA" = "$CANDIDATE_SHA" || { echo 'Candidate must match the PR head or dispatched workflow commit' >&2; exit 1; } + [[ "$WORKFLOW_SHA" =~ ^[0-9a-f]{40}$ ]] || { echo 'A full workflow commit SHA is required' >&2; exit 1; } + # PR workflows run from a merge ref, while the experiment measures the + # exact head. Refuse a merge that changes the executed workflow/harness. + git fetch --no-tags --depth=1 "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY.git" "$WORKFLOW_SHA" + git diff --quiet "$WORKFLOW_SHA" "$CANDIDATE_SHA" -- \ + .github/workflows/deploy.yml .github/workflows/postgres-benchmark.yml \ + scripts/postgres-benchmark scripts/install-postgres-ci.sh \ + || { echo 'Workflow or benchmark harness differs from candidate; update the candidate before rerunning' >&2; exit 1; } + git cat-file -e 3e21b179ecccae005aa5f6f45bc095353c94b343^{commit} + + - name: Test benchmark controls + run: python3 -m unittest discover -s scripts/postgres-benchmark -p 'test_*.py' + + - name: Use the same compiler for both revisions + uses: actions/setup-go@v6 + with: + go-version: '1.27.2' + cache-dependency-path: '**/go.sum' + + - name: Install exact PostgreSQL 18.6 binaries + run: bash scripts/install-postgres-ci.sh server + + - name: Provision owned cgroup and run supplemental comparison + run: | + set -euo pipefail + delegated="/sys/fs/cgroup/corescope-benchmark-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" + test ! -e "$delegated" + sudo mkdir "$delegated" + for controller in cpu memory io; do + grep -qw "$controller" "$delegated/cgroup.controllers" || { echo "Required cgroup controller unavailable: $controller" >&2; exit 1; } + done + printf '+cpu +memory +io\n' | sudo tee "$delegated/cgroup.subtree_control" >/dev/null + sudo mkdir "$delegated/coordinator" + sudo chown "$(id -u):$(id -g)" "$delegated" "$delegated/cgroup.procs" "$delegated/cgroup.subtree_control" "$delegated/coordinator" "$delegated/coordinator/cgroup.procs" + # The coordinator/generator live in a sibling of the measured app+DB + # group. The common ancestor is delegated so unprivileged child moves + # work; the runner validates the 3 CPU / 6 GiB measured envelope. + printf '%s\n' "$$" | sudo tee "$delegated/coordinator/cgroup.procs" >/dev/null + # Prove the controller and Linux prerequisites before preparing corpus B. + python3 scripts/postgres-benchmark/run.py --repo . \ + --candidate-sha "$CANDIDATE_SHA" --corpus S --pairs 1 --profile smoke \ + --ingest-rate 50 --http-rate 20 \ + --warmup 2 --seconds 10 --postgres-version 18.6 --go-version 1.27.2 \ + --postgres-bin /usr/lib/postgresql/18/bin \ + --output "$SMOKE_OUTPUT" --cgroup "$delegated" + python3 scripts/postgres-benchmark/run.py --repo . \ + --candidate-sha "$CANDIDATE_SHA" --corpus B --pairs 5 --profile primary \ + --ingest-rate 50 --http-rate 20 \ + --postgres-version 18.6 --go-version 1.27.2 \ + --postgres-bin /usr/lib/postgresql/18/bin \ + --output "$BENCH_OUTPUT" --cgroup "$delegated" + + - name: Upload only sanitized public evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: postgres-benchmark-${{ inputs.candidate_sha }} + path: | + ${{ env.SMOKE_OUTPUT }}/public/ + ${{ env.BENCH_OUTPUT }}/public/ + if-no-files-found: error + retention-days: 30 + + - name: Remove only this run's cgroup + if: always() + run: | + set -euo pipefail + delegated="/sys/fs/cgroup/corescope-benchmark-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" + if [ -d "$delegated" ]; then + test "$(realpath "$delegated")" = "$delegated" + if [ -f "$delegated/cgroup.kill" ]; then echo 1 | sudo tee "$delegated/cgroup.kill" >/dev/null; fi + for attempt in $(seq 1 10); do + sudo find "$delegated" -depth -type d -exec rmdir {} \; 2>/dev/null || true + test ! -d "$delegated" && break + sleep 1 + done + test ! -d "$delegated" + fi diff --git a/.gitignore b/.gitignore index 33def5772..6bc2d49ed 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ node_modules/ data/ +postgres-data/ +*.dump *.bak *.db *.db-journal diff --git a/AGENTS.md b/AGENTS.md index 293ba0426..189735883 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -13,10 +13,12 @@ Go backend + static frontend. No build step. No framework. No bundler. cmd/server/ — Go API server (REST + WebSocket broadcast + static file serving) main.go — Entry point, flags, SPA handler routes.go — All /api/* endpoints - store.go — In-memory packet store + analytics + SQLite queries + store.go — In-memory packet store + analytics + SQLite/PostgreSQL queries config.go — Configuration loading decoder.go — MeshCore packet decoder -cmd/ingestor/ — Go MQTT ingestor (separate binary, writes to shared SQLite DB) +cmd/ingestor/ — Go MQTT ingestor (separate binary, owns telemetry writes) +cmd/migrate/ — Offline schema bootstrap and verified SQLite upgrade importer +internal/pgutil/ — PostgreSQL connections, role checks, native backup helpers public/ — Frontend (vanilla JS, one file per page) — ACTIVE, NOT DEPRECATED app.js — SPA router, shared globals, theme loading roles.js — ROLE_COLORS, TYPE_COLORS, health thresholds, shared helpers @@ -34,36 +36,39 @@ public/ — Frontend (vanilla JS, one file per page) — ACTIVE, NOT live.css — Live page styles home.css — Home page styles index.html — SPA shell, script/style tags with __BUST__ placeholder (auto-replaced at server startup) -test-fixtures/ — Real data SQLite fixture from staging (used for E2E tests) +test-fixtures/ — SQLite fixture; prepare for both SQLite and PostgreSQL E2E scripts/ — Tooling (coverage collector, fixture capture, frontend instrumentation) ``` ### Data Flow -1. MQTT brokers → Go ingestor (`cmd/ingestor/`) ingests packets → decodes → writes to SQLite -2. Go server (`cmd/server/`) polls SQLite for new packets, broadcasts via WebSocket +1. MQTT brokers → Go ingestor (`cmd/ingestor/`) ingests packets → decodes → writes to the selected backend (SQLite by default, PostgreSQL optional) +2. Go server (`cmd/server/`) polls the selected backend for new packets, broadcasts via WebSocket 3. Frontend fetches via REST API (`/api/*`), filters/sorts client-side ### Read/Write Separation Invariant (#1283) -- **All DB writes live in `cmd/ingestor/`.** INSERT / UPDATE / DELETE / VACUUM / - schema migrations / retention all run in the ingestor process. -- **`cmd/server/` never writes measurement data.** It opens the analyzer DB with - `mode=ro` and must not acquire a write lock on it. Adding a write-side helper - (e.g. a `cachedRW`-style RW connection) regresses this invariant and races the - ingestor → SQLITE_BUSY. -- **Single exception: `users.db` (optional user management).** When - `userManagement.enabled`, the server owns a *separate* SQLite file through - `internal/users` only. `users.Open` refuses the analyzer DB path, and - `TestUsersOpenIsTheOnlyServerWritePath` pins the one call site. Account data - never goes into the analyzer DB, and measurement writes never go through - `internal/users`. -- **The ingestor reads `users.db`, never writes it.** With - `userManagement.channelProposals.enabled` the ingestor opens `users.db` - read-only (`mode=ro`, raw SQL, no `internal/users` import) once a minute - for the approved hashtag channel names. With - `clientRxCoverage.requireLinkedCompanion` it reads `companion_links` the same - way (`SELECT pubkey`, every 60 s, plus at most one re-read per 5 s on a miss). - `TestChannelKeySetIsReadOnly` and `TestLinkedCompanionSetIsReadOnly` pin the - read-only open. +- **Runtime telemetry writes live in `cmd/ingestor/`.** INSERT / UPDATE / + DELETE and retention use its SQLite connection or restricted PostgreSQL + writer role. The ingestor owns SQLite maintenance; PostgreSQL handles + autovacuum. Offline `cmd/migrate/` owns storage selection/conversion and + PostgreSQL schema installation and upgrades. +- **`cmd/server/` never writes measurement data.** Its telemetry connection + opens SQLite with `mode=ro`, or uses a SELECT-only PostgreSQL role and + read-only transactions. Do not add a write-side connection or grant the + reader schema/owner privileges. +- **Single exception: the separate accounts database (optional user + management).** When `userManagement.enabled`, the server writes accounts + through `internal/users` only. `users.Open` rejects the actual telemetry + database, including URL aliases, and `TestUsersOpenIsTheOnlyServerWritePath` + pins the one call site. Account and measurement databases stay separate. +- **The ingestor reads approved channel names and linked public keys, never + private account data.** Its SQLite accounts connection uses `mode=ro`. + PostgreSQL uses the dedicated projection reader for `approved_channels`, + the public-key-only `linked_companions` view and the readiness marker; + it cannot read the underlying account, session or challenge tables. + Read-only snapshots retain the last good data if refresh fails. +- **PostgreSQL runtime roles never run DDL or mark an import ready.** Bootstrap/import + uses separate owner credentials. Tests must cover denied writes and + cross-database access with the real runtime grants. - Enforcement: `cmd/server/readonly_invariant_test.go` reflect-asserts that `PruneOldPackets`, `PruneOldMetrics`, and `RemoveStaleObservers` are NOT methods on the server's `*DB`. If you need a new write, add it to @@ -105,9 +110,22 @@ Every change must consider performance impact BEFORE implementation. This codeba No proof = no merge. ### 1. No commit without tests -Every change that touches logic MUST have tests. For Go backend: `cd cmd/server && go test ./...` and `cd cmd/ingestor && go test ./...`, or `make test` for all 14 modules. For frontend: `node tests/unit/test-packet-filter.js && node tests/unit/test-aging.js && node tests/unit/test-frontend-helpers.js`. If you add new logic, add tests. No exceptions. - -The SQLite driver (`github.com/mattn/go-sqlite3`) is cgo. **`CGO_ENABLED=0` still *builds*** — that is the trap. It links a stub, and the binary dies on the first query with `go-sqlite3 requires cgo to work. This is a stub`, so a green build proves nothing. A plain `GOOS=linux go build` likewise cannot cross-compile it. Use `make build` / `make crossbuild` — the latter needs [`zig`](https://ziglang.org/download/) as the cross C compiler and produces static musl binaries. Never re-add `CGO_ENABLED=0`. +Every change that touches logic MUST have tests. For Go backend: `cd cmd/server && go test ./...` and `cd cmd/ingestor && go test ./...`, or `make test` for all Go modules. For frontend: `node tests/unit/test-packet-filter.js && node tests/unit/test-aging.js && node tests/unit/test-frontend-helpers.js`. If you add new logic, add tests. No exceptions. + +Both backends ship in the same binaries. PostgreSQL uses native Go (`pgx`); +SQLite runtime and the offline importer require cgo +(`github.com/mattn/go-sqlite3`). `CGO_ENABLED=0` links a stub that fails on +the first SQLite query. Use `make build` / `make crossbuild` for the complete +release, including the importer. Cross-builds require +[`zig`](https://ziglang.org/download/) or another suitable C cross compiler. +Validate real SQLite startup and import, not just a successful build. + +Default database tests use SQLite and need no external service. The explicit +`CORESCOPE_TEST_BACKEND=postgres` matrix requires a disposable PostgreSQL 18 instance and +`CORESCOPE_TEST_POSTGRES_URL` with test schema/database/role creation rights. +They fail when the database is unavailable rather than silently skipping. +Never point tests at a live instance. Runtime credentials stay in deployment +environment variables and must not appear in logs or committed fixtures. ### 2. No commit without browser validation After pushing, verify the change works in an actual browser. Use `browser profile=openclaw` against the running instance. Take a screenshot if the change is visual. If you can't validate it, say so — don't claim it works. diff --git a/DEPLOY.md b/DEPLOY.md index af011657a..4a71895eb 100644 --- a/DEPLOY.md +++ b/DEPLOY.md @@ -4,33 +4,38 @@ Pre-built images are published to GHCR for `linux/amd64` and `linux/arm64` (Rasp ## Quick Start -### Docker run +### Complete Compose checkout + +SQLite is the default and runs inside the application container. PostgreSQL18.6 is optional. Existing installations keep their recorded backend; changing it is a verified [offline storage switch](docs/storage.md), not an image update. + +Set `CORESCOPE_REF` to the reviewed application revision, then retain the complete checkout: the Compose variants require `docker/postgres.compose.yml` and its initialization scripts. ```bash -docker run -d --name corescope \ - -p 80:80 \ - -v corescope-data:/app/data \ - -e DISABLE_CADDY=true \ - ghcr.io/kpa-clawbot/corescope:latest +git clone https://github.com/Kpa-clawbot/CoreScope.git +cd CoreScope +git checkout --detach "$CORESCOPE_REF" +test -f .env || cp .env.example .env +chmod 600 .env +# SQLite needs no database credentials. Preserve an existing .env and DATA_DIR. +# Set CORESCOPE_IMAGE in .env to the matching reviewed image tag or digest. +docker compose -f docker-compose.example.yml up -d ``` -Open `http://localhost` — done. +Packaged setup validates an existing SQLite installation or initializes a provably fresh one before starting the application. Open `http://localhost` and verify `/api/healthz` plus real packet ingestion. The default HTTP port is 80; adjust `HTTP_PORT` when using the example variant. -### Docker Compose +For a **new PostgreSQL installation**, merge `.env.postgres.example` into the private `.env`, generate distinct passwords only for the first empty PostgreSQL directory, and run: -```bash -curl -sL https://raw.githubusercontent.com/Kpa-clawbot/CoreScope/master/docker-compose.example.yml \ - -o docker-compose.yml -docker compose up -d +```sh +docker compose -f docker-compose.example.yml -f docker-compose.example.postgres.yml up -d ``` +The optional overlay adds PostgreSQL and owner bootstrap. Runtime services receive only restricted role credentials. Adding/removing an overlay does not convert existing data; use [storage switching](docs/storage.md). Keep the same overlay on PostgreSQL updates. + ## Image Tags | Tag | Description | |-----|-------------| -| `v3.4.1` | Pinned release (recommended for production) | -| `v3.4` | Latest patch in v3.4.x | -| `v3` | Latest minor+patch in v3.x | +| Reviewed release tag or image digest | Pin the application and its matching repository files together | | `latest` | Latest release tag | | `edge` | Built from master — unstable, for testing | @@ -40,7 +45,7 @@ Settings can be overridden via environment variables: | Variable | Default | Description | |----------|---------|-------------| -| `DISABLE_CADDY` | `false` | Skip internal Caddy (set `true` behind a reverse proxy) | +| `DISABLE_CADDY` | `false` in the example | Caddy forwards published port 80 to the Go server on 3000. If disabled, also change your proxy/port mapping to target 3000. | | `DISABLE_MOSQUITTO` | `true` in `docker-compose.staging.yml`; `false` elsewhere | Skip internal MQTT broker. Default flipped to `true` for the staging deploy in v3.7+ because a standalone `mqtt-broker` container owns MQTT on that host — see "Standalone MQTT broker (staging)" below. | | `HTTP_PORT` | `80` | Host port mapping | | `DATA_DIR` | `./data` | Host path for persistent data | @@ -50,22 +55,19 @@ For advanced configuration, mount a `config.json` into `/app/data/config.json`. ## Updating ```bash -docker compose pull -docker compose up -d +docker compose -f docker-compose.example.yml pull +docker compose -f docker-compose.example.yml up -d ``` ## Data -All persistent data lives in `/app/data`: -- `meshcore.db` — SQLite database (packets, nodes) -- `config.json` — custom config (optional) -- `theme.json` — custom theme (optional) +SQLite stores telemetry and accounts as separate files under the data mount. When selected, PostgreSQL persists them in its own host directory (`POSTGRES_DATA_DIR`, or the production/staging equivalent). `/app/data` holds config, theme, queues, statistics and account backup files. Keep these locations separate and preserve the private `.env` with your recovery material. -**Backup:** `cp data/meshcore.db ~/backup/` +Use native SQLite snapshots or PostgreSQL `pg_dump` archives, selected automatically by the recorded backend, or the authenticated backup endpoints. `./manage.sh backup ` and `restore` operate the production Compose variant managed by that script; they do not select `docker-compose.example.yml`. The [backup and restore instructions](docs/postgresql-upgrade.md#native-backups-and-restores) include the example variant. Do not copy live PostgreSQL storage or a SQLite main file while ignoring its WAL. Preserve the private state archive and configuration with the native backup pair. ## TLS -Option A — **External reverse proxy** (recommended): Run with `DISABLE_CADDY=true`, put nginx/traefik/Cloudflare in front. +Option A — **External reverse proxy**: Keep the default internal Caddy and proxy to the published HTTP port. Alternatively, set `DISABLE_CADDY=true` and have the proxy reach container port 3000; change the Compose port mapping from `:80` to `:3000` if the proxy runs on the host. Setting the flag alone leaves the existing port-80 mapping without a listener. Option B — **Built-in Caddy**: Mount a custom Caddyfile at `/etc/caddy/Caddyfile` and expose ports 80+443. @@ -134,13 +136,13 @@ your fork or ensure the network exists. If you're currently deploying with `manage.sh` (git clone + local build), you have two options going forward: -### Option A: Keep using manage.sh (no changes needed) +### Option A: Keep using manage.sh -`manage.sh update` continues to work exactly as before — it fetches the latest tag, builds locally, and restarts. Nothing breaks. +`manage.sh setup` offers SQLite (default) or PostgreSQL. Start/update use the recorded selection, and storage changes require `./manage.sh storage switch sqlite|postgres` with writers stopped. Backups/restores use the selected engine's native tools; existing staging data is retained, and only telemetry is cloned into a fresh staging installation. See [storage operations](docs/storage.md). ```bash ./manage.sh update # latest release -./manage.sh update v3.5.0 # specific version +./manage.sh update "$CORESCOPE_REF" # reviewed version ``` ### Option B: Switch to pre-built images (recommended) @@ -154,16 +156,18 @@ Pre-built images skip the build step entirely — faster updates, no Go toolchai ./manage.sh stop ``` -2. Your data is in `~/meshcore-data/` (or whatever `PROD_DATA_DIR` is set to). It's untouched — the database, config, and theme files persist. +2. Preserve `PROD_DATA_DIR`, the separate PostgreSQL directory, credentials and the native backup pair. Keep the selected backend and its recovery files; use an explicit offline switch only when changing engines. -3. Copy `docker-compose.example.yml` to where you want to run from: +3. Keep the complete pinned checkout and choose the matching image in `.env`: ```bash - cp docker-compose.example.yml ~/docker-compose.yml + # CORESCOPE_IMAGE= + # DATA_DIR= + # POSTGRES_DATA_DIR= ``` 4. Start with the pre-built image: ```bash - cd ~ && docker compose up -d + docker compose -f docker-compose.example.yml up -d ``` 5. Verify it picked up your existing data: @@ -173,19 +177,20 @@ Pre-built images skip the build step entirely — faster updates, no Go toolchai **Updates after migration:** ```bash -docker compose pull && docker compose up -d +docker compose -f docker-compose.example.yml pull +docker compose -f docker-compose.example.yml up -d ``` ### What about manage.sh features? | manage.sh command | Pre-built equivalent | |---|---| -| `./manage.sh update` | `docker compose pull && docker compose up -d` | -| `./manage.sh stop` | `docker compose down` | -| `./manage.sh start` | `docker compose up -d` | -| `./manage.sh logs` | `docker compose logs -f` | -| `./manage.sh status` | `docker compose ps` | -| `./manage.sh setup` | Copy `docker-compose.example.yml`, edit env vars | +| `./manage.sh update` | `docker compose -f docker-compose.example.yml pull` then `up -d` with the same file | +| `./manage.sh stop` | `docker compose -f docker-compose.example.yml stop corescope` | +| `./manage.sh start` | `docker compose -f docker-compose.example.yml up -d` | +| `./manage.sh logs` | `docker compose -f docker-compose.example.yml logs -f` | +| `./manage.sh status` | `docker compose -f docker-compose.example.yml ps` | +| `./manage.sh setup` | Retain the full checkout, configure private credentials and select the Compose variant | `manage.sh` remains available for advanced use cases (building from source, custom patches, development). Pre-built images are recommended for most production deployments. diff --git a/Dockerfile b/Dockerfile index 35499abe1..4fe547e47 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,11 +2,9 @@ # Build stage always runs natively on the builder's arch ($BUILDPLATFORM) and # cross-compiles to $TARGETOS/$TARGETARCH. No QEMU for compilation. # -# The SQLite driver is github.com/mattn/go-sqlite3, which is cgo, so the Go -# toolchain alone can no longer cross-compile this: it needs a C compiler that -# can target the other architecture. `zig cc` is that compiler. Targeting musl -# makes the result fully static (see -extldflags below), so the runtime stage -# has no libc dependency on the base image at all. +# All four binaries support native SQLite and PostgreSQL. SQLite requires cgo; +# the upstream Zig cross-compiler targets musl so both release architectures +# remain fully static and independent of the runtime image libc. # # BUILDPLATFORM is auto-set by buildx; default to linux/amd64 so plain # `docker build` (without buildx) doesn't fail on an empty platform string. @@ -20,9 +18,8 @@ ARG BUILD_TIME=unknown ARG TARGETOS ARG TARGETARCH -# Keep these in step with the Makefile: netgo/osusergo preserve the pure-Go -# resolver and user lookup the binaries had under CGO_ENABLED=0, and -# sqlite_omit_load_extension drops the dlopen path so -static links cleanly. +# Keep these in step with the Makefile. netgo/osusergo retain Go DNS/user +# lookup; sqlite_omit_load_extension permits static linking. ENV GO_BUILD_TAGS=netgo,osusergo,sqlite_omit_load_extension \ ZIG_GLOBAL_CACHE_DIR=/tmp/zig-cache @@ -42,8 +39,7 @@ RUN apk add --no-cache curl xz && \ ln -s /opt/zig/zig /usr/local/bin/zig && rm /tmp/zig.tar.xz && \ zig version -# zigcc resolves TARGETARCH to a zig target triple once, so the three build -# steps below stay readable and cannot disagree with each other. +# zigcc selects the native SQLite C target from the requested image architecture. RUN printf '%s\n' '#!/bin/sh' \ 'case "$TARGETARCH" in' \ ' amd64) t=x86_64-linux-musl ;;' \ @@ -68,6 +64,8 @@ COPY internal/lora/ ../../internal/lora/ COPY internal/users/ ../../internal/users/ COPY internal/channel/ ../../internal/channel/ COPY internal/mailer/ ../../internal/mailer/ +COPY internal/pgutil/ ../../internal/pgutil/ +COPY internal/sqliteutil/ ../../internal/sqliteutil/ RUN --mount=type=cache,target=/root/.cache/go-build \ --mount=type=cache,target=/go/pkg/mod \ --mount=type=cache,target=/tmp/zig-cache \ @@ -93,6 +91,7 @@ COPY internal/dbschema/ ../../internal/dbschema/ COPY internal/prunequeue/ ../../internal/prunequeue/ COPY internal/perfio/ ../../internal/perfio/ COPY internal/mbcapqueue/ ../../internal/mbcapqueue/ +COPY internal/pgutil/ ../../internal/pgutil/ RUN --mount=type=cache,target=/root/.cache/go-build \ --mount=type=cache,target=/go/pkg/mod \ --mount=type=cache,target=/tmp/zig-cache \ @@ -110,6 +109,9 @@ RUN --mount=type=cache,target=/root/.cache/go-build \ WORKDIR /build/decrypt COPY cmd/decrypt/go.mod cmd/decrypt/go.sum ./ COPY internal/channel/ ../../internal/channel/ +COPY internal/dbschema/ ../../internal/dbschema/ +COPY internal/pgutil/ ../../internal/pgutil/ +COPY internal/dbconfig/ ../../internal/dbconfig/ RUN --mount=type=cache,target=/root/.cache/go-build \ --mount=type=cache,target=/go/pkg/mod \ --mount=type=cache,target=/tmp/zig-cache \ @@ -123,10 +125,31 @@ RUN --mount=type=cache,target=/root/.cache/go-build \ -ldflags "-s -w -extldflags '-static -Wl,-s' -X main.version=${APP_VERSION}" \ -o /corescope-decrypt . -# Runtime image -FROM alpine:3.20 +# Offline bootstrap and verified conversion CLI for both storage engines. +WORKDIR /build/migrate +COPY cmd/migrate/go.mod cmd/migrate/go.sum ./ +COPY internal/packetpath/ ../../internal/packetpath/ +COPY internal/dbschema/ ../../internal/dbschema/ +COPY internal/users/ ../../internal/users/ +COPY internal/pgutil/ ../../internal/pgutil/ +COPY internal/sqliteutil/ ../../internal/sqliteutil/ +COPY internal/dbconfig/ ../../internal/dbconfig/ +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + go mod download +COPY cmd/migrate/ ./ +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + --mount=type=cache,target=/tmp/zig-cache \ + GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ + go build -trimpath -tags ${GO_BUILD_TAGS} \ + -ldflags "-s -w -extldflags '-static -Wl,-s'" \ + -o /corescope-migrate . + +# The client major matches PostgreSQL 18 for native backup/restore support. +FROM alpine:3.24 -RUN apk add --no-cache mosquitto mosquitto-clients supervisor caddy wget +RUN apk add --no-cache mosquitto mosquitto-clients supervisor caddy wget postgresql18-client sqlite jq # Account for the Go services when a deployment opts into running them # unprivileged with RUN_AS_UID=1000, the usual choice. Creating it costs @@ -148,7 +171,7 @@ ENV CORESCOPE_SERVICE_USER=root WORKDIR /app # Go binaries (statically linked; they do not use this image's libc) -COPY --from=builder /corescope-server /corescope-ingestor /corescope-decrypt /app/ +COPY --from=builder /corescope-server /corescope-ingestor /corescope-decrypt /corescope-migrate /app/ # Frontend assets + config COPY public/ ./public/ @@ -172,9 +195,10 @@ RUN mkdir -p /app/data /var/lib/mosquitto /data/caddy && \ # Entrypoint COPY docker/entrypoint-go.sh /entrypoint.sh +COPY docker/storage.sh docker/postgres-bootstrap.sh docker/postgres-grants.sql /app/ # Sourced by the entrypoint, so deliberately not executable. COPY docker/select-service-user.sh /select-service-user.sh -RUN chmod +x /entrypoint.sh +RUN chmod +x /entrypoint.sh /app/storage.sh /app/postgres-bootstrap.sh EXPOSE 80 443 1883 diff --git a/Dockerfile.go b/Dockerfile.go deleted file mode 100644 index ec2725f52..000000000 --- a/Dockerfile.go +++ /dev/null @@ -1,61 +0,0 @@ -FROM golang:1.27-alpine AS builder - -RUN apk add --no-cache build-base - -ARG APP_VERSION=unknown -ARG GIT_COMMIT=unknown -ARG BUILD_TIME=unknown - -# Build server -WORKDIR /build/server -COPY cmd/server/go.mod cmd/server/go.sum ./ -RUN go mod download -COPY cmd/server/ ./ -RUN go build -ldflags "-X main.Version=${APP_VERSION} -X main.Commit=${GIT_COMMIT} -X main.BuildTime=${BUILD_TIME}" -o /corescope-server . - -# Build ingestor -WORKDIR /build/ingestor -COPY cmd/ingestor/go.mod cmd/ingestor/go.sum ./ -RUN go mod download -COPY cmd/ingestor/ ./ -RUN go build -o /corescope-ingestor . - -# Runtime image -FROM alpine:3.20 - -RUN apk add --no-cache mosquitto mosquitto-clients supervisor caddy wget - -WORKDIR /app - -# Go binaries -COPY --from=builder /corescope-server /corescope-ingestor /app/ - -# Frontend assets + config -COPY public/ ./public/ -COPY config.example.json channel-rainbow.json ./ - -# Bake git commit SHA (CI writes .git-commit before build; fallback for non-ldflags usage) -COPY .git-commi[t] ./ -RUN if [ ! -f .git-commit ]; then echo "unknown" > .git-commit; fi - -# Supervisor + Mosquitto + Caddy config -COPY docker/supervisord-go.conf /etc/supervisor/conf.d/supervisord.conf -COPY docker/supervisord-go-no-mosquitto.conf /etc/supervisor/conf.d/supervisord-no-mosquitto.conf -COPY docker/supervisord-go-no-caddy.conf /etc/supervisor/conf.d/supervisord-no-caddy.conf -COPY docker/supervisord-go-no-mosquitto-no-caddy.conf /etc/supervisor/conf.d/supervisord-no-mosquitto-no-caddy.conf -COPY docker/mosquitto.conf /etc/mosquitto/mosquitto.conf -COPY docker/Caddyfile /etc/caddy/Caddyfile - -# Data directory -RUN mkdir -p /app/data /var/lib/mosquitto /data/caddy && \ - chown -R mosquitto:mosquitto /var/lib/mosquitto - -# Entrypoint -COPY docker/entrypoint-go.sh /entrypoint.sh -RUN chmod +x /entrypoint.sh - -EXPOSE 80 443 1883 - -VOLUME ["/app/data", "/data/caddy"] - -ENTRYPOINT ["/entrypoint.sh"] diff --git a/Makefile b/Makefile index 34f71ccd5..51037f30f 100644 --- a/Makefile +++ b/Makefile @@ -1,13 +1,8 @@ # corescope build entry point. # -# This repo is 16 Go modules wired with `replace ../../internal/*` and no -# go.work, so every recipe changes into its module. The build flags live here -# because the SQLite driver is cgo now (github.com/mattn/go-sqlite3) and the -# flags stopped being something you can safely retype at each call site. -# -# Cross-compilation uses `zig cc` as the C toolchain, targeting musl so the -# binaries are fully static and run on the alpine runtime image (or scratch) -# with no libc dependency at all. +# Each Go module resolves internal dependencies through local replace directives. +# SQLite and PostgreSQL are supported by the same binaries. Native SQLite +# needs cgo; Linux cross-builds use zig cc with musl for fully static binaries. # # Quick reference: # make build # all four binaries for the host @@ -21,21 +16,16 @@ GIT_VERSION ?= $(shell git describe --tags --match "v*" 2>/dev/null || ech GIT_COMMIT ?= $(shell git rev-parse --short HEAD 2>/dev/null || echo unknown) BUILD_TIME ?= $(shell date -u +%Y-%m-%dT%H:%M:%SZ) -GOENV_GOOS := $(shell $(GO) env GOOS) -GOENV_GOARCH := $(shell $(GO) env GOARCH) +GOENV_GOOS := $(shell "$(GO)" env GOOS) +GOENV_GOARCH := $(shell "$(GO)" env GOARCH) GOOS ?= $(GOENV_GOOS) GOARCH ?= $(GOENV_GOARCH) -# Exported: a bare make variable never reaches the go toolchain. -export CGO_ENABLED ?= 1 - -# netgo/osusergo keep the pure-Go resolver and user lookup the binaries had -# under CGO_ENABLED=0, so enabling cgo for SQLite does not quietly switch DNS -# resolution to the C resolver. sqlite_omit_load_extension drops the dlopen -# path, which is what lets -extldflags -static link cleanly. -Wl,-s on the -# cross builds strips the musl objects zig links in: Go's own -s -w does not -# reach them, and they account for well over half the binary. -GO_BUILD_TAGS ?= netgo,osusergo,sqlite_omit_load_extension +CGO_server := 1 +CGO_ingestor := 1 +CGO_decrypt := 1 +CGO_migrate := 1 +GO_BUILD_TAGS ?= netgo,osusergo,sqlite_omit_load_extension GO_BUILD_FLAGS ?= -trimpath GO_LDFLAGS_OPTIMS ?= -s -w @@ -53,6 +43,7 @@ LDFLAGS_migrate := # zig target triples for the platforms we ship. musl, so the result is static. ZIG_TARGET_linux_amd64 := x86_64-linux-musl ZIG_TARGET_linux_arm64 := aarch64-linux-musl +comma := , CROSS_PLATFORMS := linux/amd64 linux/arm64 DOCKER_IMAGE ?= ghcr.io/kpa-clawbot/corescope @@ -73,7 +64,7 @@ build: $(addprefix build-,$(CMDS)) build-%: @mkdir -p $(DIST) - cd cmd/$* && GOOS=$(GOOS) GOARCH=$(GOARCH) $(GO) build \ + cd cmd/$* && CGO_ENABLED=$(CGO_$*) GOOS=$(GOOS) GOARCH=$(GOARCH) "$(GO)" build \ -tags $(GO_BUILD_TAGS) $(GO_BUILD_FLAGS) \ -ldflags "$(GO_LDFLAGS_OPTIMS) $(LDFLAGS_$*)" \ -o ../../$(DIST)/corescope-$* . @@ -89,10 +80,10 @@ define CROSSBUILD_RULE .PHONY: crossbuild-$(2)-$(3)-$(4) crossbuild-$(2)-$(3)-$(4): @mkdir -p $$(DIST) - @command -v zig >/dev/null || { echo "zig not found: needed to cross-compile cgo. See https://ziglang.org/download/"; exit 1; } - cd cmd/$(2) && CGO_ENABLED=1 GOOS=$(3) GOARCH=$(4) CC="zig cc -target $(1)" \ - $$(GO) build -tags $$(GO_BUILD_TAGS) $$(GO_BUILD_FLAGS) \ - -ldflags '$$(GO_LDFLAGS_OPTIMS) -extldflags "-static -Wl,-s" $$(LDFLAGS_$(2))' \ + @command -v zig >/dev/null || { echo "zig not found: needed to cross-compile native SQLite. See https://ziglang.org/download/"; exit 1; } + cd cmd/$(2) && CGO_ENABLED=$$(CGO_$(2)) GOOS=$(3) GOARCH=$(4) CC="zig cc -target $(1)" \ + "$$(GO)" build -tags $$(GO_BUILD_TAGS) $$(GO_BUILD_FLAGS) \ + -ldflags '$$(GO_LDFLAGS_OPTIMS) -extldflags "-static -Wl$(comma)-s" $$(LDFLAGS_$(2))' \ -o ../../$$(DIST)/corescope-$(2)-$(3)-$(4) . endef @@ -103,13 +94,13 @@ $(foreach c,$(CMDS),\ # -- checks -------------------------------------------------------------------- test: - bash scripts/allmod.sh test ./... + CGO_ENABLED=1 bash scripts/allmod.sh test ./... vet: bash scripts/allmod.sh vet ./... fmt-check: - @unformatted=$$(gofmt -l $$(git ls-files '*.go' | grep -vi 'Dockerfile.go')); \ + @unformatted=$$(gofmt -l $$(git ls-files '*.go')); \ if [ -n "$$unformatted" ]; then echo "gofmt required on:"; echo "$$unformatted"; exit 1; fi; \ echo "gofmt: clean" diff --git a/README.md b/README.md index 5b3822600..1da586204 100644 --- a/README.md +++ b/README.md @@ -6,24 +6,15 @@ [![Frontend Coverage](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/Kpa-clawbot/CoreScope/badges/frontend-coverage.json)](https://github.com/Kpa-clawbot/CoreScope/actions/workflows/deploy.yml) [![Deploy](https://github.com/Kpa-clawbot/CoreScope/actions/workflows/deploy.yml/badge.svg)](https://github.com/Kpa-clawbot/CoreScope/actions/workflows/deploy.yml) -> High-performance mesh network analyzer powered by Go. Sub-millisecond packet queries, ~300 MB memory for 56K+ packets, real-time WebSocket broadcast, full channel decryption. +> MeshCore packet analysis with a Go backend, SQLite persistence by default, with optional PostgreSQL, real-time WebSocket updates and channel decryption. Self-hosted, open-source MeshCore packet analyzer. Collects MeshCore packets via MQTT, decodes them in real time, and presents a full web UI with live packet feed, interactive maps, channel chat, packet tracing, and per-node analytics. ## ⚡ Performance -The Go backend serves its packet and analytics API from an in-memory packet store with 7 indexes (hash, txID, obsID, observer, node, path hop, payload type). SQLite is for persistence only — reads never touch disk. +The Go backend combines a packet cache with SQLite or PostgreSQL storage and SQL-backed analytics. SQLite is the default. The ingestor owns telemetry writes; the server reads telemetry and uses a separate optional account store. Ordinary updates keep the recorded backend. Use the [offline storage workflow](docs/storage.md) to change engines without discarding existing data. -| Metric | Value | -|--------|-------| -| Packet queries | **< 1 ms** (in-memory) | -| All API endpoints | **< 100 ms** | -| Memory (56K packets) | **~300 MB** (vs 1.3 GB on Node.js) | -| WebSocket broadcast | **Real-time** to all connected browsers | -| Channel decryption | **AES-128-ECB** with rainbow table | -| GOMEMLIMIT (memory-constrained hosts) | **set to ≥1.5× working set** (e.g. 1536 MiB on a 2 GB Pi for a ~1 GB store). Lower values trigger a GC death-spiral. Configure via the `GOMEMLIMIT` env var or `runtime.maxMemoryMB` in `config.json`; env wins. Applies to both server and ingestor. See [#1010](https://github.com/Kpa-clawbot/CoreScope/issues/1010). | - -See [PERFORMANCE.md](PERFORMANCE.md) for full benchmarks. +Budget for the application and PostgreSQL together, including retained data, indexes, WAL and the packet cache. Historical SQLite measurements in [PERFORMANCE.md](PERFORMANCE.md) are baseline context, not PostgreSQL results. Use the [paired benchmark workflow](.github/workflows/postgres-benchmark.yml) for an exact candidate comparison. ## ✨ Features @@ -83,50 +74,32 @@ Off by default, enabled in `config.json`: ## Quick Start -### Pre-built Image (Recommended) +### Pre-built image — SQLite by default -No build step required — just run: - -```bash -docker run -d --name corescope \ - --restart=unless-stopped \ - -p 80:80 -p 1883:1883 \ - -v /your/data:/app/data \ - ghcr.io/kpa-clawbot/corescope:latest -``` - -Open `http://localhost` — done. No config file needed; CoreScope starts with sensible defaults. - -For HTTPS with a custom domain, add `-p 443:443` and mount your Caddyfile: -```bash -docker run -d --name corescope \ - --restart=unless-stopped \ - -p 80:80 -p 443:443 -p 1883:1883 \ - -v /your/data:/app/data \ - -v /your/Caddyfile:/etc/caddy/Caddyfile:ro \ - -v /your/caddy-data:/data/caddy \ - ghcr.io/kpa-clawbot/corescope:latest -``` - -Disable built-in services with `-e DISABLE_MOSQUITTO=true` or `-e DISABLE_CADDY=true`, or drop a `.env` file in your data volume. See [docs/deployment.md](docs/deployment.md) for the full reference. - -### Build from Source +Retain the complete repository at the same reviewed revision as the image. The optional PostgreSQL overrides use the matching shared initialization scripts. ```bash git clone https://github.com/Kpa-clawbot/CoreScope.git cd CoreScope -./manage.sh setup +git checkout --detach "$CORESCOPE_REF" +test -f .env || cp .env.example .env +chmod 600 .env +# No PostgreSQL service or credentials are needed for the default SQLite install. +# Set CORESCOPE_IMAGE to the matching reviewed image tag or digest. +docker compose -f docker-compose.example.yml up -d ``` -The setup wizard walks you through config, domain, HTTPS, build, and run. +Open `http://localhost` and verify `/api/healthz` plus actual ingestion. Packaged setup validates existing SQLite data or initializes a provably fresh installation before the application starts. Keep the existing `DATA_DIR` on updates. For a new PostgreSQL install, merge `.env.postgres.example` into the private `.env` and add `-f docker-compose.example.postgres.yml`; existing installations use the verified [storage switch](docs/storage.md). See [DEPLOY.md](DEPLOY.md) for ports, MQTT, HTTPS and backups. + +### Build from source + +Use the same complete checkout and private `.env`, then run `./manage.sh setup`. The wizard offers SQLite (default) or PostgreSQL, configures, builds and starts the deployment. It keeps an existing recorded backend; PostgreSQL credentials are generated only for a new empty database directory, preserving existing private settings. ```bash -./manage.sh status # Health check + packet/node counts -./manage.sh logs # Follow logs -./manage.sh backup # Backup database -./manage.sh update # Pull latest + rebuild + restart -./manage.sh mqtt-test # Check if observer data is flowing -./manage.sh help # All commands +./manage.sh status +./manage.sh logs +./manage.sh backup ./backups/instance +./manage.sh update ``` ### Configure @@ -164,34 +137,39 @@ Copy `config.example.json` to `config.json` and edit: | `mqttSources` | External MQTT broker connections (optional) | | `channelKeys` | Channel decryption keys (hex). Hashtag channels auto-derived via SHA256 | | `defaultRegion` | Default IATA region code for the UI | -| `dbPath` | SQLite database path (default: `data/meshcore.db`) | +| `db.backend` | Initial choice: `sqlite` (default) or `postgres`; the installation record wins after setup | +| `dbPath` | SQLite telemetry path; `DB_PATH` is its environment override | +| `databaseURL` | Optional PostgreSQL telemetry URL; prefer the per-process role credentials below | +| `stateDir` | Filesystem directory for queues, statistics and account backups (default: `data`) | ### Environment Variables | Variable | Description | |----------|-------------| -| `DB_PATH` | Override SQLite database path | +| `DB_PATH` | Bootstrap SQLite telemetry path; recorded targets take precedence after setup | | `GOMEMLIMIT` | Go soft memory limit, wins over `runtime.maxMemoryMB` | +| `CORESCOPE_DB_BACKEND` | Bootstrap choice only; use an offline switch to change an installed backend | +| `CORESCOPE_READER_DATABASE_URL` | Server telemetry reader URL | +| `CORESCOPE_WRITER_DATABASE_URL` | Ingestor telemetry writer URL | +| `CORESCOPE_USERS_DATABASE_URL` | Optional account writer URL in a separate database | +| `CORESCOPE_APPROVED_CHANNELS_DATABASE_URL` | Ingestor's restricted approved-channel and linked-companion projection reader URL | +| `CORESCOPE_STATE_DIR` | Shared state-directory path; never a database URL | The HTTP port is set with `port` in `config.json` or the server's `-port` flag. ## Architecture -``` - ┌─────────────────────────────────────────────┐ - │ Docker Container │ - │ │ -Observer → USB → │ Mosquitto ──→ Go Ingestor ──→ SQLite DB │ - meshcoretomqtt → MQTT ──→│ │ │ - │ Go HTTP Server ──→ WebSocket │ - │ │ │ │ - │ Caddy (HTTPS) ←───────┘ │ - └────────────────────┼────────────────────────┘ - │ - Browser +```mermaid +flowchart LR + MQTT[MQTT observers] --> Ingestor[Go ingestor] + Ingestor -->|telemetry writer| Telemetry[(SQLite or PostgreSQL telemetry)] + Telemetry -->|read-only role| Server[Go API server] + Server <-->|account writer| Accounts[(Separate account database)] + Accounts -->|approved-channel view| Ingestor + Server <-->|REST and WebSocket| Browser ``` -**Two-process model:** The Go ingestor handles MQTT ingestion and packet decoding. The Go HTTP server loads all packets into an in-memory store on startup (5 indexes for fast lookups) and serves the REST API + WebSocket broadcast. Both are managed by supervisord inside a single container with Caddy for HTTPS and Mosquitto for local MQTT. +**Two-process model:** The ingestor handles MQTT, decoding, telemetry writes and retention. The HTTP server loads its packet cache and serves REST/WebSocket requests using a read-only telemetry role. Accounts use a separate SQLite file or PostgreSQL database. Supervisord runs the application processes with Caddy and optional Mosquitto; PostgreSQL runs as a separate service only when selected. Bootstrap/conversion credentials are not runtime credentials. ## MQTT Setup @@ -211,14 +189,14 @@ corescope/ │ │ ├── main.go # Entry point │ │ ├── routes.go # 40+ API endpoint handlers │ │ ├── store.go # In-memory packet store (5 indexes) -│ │ ├── db.go # SQLite persistence layer +│ │ ├── db.go # Read-only native storage access │ │ ├── decoder.go # MeshCore packet decoder │ │ ├── websocket.go # WebSocket broadcast │ │ └── *_test.go # Server tests │ └── ingestor/ # Go MQTT ingestor │ ├── main.go # MQTT subscription + packet processing │ ├── decoder.go # Packet decoder (shared logic) -│ ├── db.go # SQLite write path +│ ├── db.go # Native writer and retention path │ └── *_test.go # Ingestor tests ├── proto/ # Protobuf API definitions ├── public/ # Vanilla JS frontend (no build step) @@ -253,9 +231,9 @@ make build-server # just one make crossbuild # static linux/amd64 + linux/arm64 binaries ``` -The SQLite driver is [`mattn/go-sqlite3`](https://github.com/mattn/go-sqlite3), which -is cgo, so a plain `GOOS=linux go build` from a Mac will not work: cross-compiling -needs a C compiler that can target the other platform. `make crossbuild` uses +Runtime database access uses pgx through `database/sql`. The offline importer retains +the cgo [`mattn/go-sqlite3`](https://github.com/mattn/go-sqlite3) driver for legacy +sources, so cross-compiling that binary needs a C compiler. `make crossbuild` uses [`zig`](https://ziglang.org/download/) as that compiler (install it and it just works) and links statically against musl, so the result is one self-contained file that runs on Alpine or scratch. The container build does the same thing — see @@ -263,14 +241,19 @@ that runs on Alpine or scratch. The container build does the same thing — see ### Test Suite -Go tests cover the backend, Node.js unit tests in `tests/unit/` cover the frontend logic, and Playwright tests in `tests/e2e/` validate the UI in a browser. +Default Go suites exercise native SQLite without a PostgreSQL service. The explicit PostgreSQL matrix covers role boundaries, concurrency, migration and native backup/restore. The standalone frontend list remains `test-all.sh`; Playwright covers the imported fixture and accounts on/off. CI pins Go 1.27.2 and PostgreSQL 18.6. ```bash +# Default SQLite tests need no service. For the separate PostgreSQL matrix: +export CORESCOPE_TEST_BACKEND=postgres +# Point this at an isolated PostgreSQL18 administrator database. +# Keep the URL in private environment configuration; pg_dump/pg_restore 18 must be on PATH. +# CORESCOPE_TEST_POSTGRES_URL is required; database tests do not silently skip. # Go backend tests cd cmd/server && go test ./... -v cd cmd/ingestor && go test ./... -v -# Or across all 16 modules at once +# Or across all modules make test # Node.js frontend + integration tests @@ -288,7 +271,7 @@ node tools/generate-packets.js --api --count 200 ### Migrating from Node.js -If you're running an existing Node.js deployment, see [docs/go-migration.md](docs/go-migration.md) for a step-by-step guide. The Go engine reads the same SQLite database and `config.json` — no data migration needed. +Historical Node.js/SQLite deployments must first reach a supported SQLite layout using the prior version's explicit upgrade procedure. They can keep SQLite. Use the [offline storage switch](docs/storage.md) only when changing engines. Preserve original data, configuration and keys; a container/image update does not perform a backend conversion. ## Contributing diff --git a/cmd/decrypt/README.md b/cmd/decrypt/README.md index d6bf5ee78..68d8b47c3 100644 --- a/cmd/decrypt/README.md +++ b/cmd/decrypt/README.md @@ -1,6 +1,6 @@ # corescope-decrypt -Standalone CLI tool to decrypt and export MeshCore hashtag channel messages from a CoreScope SQLite database. +Standalone CLI tool to decrypt and export MeshCore hashtag channel messages from CoreScope storage. SQLite is the default; PostgreSQL is optional. Both are opened read-only. ## Why @@ -24,7 +24,7 @@ docker exec corescope-prod /app/corescope-decrypt --channel "#wardriving" --db / ### From GitHub release -Download the static binary from the [Releases](https://github.com/Kpa-clawbot/CoreScope/releases) page: +Download the binary for your platform from the [Releases](https://github.com/Kpa-clawbot/CoreScope/releases) page: ```bash # Linux amd64 @@ -40,16 +40,28 @@ cd cmd/decrypt go build -ldflags="-s -w" -o corescope-decrypt . # cgo: the SQLite driver needs it ``` -The binary is statically linked — no dependencies, runs on any Linux. +Building SQLite support requires a C compiler and CGO enabled. Use the packaged image or a release binary matching your platform. ## Usage ``` -corescope-decrypt --channel NAME --db PATH [--format FORMAT] [--output FILE] +corescope-decrypt --channel NAME [--db PATH] [--format FORMAT] [--output FILE] ``` Run `corescope-decrypt --help` for full flag documentation. +### Storage selection + +The tool reads storage fields from `config.json`, then `data/config.json` if the first file is absent. `--config FILE` selects an explicit file. An unreadable or malformed selected config is an error. Relative paths use the working directory. + +For an installed instance, `storage-selection.json` is authoritative: stale `--db` paths or bootstrap backend settings do not override a completed backend switch. The tool finds that record using `--state-dir`, `CORESCOPE_STATE_DIR`, or configured `stateDir`; otherwise it uses the legacy SQLite database directory (default `data`). Keep the installation's stable state directory when exporting after a switch. A shared lease blocks a switch until the export exits. + +Without a recorded installation, `--db PATH` reads an existing SQLite file; the default is `data/meshcore.db`. It never creates, adopts, or migrates a database. Incomplete conversions, missing files, and incompatible schemas fail. Offline legacy files remain readable when they contain the columns needed for export. + +For PostgreSQL, supply a restricted reader URL through `CORESCOPE_READER_DATABASE_URL`, or the generic `CORESCOPE_DATABASE_URL`. `--database-url` is available but environment settings keep credentials out of command history. PostgreSQL role and schema-readiness checks remain enforced; a failed connection never falls back to SQLite. + +Before a selection has been recorded, `db.backend`, `CORESCOPE_DB_BACKEND`, or `--backend sqlite|postgres` makes the choice explicit; conflicting config/environment selectors fail. Otherwise unambiguous native settings determine the engine. CLI paths/URLs override their environment/config values; environment values override config. `DB_PATH` is the legacy SQLite path variable. Mixed unrecorded file and URL settings require an explicit backend choice. + ### JSON output (default) Machine-readable, includes all metadata (observers, path hops, raw hex): @@ -123,6 +135,9 @@ See the firmware source at `firmware/src/helpers/BaseChatMesh.cpp` for the canon cd cmd/decrypt go test ./... +# Explicit PostgreSQL matrix: requires a disposable administrator URL. +CORESCOPE_TEST_BACKEND=postgres go test ./... + # Manual test with the real fixture: go run . --channel "#wardriving" --db ../../test-fixtures/e2e-fixture.db --format irc ``` diff --git a/cmd/decrypt/cli_test.go b/cmd/decrypt/cli_test.go new file mode 100644 index 000000000..0d13ef233 --- /dev/null +++ b/cmd/decrypt/cli_test.go @@ -0,0 +1,80 @@ +package main + +import ( + "bytes" + "encoding/json" + "flag" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// Run the real flag/storage/query/render path without building another binary. +func TestDecryptCLIProcess(t *testing.T) { + if os.Getenv("CORESCOPE_DECRYPT_TEST_CHILD") != "1" { + return + } + for i, arg := range os.Args { + if arg == "--" { + os.Args = append([]string{"corescope-decrypt"}, os.Args[i+1:]...) + break + } + } + flag.CommandLine = flag.NewFlagSet("corescope-decrypt", flag.ExitOnError) + main() + os.Exit(0) +} + +func runDecryptCLI(t *testing.T, base string, args ...string) ([]byte, string, error) { + t.Helper() + executable, err := os.Executable() + if err != nil { + t.Fatal(err) + } + cmd := exec.Command(executable, append([]string{"-test.run=^TestDecryptCLIProcess$", "--"}, args...)...) + cmd.Dir = base + for _, value := range os.Environ() { + key, _, _ := strings.Cut(value, "=") + switch key { + case "DB_PATH", "CORESCOPE_DB_BACKEND", "CORESCOPE_DATABASE_URL", "CORESCOPE_READER_DATABASE_URL", "CORESCOPE_STATE_DIR", "CORESCOPE_DECRYPT_TEST_CHILD": + continue + } + cmd.Env = append(cmd.Env, value) + } + cmd.Env = append(cmd.Env, "CORESCOPE_DECRYPT_TEST_CHILD=1") + var out, diagnostic bytes.Buffer + cmd.Stdout, cmd.Stderr = &out, &diagnostic + err = cmd.Run() + return out.Bytes(), diagnostic.String(), err +} + +func TestDecryptNativeCLI(t *testing.T) { + base := t.TempDir() + data := filepath.Join(base, "data") + if err := os.Mkdir(data, 0700); err != nil { + t.Fatal(err) + } + path := filepath.Join(data, "meshcore.db") + db := sqliteExportFixture(t, path, true) + for _, args := range [][]string{ + {"--channel", "#example"}, + {"--channel", "#example", "--db", path}, + } { + out, diagnostic, err := runDecryptCLI(t, base, args...) + if err != nil || !json.Valid(out) || !strings.Contains(diagnostic, "Scanned 0 GRP_TXT packets") { + t.Fatalf("native CLI export failed: %v; %s", err, diagnostic) + } + } + db.Close() + if err := os.Remove(path); err != nil { + t.Fatal(err) + } + if _, _, err := runDecryptCLI(t, base, "--channel", "#example"); err == nil { + t.Fatal("default CLI accepted a missing database") + } + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Fatal("default CLI created a missing database") + } +} diff --git a/cmd/decrypt/go.mod b/cmd/decrypt/go.mod index 226dbcf0e..fc8c6bb40 100644 --- a/cmd/decrypt/go.mod +++ b/cmd/decrypt/go.mod @@ -1,10 +1,29 @@ module github.com/corescope/decrypt -go 1.22 +go 1.25.0 require ( - github.com/mattn/go-sqlite3 v1.14.52 github.com/meshcore-analyzer/channel v0.0.0 + github.com/meshcore-analyzer/dbschema v0.0.0 + github.com/meshcore-analyzer/pgutil v0.0.0 +) + +require ( + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/pgx/v5 v5.7.6 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/mattn/go-sqlite3 v1.14.52 + github.com/meshcore-analyzer/dbconfig v0.0.0 + golang.org/x/crypto v0.37.0 // indirect + golang.org/x/sync v0.13.0 // indirect + golang.org/x/text v0.24.0 // indirect ) replace github.com/meshcore-analyzer/channel => ../../internal/channel + +replace github.com/meshcore-analyzer/dbschema => ../../internal/dbschema + +replace github.com/meshcore-analyzer/pgutil => ../../internal/pgutil + +replace github.com/meshcore-analyzer/dbconfig => ../../internal/dbconfig diff --git a/cmd/decrypt/go.sum b/cmd/decrypt/go.sum index 75bf6eb53..c83dde8cc 100644 --- a/cmd/decrypt/go.sum +++ b/cmd/decrypt/go.sum @@ -1,2 +1,30 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.7.6 h1:rWQc5FwZSPX58r1OQmkuaNicxdmExaEz5A2DO2hUuTk= +github.com/jackc/pgx/v5 v5.7.6/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/mattn/go-sqlite3 v1.14.52 h1:wVbm2Qnf4OXkqhBTSPuCRZDRnxfbVrrmiCEroVdog8U= github.com/mattn/go-sqlite3 v1.14.52/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= +golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= +golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= +golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= +golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/cmd/decrypt/main.go b/cmd/decrypt/main.go index 8c897d6ff..48583de9d 100644 --- a/cmd/decrypt/main.go +++ b/cmd/decrypt/main.go @@ -1,8 +1,8 @@ -// corescope-decrypt decrypts and exports hashtag channel messages from a CoreScope SQLite database. +// corescope-decrypt decrypts and exports hashtag channel messages from CoreScope storage. // // Usage: // -// corescope-decrypt --channel "#wardriving" --db meshcore.db [--format json|html] [--output file] +// corescope-decrypt --channel "#wardriving" [--format json|html] [--output file] package main import ( @@ -18,8 +18,8 @@ import ( "strings" "time" - _ "github.com/mattn/go-sqlite3" "github.com/meshcore-analyzer/channel" + "github.com/meshcore-analyzer/dbconfig" ) // Version info (set via ldflags). @@ -47,7 +47,11 @@ type Observer struct { func main() { channelName := flag.String("channel", "", "Channel name (e.g. \"#wardriving\")") - dbPath := flag.String("db", "", "Path to CoreScope SQLite database") + databaseURL := flag.String("database-url", "", "PostgreSQL reader URL (prefer CORESCOPE_READER_DATABASE_URL)") + dbPath := flag.String("db", "", "Existing SQLite database path (default: data/meshcore.db)") + backend := flag.String("backend", "", "Bootstrap backend: sqlite or postgres (installed selection wins)") + stateDir := flag.String("state-dir", "", "Installation state directory (CORESCOPE_STATE_DIR)") + configPath := flag.String("config", "", "Storage config file (default: config.json, then data/config.json)") format := flag.String("format", "json", "Output format: json, html, irc (or log)") output := flag.String("output", "", "Output file (default: stdout)") showVersion := flag.Bool("version", false, "Print version and exit") @@ -56,12 +60,17 @@ func main() { fmt.Fprintf(os.Stderr, `corescope-decrypt — Decrypt and export MeshCore hashtag channel messages USAGE - corescope-decrypt --channel NAME --db PATH [--format FORMAT] [--output FILE] + corescope-decrypt --channel NAME [--format FORMAT] [--output FILE] FLAGS --channel NAME Channel name to decrypt (e.g. "#wardriving", "wardriving") The "#" prefix is added automatically if missing. - --db PATH Path to a CoreScope SQLite database file (read-only access). + --db PATH Existing SQLite database (default: data/meshcore.db). + --backend NAME Bootstrap choice: sqlite or postgres. + --state-dir DIR Find the installed storage selection in this directory. + --config FILE Read storage settings from this config file. + --database-url PostgreSQL reader URL. Prefer CORESCOPE_READER_DATABASE_URL + or CORESCOPE_DATABASE_URL to keep credentials out of argv. --format FORMAT Output format (default: json): json — Machine-readable JSON array with full metadata html — Self-contained HTML viewer with search and sorting @@ -70,19 +79,22 @@ FLAGS --output FILE Write output to FILE instead of stdout. --version Print version and exit. + The installed selection takes precedence over bootstrap paths and backend flags. + Export holds a shared selection lease and never creates or migrates a database. + EXAMPLES # Export #wardriving messages as JSON - corescope-decrypt --channel "#wardriving" --db /app/data/meshcore.db + corescope-decrypt --channel "#wardriving" # Generate an interactive HTML viewer - corescope-decrypt --channel wardriving --db meshcore.db --format html --output wardriving.html + corescope-decrypt --channel wardriving --format html --output wardriving.html # Greppable IRC log - corescope-decrypt --channel "#MeshCore" --db meshcore.db --format irc --output meshcore.log + corescope-decrypt --channel "#MeshCore" --format irc --output meshcore.log grep "KE6QR" meshcore.log # From the Docker container - docker exec corescope-prod /app/corescope-decrypt --channel "#wardriving" --db /app/data/meshcore.db + docker exec corescope-prod /app/corescope-decrypt --channel "#wardriving" RETROACTIVE DECRYPTION MeshCore hashtag channels use symmetric encryption — the key is derived from the @@ -107,7 +119,7 @@ LIMITATIONS os.Exit(0) } - if *channelName == "" || *dbPath == "" { + if *channelName == "" { flag.Usage() os.Exit(1) } @@ -121,14 +133,21 @@ LIMITATIONS key := channel.DeriveKey(ch) chHash := channel.ChannelHash(key) - db, err := sql.Open("sqlite3", "file:"+*dbPath+"?mode=ro") + raw, err := exportStorageInputs(".", exportOptions{Backend: dbconfig.Backend(*backend), DBPath: *dbPath, DatabaseURL: *databaseURL, StateDir: *stateDir, ConfigPath: *configPath}, os.Getenv) + if err != nil { + log.Fatalf("Storage configuration: %v", err) + } + db, lease, err := openSelectedExport(raw) if err != nil { log.Fatalf("Failed to open database: %v", err) } + if lease != nil { + defer lease.Close() + } defer db.Close() // Query all GRP_TXT packets - rows, err := db.Query(`SELECT id, hash, raw_hex, first_seen FROM transmissions WHERE payload_type = 5`) + rows, err := db.Query(exportTransmissionsSQL) if err != nil { log.Fatalf("Query failed: %v", err) } @@ -278,7 +297,7 @@ func extractGRPPayload(rawHex string) ([]byte, error) { func getPathFromDB(db *sql.DB, txID int) []string { var decodedJSON sql.NullString - err := db.QueryRow(`SELECT decoded_json FROM transmissions WHERE id = ?`, txID).Scan(&decodedJSON) + err := db.QueryRow(exportPathSQL, txID).Scan(&decodedJSON) if err != nil || !decodedJSON.Valid { return nil } @@ -295,13 +314,7 @@ func getPathFromDB(db *sql.DB, txID int) []string { } func getObservers(db *sql.DB, txID int) []Observer { - rows, err := db.Query(` - SELECT o.name, obs.snr, obs.rssi, obs.timestamp - FROM observations obs - LEFT JOIN observers o ON o.id = CAST(obs.observer_idx AS TEXT) - WHERE obs.transmission_id = ? - ORDER BY obs.timestamp - `, txID) + rows, err := db.Query(exportObserversSQL, txID) if err != nil { return nil } diff --git a/cmd/decrypt/main_test.go b/cmd/decrypt/main_test.go index 76588a458..5d173a80f 100644 --- a/cmd/decrypt/main_test.go +++ b/cmd/decrypt/main_test.go @@ -4,6 +4,7 @@ import ( "encoding/hex" "encoding/json" "os" + "path/filepath" "strings" "testing" @@ -113,17 +114,27 @@ func TestJSONOutputParseable(t *testing.T) { } } -// Integration test against fixture DB (skipped if DB not found) +// Exercise the actual CLI against the checked-in, captured packet fixture. func TestFixtureDecrypt(t *testing.T) { - dbPath := "../../test-fixtures/e2e-fixture.db" - if _, err := os.Stat(dbPath); os.IsNotExist(err) { - t.Skip("fixture DB not found") + dbPath, err := filepath.Abs("../../test-fixtures/e2e-fixture.db") + if err != nil { + t.Fatal(err) } - - // We know the fixture has #wardriving messages with channelHash 0x81 - key := channel.DeriveKey("#wardriving") - ch := channel.ChannelHash(key) - if ch != 0x81 { - t.Fatalf("unexpected channel hash: %02X", ch) + if _, err := os.Stat(dbPath); err != nil { + t.Fatal("checked-in fixture database is required") + } + base := t.TempDir() + out, diagnostic, err := runDecryptCLI(t, base, "--channel", "#wardriving", "--db", dbPath, "--state-dir", base) + if err != nil { + t.Fatalf("fixture export failed: %v; %s", err, diagnostic) + } + var messages []ChannelMessage + if err := json.Unmarshal(out, &messages); err != nil || len(messages) == 0 { + t.Fatalf("fixture export produced no valid messages: %v", err) + } + for _, message := range messages { + if message.Channel != "#wardriving" || message.RawHex == "" || message.Timestamp == "" { + t.Fatal("fixture export lost message fields") + } } } diff --git a/cmd/decrypt/postgres_test.go b/cmd/decrypt/postgres_test.go new file mode 100644 index 000000000..9441d1e70 --- /dev/null +++ b/cmd/decrypt/postgres_test.go @@ -0,0 +1,64 @@ +package main + +import ( + "os" + "path/filepath" + "testing" + + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/pgutil" + "github.com/meshcore-analyzer/pgutil/pgtest" +) + +func TestPostgresExportReadsPathAndObservers(t *testing.T) { + if backend := os.Getenv("CORESCOPE_TEST_BACKEND"); backend != "postgres" { + if backend != "" && backend != "sqlite" { + t.Fatal("CORESCOPE_TEST_BACKEND must be sqlite or postgres") + } + t.Skip("PostgreSQL matrix is not selected") + } + dsn := pgtest.NewSchema(t) + writer, err := pgutil.Open(dsn, false) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { writer.Close() }) + if err := dbschema.Apply(writer, t.Logf); err != nil { + t.Fatal(err) + } + for _, query := range []string{ + `INSERT INTO transmissions(id,raw_hex,hash,first_seen,decoded_json) VALUES(1,'00','export-fixture','2026-01-01','{"path":{"hops":["ab","cd"]}}')`, + `INSERT INTO observers(rowid,id,name) VALUES(10,'observer','Example observer')`, + `INSERT INTO observations(transmission_id,observer_idx,snr,rssi,timestamp,path_json) VALUES(1,10,1.5,-90,1767225600,'["ab"]')`, + } { + if _, err := writer.Exec(query); err != nil { + t.Fatal(err) + } + } + reader, err := openExportDB(pgtest.ReadOnly(t, dsn)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { reader.Close() }) + assertExportMetadata(t, reader) + if _, err := reader.Exec(`DELETE FROM observations`); err == nil { + t.Fatal("PostgreSQL export reader permitted a write") + } + base := t.TempDir() + readerURL := pgtest.ReadOnly(t, dsn) + recordExportStorage(t, dbconfig.Storage{Backend: dbconfig.Postgres, StateDir: base, ReaderDatabaseURL: readerURL}) + selected, lease, err := openSelectedExport(dbconfig.StorageInputs{BaseDir: base, DBPath: filepath.Join(base, "stale.db"), Backend: dbconfig.SQLite, ReaderDatabaseURL: readerURL}) + if err != nil { + t.Fatal(err) + } + assertExportMetadata(t, selected) + selected.Close() + lease.Close() + if _, err := writer.Exec(`UPDATE corescope_schema SET ready=false`); err != nil { + t.Fatal(err) + } + if db, lease, err := openSelectedExport(dbconfig.StorageInputs{BaseDir: base, StateDir: base, ReaderDatabaseURL: readerURL}); err == nil || db != nil || lease != nil { + t.Fatal("export accepted an incomplete PostgreSQL import") + } +} diff --git a/cmd/decrypt/storage.go b/cmd/decrypt/storage.go new file mode 100644 index 000000000..51f4762ad --- /dev/null +++ b/cmd/decrypt/storage.go @@ -0,0 +1,173 @@ +package main + +import ( + "database/sql" + "errors" + "io" + "net/url" + "os" + "path/filepath" + "strings" + + _ "github.com/mattn/go-sqlite3" + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/pgutil" +) + +const exportTransmissionsSQL = `SELECT id, hash, raw_hex, first_seen FROM transmissions WHERE payload_type = 5` +const exportPathSQL = `SELECT decoded_json FROM transmissions WHERE id = $1` +const exportObserversSQL = ` + SELECT o.name, obs.snr, obs.rssi, obs.timestamp + FROM observations obs + LEFT JOIN observers o ON o.rowid = obs.observer_idx + WHERE obs.transmission_id = $1 + ORDER BY obs.timestamp, obs.id` + +type exportOptions struct { + Backend dbconfig.Backend + DBPath, DatabaseURL, StateDir, ConfigPath string +} + +// Use the installation's storage fields without depending on the server's full +// configuration. Only absent search candidates fall through to defaults. +func exportStorageInputs(base string, flags exportOptions, getenv func(string) string) (dbconfig.StorageInputs, error) { + base, err := filepath.Abs(base) + if err != nil { + return dbconfig.StorageInputs{}, err + } + paths := []string{filepath.Join(base, "config.json"), filepath.Join(base, "data", "config.json")} + if flags.ConfigPath != "" { + paths = []string{flags.ConfigPath} + } + var raw dbconfig.StorageInputs + for _, path := range paths { + raw, err = dbconfig.ReadStorageConfig(path) + if err == nil { + break + } + if flags.ConfigPath != "" || !errors.Is(err, os.ErrNotExist) { + return raw, err + } + } + raw.BaseDir = base + raw.EnvBackend = dbconfig.Backend(getenv("CORESCOPE_DB_BACKEND")) + for key, target := range map[string]*string{ + "DB_PATH": &raw.DBPath, "CORESCOPE_STATE_DIR": &raw.StateDir, + "CORESCOPE_DATABASE_URL": &raw.DatabaseURL, "CORESCOPE_READER_DATABASE_URL": &raw.ReaderDatabaseURL, + } { + if value := strings.TrimSpace(getenv(key)); value != "" { + *target = value + } + } + if flags.Backend != "" { + raw.Backend = flags.Backend + } + if flags.DBPath != "" { + raw.DBPath = flags.DBPath + } + if flags.DatabaseURL != "" { + raw.DatabaseURL, raw.ReaderDatabaseURL = flags.DatabaseURL, flags.DatabaseURL + } + if flags.StateDir != "" { + raw.StateDir = flags.StateDir + } + return raw, nil +} + +func openSelectedExport(raw dbconfig.StorageInputs) (*sql.DB, io.Closer, error) { + // The legacy SQLite directory remains the discovery anchor after a switch. + state := raw.StateDir + if state == "" { + path := raw.DBPath + if path == "" { + path = filepath.Join("data", "meshcore.db") + } + if strings.Contains(path, "://") || strings.HasPrefix(path, "postgres:") || strings.HasPrefix(path, "postgresql:") { + return nil, nil, errors.New("--db requires a SQLite filesystem path") + } + state = filepath.Dir(path) + } + if strings.Contains(state, "://") || strings.ContainsRune(state, 0) || (filepath.VolumeName(state) != "" && !filepath.IsAbs(state)) { + return nil, nil, errors.New("invalid storage state directory") + } + if !filepath.IsAbs(state) { + state = filepath.Join(raw.BaseDir, state) + } + selected, lease, err := dbconfig.OpenSelection(dbconfig.SelectionPath(filepath.Clean(state))) + if err == nil { + raw, err = selected.ApplyTo(raw) + } else if errors.Is(err, dbconfig.ErrSelectionMissing) { + // An offline/legacy export can read an existing source without adopting + // it. This default is never permission to create or migrate a database. + err = nil + if raw.Backend == "" && raw.EnvBackend == "" && raw.DBPath == "" && raw.UsersDBPath == "" && raw.DatabaseURL == "" && raw.ReaderDatabaseURL == "" && raw.WriterDatabaseURL == "" && raw.UsersDatabaseURL == "" && raw.ApprovedChannelsDatabaseURL == "" { + raw.DBPath = filepath.Join("data", "meshcore.db") + } + } + var db *sql.DB + if err == nil { + var storage dbconfig.Storage + storage, err = dbconfig.ResolveStorage(raw) + if err == nil { + target := storage.DBPath + if storage.Backend == dbconfig.Postgres { + target = storage.ReaderDatabaseURL + } + db, err = openExportDB(target) + } + } + if err != nil && lease != nil { + lease.Close() + lease = nil + } + return db, lease, err +} + +func openExportDB(target string) (*sql.DB, error) { + postgres := strings.HasPrefix(target, "postgres://") || strings.HasPrefix(target, "postgresql://") + var db *sql.DB + var err error + if postgres { + db, err = pgutil.Open(target, true) + } else { + if target == "" || strings.Contains(target, "://") || strings.HasPrefix(target, "postgres:") || strings.HasPrefix(target, "postgresql:") { + return nil, errors.New("export requires a SQLite filesystem path or PostgreSQL reader URL") + } + var dsn string + dsn, err = dbconfig.SQLiteURI(target, url.Values{"mode": {"ro"}, "_query_only": {"on"}, "_busy_timeout": {"5000"}}) + if err == nil { + db, err = sql.Open("sqlite3", dsn) + } + } + if err != nil { + return nil, err + } + db.SetMaxOpenConns(4) + db.SetMaxIdleConns(4) + if postgres { + err = pgutil.AssertReadOnly(db) + if err == nil { + err = dbschema.AssertPostgresReady(db) + } + } else { + err = dbconfig.AssertSQLiteImportComplete(db) + // Export needs only its existing read columns, including on older + // offline snapshots. Do not run migrations or require writer indexes. + for _, query := range []string{exportTransmissionsSQL, exportPathSQL, exportObserversSQL} { + if err != nil { + break + } + var stmt *sql.Stmt + stmt, err = db.Prepare(query) + if err == nil { + err = stmt.Close() + } + } + } + if err != nil { + db.Close() + return nil, err + } + return db, nil +} diff --git a/cmd/decrypt/storage_test.go b/cmd/decrypt/storage_test.go new file mode 100644 index 000000000..2b342eda3 --- /dev/null +++ b/cmd/decrypt/storage_test.go @@ -0,0 +1,306 @@ +package main + +import ( + "bytes" + "database/sql" + "errors" + "net" + "os" + "path/filepath" + "strings" + "testing" + + _ "github.com/mattn/go-sqlite3" + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/dbschema/legacy" +) + +func sqliteExportFixture(t *testing.T, path string, current bool) *sql.DB { + t.Helper() + dsn, err := legacy.WriterDSN(path) + if err != nil { + t.Fatal(err) + } + db, err := sql.Open("sqlite3", dsn) + if err != nil { + t.Fatal(err) + } + db.SetMaxOpenConns(1) + t.Cleanup(func() { db.Close() }) + if current { + err = dbschema.ApplySQLite(db, t.Logf) + } else if err = legacy.ApplyBase(db); err == nil { + err = legacy.Apply(db, t.Logf) + } + if err != nil { + t.Fatal(err) + } + for _, query := range []string{ + `INSERT INTO transmissions(id,raw_hex,hash,first_seen,decoded_json) VALUES(1,'00','export-fixture','2026-01-01','{"path":{"hops":["ab","cd"]}}')`, + `INSERT INTO observers(rowid,id,name) VALUES(10,'observer','Example observer')`, + `INSERT INTO observations(transmission_id,observer_idx,snr,rssi,timestamp,path_json) VALUES(1,10,1.5,-90,1767225600,'["ab"]')`, + } { + if _, err := db.Exec(query); err != nil { + t.Fatal(err) + } + } + return db +} + +func assertExportMetadata(t *testing.T, db *sql.DB) { + t.Helper() + path := getPathFromDB(db, 1) + if len(path) != 2 || path[0] != "ab" || path[1] != "cd" { + t.Fatalf("path query lost data: %v", path) + } + observers := getObservers(db, 1) + if len(observers) != 1 || observers[0].Name != "Example observer" || observers[0].SNR != 1.5 || observers[0].RSSI != -90 || observers[0].Timestamp != "2026-01-01T00:00:00Z" { + t.Fatalf("observer query lost data: %+v", observers) + } +} + +func TestSQLiteExportReadsPathAndObservers(t *testing.T) { + for _, current := range []bool{false, true} { + name := "legacy" + if current { + name = "current" + } + t.Run(name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "telemetry #percent%.db") + sqliteExportFixture(t, path, current) + db, err := openExportDB(path) + if err != nil { + t.Fatalf("native SQLite export should open its existing source: %v", err) + } + defer db.Close() + assertExportMetadata(t, db) + if _, err := db.Exec(`DELETE FROM observations`); err == nil { + t.Fatal("export reader permitted a write") + } + var queryOnly int + if err := db.QueryRow(`PRAGMA query_only`).Scan(&queryOnly); err != nil || queryOnly != 1 { + t.Fatalf("query_only=%d err=%v", queryOnly, err) + } + }) + } +} + +func TestSQLiteExportRejectsMissingEmptyWrongAndPendingSources(t *testing.T) { + for _, kind := range []string{"missing", "missing-directory", "empty", "header-only", "wrong-schema", "pending"} { + t.Run(kind, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "telemetry.db") + switch kind { + case "missing-directory": + path = filepath.Join(filepath.Dir(path), "absent", "telemetry.db") + case "empty": + if err := os.WriteFile(path, nil, 0600); err != nil { + t.Fatal(err) + } + case "header-only", "wrong-schema": + dsn, err := dbconfig.SQLiteURI(path, nil) + if err != nil { + t.Fatal(err) + } + db, err := sql.Open("sqlite3", dsn) + if err != nil { + t.Fatal(err) + } + query := `PRAGMA user_version=123` + if kind == "wrong-schema" { + query = `CREATE TABLE keep(value TEXT); INSERT INTO keep VALUES('preserve')` + } + if _, err := db.Exec(query); err != nil { + t.Fatal(err) + } + db.Close() + case "pending": + db := sqliteExportFixture(t, path, true) + if _, err := db.Exec(`CREATE TABLE corescope_reverse_progress(value TEXT)`); err != nil { + t.Fatal(err) + } + db.Close() + } + before, beforeErr := os.ReadFile(path) + db, err := openExportDB(path) + if err == nil { + db.Close() + t.Fatal("invalid source was accepted") + } + if kind == "pending" && !errors.Is(err, dbconfig.ErrSQLiteImportIncomplete) { + t.Fatalf("pending import guard lost: %v", err) + } + after, afterErr := os.ReadFile(path) + if errors.Is(beforeErr, os.ErrNotExist) { + if !errors.Is(afterErr, os.ErrNotExist) { + t.Fatal("export created a replacement database") + } + } else if beforeErr != nil || afterErr != nil || !bytes.Equal(before, after) { + t.Fatal("export modified its rejected source") + } + }) + } +} + +func recordExportStorage(t *testing.T, storage dbconfig.Storage) dbconfig.Selection { + t.Helper() + selected, err := dbconfig.NewSelection(storage) + if err != nil { + t.Fatal(err) + } + selected, err = dbconfig.AdoptSelection(dbconfig.SelectionPath(storage.StateDir), selected) + if err != nil { + t.Fatal(err) + } + return selected +} + +func TestExportHonorsRecordedSQLiteAndHoldsLease(t *testing.T) { + base := t.TempDir() + path := filepath.Join(base, "installed.db") + sqliteExportFixture(t, path, true) + selected := recordExportStorage(t, dbconfig.Storage{Backend: dbconfig.SQLite, StateDir: base, DBPath: path}) + db, lease, err := openSelectedExport(dbconfig.StorageInputs{BaseDir: base, StateDir: base, Backend: dbconfig.Postgres, DBPath: "stale.db"}) + if err != nil { + t.Fatal(err) + } + defer db.Close() + defer lease.Close() + assertExportMetadata(t, db) + update, err := dbconfig.BeginSelectionUpdate(dbconfig.SelectionPath(base), selected.Generation) + if err == nil { + update.Abort() + update.Close() + t.Fatal("export did not hold the selection lease") + } + if !errors.Is(err, dbconfig.ErrSelectionBusy) { + t.Fatalf("unexpected lease error: %v", err) + } + db.Close() + lease.Close() + update, err = dbconfig.BeginSelectionUpdate(dbconfig.SelectionPath(base), selected.Generation) + if err != nil { + t.Fatalf("export did not release the lease: %v", err) + } + defer update.Close() + if err := update.Abort(); err != nil { + t.Fatal(err) + } +} + +func TestRecordedSQLiteExportDoesNotFallBackOrCreate(t *testing.T) { + base := t.TempDir() + path := filepath.Join(base, "lost.db") + recordExportStorage(t, dbconfig.Storage{Backend: dbconfig.SQLite, StateDir: base, DBPath: path}) + stale := filepath.Join(base, "stale.db") + sqliteExportFixture(t, stale, true) + db, lease, err := openSelectedExport(dbconfig.StorageInputs{BaseDir: base, StateDir: base, DBPath: stale}) + if err == nil || db != nil || lease != nil { + t.Fatalf("lost recorded source did not fail closed: db=%t lease=%t err=%v", db != nil, lease != nil, err) + } + if _, err := os.Stat(path); !errors.Is(err, os.ErrNotExist) { + t.Fatal("export created the missing selected source") + } +} + +func TestExportSelectionErrorsCannotFallBack(t *testing.T) { + for _, kind := range []string{"corrupt", "switching"} { + t.Run(kind, func(t *testing.T) { + base := t.TempDir() + path := filepath.Join(base, "legacy.db") + sqliteExportFixture(t, path, true) + metadata := dbconfig.SelectionPath(base) + if kind == "switching" { + metadata += ".switch.json" + } + if err := os.WriteFile(metadata, []byte(`{}`), 0600); err != nil { + t.Fatal(err) + } + if db, lease, err := openSelectedExport(dbconfig.StorageInputs{BaseDir: base, DBPath: path}); err == nil || db != nil || lease != nil { + t.Fatal("invalid selection fell back to legacy SQLite") + } + }) + } +} + +func TestRecordedPostgresExportNeverUsesStaleSQLite(t *testing.T) { + base := t.TempDir() + stale := filepath.Join(base, "meshcore.db") + w := sqliteExportFixture(t, stale, true) + w.Close() + before, err := os.ReadFile(stale) + if err != nil { + t.Fatal(err) + } + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + endpoint := listener.Addr().String() + listener.Close() + dsn := "postgres://reader:never-print-this@" + endpoint + "/unavailable?sslmode=disable&connect_timeout=1" + recordExportStorage(t, dbconfig.Storage{Backend: dbconfig.Postgres, StateDir: base, ReaderDatabaseURL: dsn}) + for _, reader := range []string{"", dsn} { + db, lease, err := openSelectedExport(dbconfig.StorageInputs{BaseDir: base, DBPath: stale, Backend: dbconfig.SQLite, ReaderDatabaseURL: reader}) + if err == nil || db != nil || lease != nil { + t.Fatal("missing PostgreSQL selection fell back to SQLite") + } + if strings.Contains(err.Error(), "never-print-this") { + t.Fatal("connection failure exposed its password") + } + } + after, err := os.ReadFile(stale) + if err != nil || !bytes.Equal(before, after) { + t.Fatal("stale SQLite source changed") + } +} + +func TestExportConfigResolution(t *testing.T) { + base := t.TempDir() + data := filepath.Join(base, "data") + if err := os.Mkdir(data, 0700); err != nil { + t.Fatal(err) + } + path := filepath.Join(data, "meshcore.db") + sqliteExportFixture(t, path, true) + noEnv := func(string) string { return "" } + raw, err := exportStorageInputs(base, exportOptions{}, noEnv) + if err != nil { + t.Fatal(err) + } + db, lease, err := openSelectedExport(raw) + if err != nil { + t.Fatalf("default SQLite export: %v", err) + } + if lease != nil { + lease.Close() + t.Fatal("unrecorded export must not adopt a selection") + } + assertExportMetadata(t, db) + db.Close() + if _, err := os.Stat(dbconfig.SelectionPath(data)); !errors.Is(err, os.ErrNotExist) { + t.Fatal("export persisted a selection") + } + if err := os.WriteFile(filepath.Join(data, "config.json"), []byte(`{"dbPath":"data/meshcore.db"}`), 0600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(base, "config.json"), []byte(`{"db":`), 0600); err != nil { + t.Fatal(err) + } + if _, err := exportStorageInputs(base, exportOptions{}, noEnv); err == nil { + t.Fatal("malformed primary config fell back to a valid secondary config") + } + if _, err := exportStorageInputs(base, exportOptions{ConfigPath: filepath.Join(base, "missing.json")}, noEnv); err == nil { + t.Fatal("missing explicit config was ignored") + } + getenv := func(key string) string { + return map[string]string{"DB_PATH": "env.db", "CORESCOPE_DATABASE_URL": "postgres://reader@localhost/example", "CORESCOPE_READER_DATABASE_URL": "postgres://reader@localhost/reader", "CORESCOPE_STATE_DIR": "env-state"}[key] + } + raw, err = exportStorageInputs(base, exportOptions{ConfigPath: filepath.Join(data, "config.json"), DBPath: "cli.db", DatabaseURL: "postgres://reader@localhost/cli", StateDir: "cli-state"}, getenv) + if err != nil || raw.DBPath != "cli.db" || raw.DatabaseURL != "postgres://reader@localhost/cli" || raw.ReaderDatabaseURL != raw.DatabaseURL || raw.StateDir != "cli-state" { + t.Fatal("explicit flag precedence was lost") + } + if _, err := dbconfig.ResolveStorage(raw); err == nil { + t.Fatal("ambiguous unrecorded file and URL inputs were accepted") + } +} diff --git a/cmd/ingestor/README.md b/cmd/ingestor/README.md index 0ad2986b3..8655ed76e 100644 --- a/cmd/ingestor/README.md +++ b/cmd/ingestor/README.md @@ -1,25 +1,25 @@ # MeshCore MQTT Ingestor (Go) -Standalone MQTT ingestion service for CoreScope. Connects to MQTT brokers, decodes raw MeshCore packets, and writes to the same SQLite database used by the Node.js web server. +Standalone MQTT ingestion service for CoreScope. Connects to MQTT brokers, decodes raw MeshCore packets, and writes to PostgreSQL. The separate Go API server uses a restricted reader credential. -This is the first step of a larger Go rewrite — separating MQTT ingestion from the web server. +The migration command owns schema changes. The ingestor only connects to an initialized, ready telemetry schema. ## Architecture ``` -MQTT Broker(s) → Go Ingestor → SQLite DB ← Node.js Web Server +MQTT Broker(s) → Go Ingestor → PostgreSQL ← Go API Server (this binary) (shared) ``` -- **Single static binary** — no runtime dependencies, no CGO -- **SQLite** via `github.com/mattn/go-sqlite3` (cgo; cross-compiled with `zig cc`, see the root `Makefile`) +- **Single binary** — PostgreSQL runtime driver is pure Go +- **PostgreSQL 18** via pgx through `database/sql` - **MQTT** via `github.com/eclipse/paho.mqtt.golang` -- Runs **alongside** the Node.js server — they share the DB file -- Does NOT serve HTTP/WebSocket — that stays in Node.js +- Runs alongside the Go API server with separate writer and reader roles +- Does not serve HTTP/WebSocket — that stays in the Go API server ## Build -Requires Go 1.22+. +Requires Go 1.25+. ```bash cd cmd/ingestor @@ -38,16 +38,19 @@ GOOS=linux GOARCH=amd64 go build -o corescope-ingestor . ./corescope-ingestor -config /path/to/config.json ``` -The config file uses the same format as the Node.js `config.json`. The ingestor reads the `mqttSources` array (or legacy `mqtt` object) and `dbPath` fields. +The ingestor reads `mqttSources` (or the legacy `mqtt` object), `databaseURL` and `stateDir`. `-database-url` and `-state-dir` override configuration. `dbPath` and `-db` are never interpreted as database URLs: import existing SQLite snapshots offline first. Do not place connection URLs in command logs. ### Environment Variables | Variable | Description | Default | |----------|-------------|---------| -| `DB_PATH` | SQLite database path | `data/meshcore.db` | +| `CORESCOPE_WRITER_DATABASE_URL` | Ingestor-specific PostgreSQL writer URL; overrides the generic URL | required when no generic/config URL exists | +| `CORESCOPE_DATABASE_URL` | Generic PostgreSQL URL for standalone runs | config `databaseURL` | +| `CORESCOPE_STATE_DIR` | Local queue/statistics directory shared with the API server | `data` | +| `CORESCOPE_APPROVED_CHANNELS_DATABASE_URL` | Separate restricted account reader for `approved_channels` view | config `userManagement.approvedChannelsDatabaseURL` | | `MQTT_BROKER` | Single MQTT broker URL (overrides config) | — | | `MQTT_TOPIC` | MQTT topic (used with `MQTT_BROKER`) | `meshcore/#` | -| `CORESCOPE_INGESTOR_STATS` | Path to the per-second stats JSON file consumed by the server's `/api/perf/io` and `/api/perf/write-sources` endpoints (#1120) | `/tmp/corescope-ingestor-stats.json` | +| `CORESCOPE_INGESTOR_STATS` | Path to the per-second stats JSON file consumed by the server's `/api/perf/io` and `/api/perf/write-sources` endpoints (#1120) | `/ingestor-stats.json` | ### Stats file (`CORESCOPE_INGESTOR_STATS`) @@ -60,17 +63,15 @@ the Perf page so operators can self-diagnose write-volume anomalies. The writer uses `O_NOFOLLOW | O_CREAT | O_TRUNC` mode `0o600`, so a pre-planted symlink at the path cannot be used to clobber an arbitrary file. -**Security note:** the default lives in `/tmp`, which is world-writable on -most hosts (sticky bit only protects deletion, not creation). On -shared/multi-tenant hosts, override `CORESCOPE_INGESTOR_STATS` to point at a -private directory (e.g. `/var/lib/corescope/ingestor-stats.json`) that only -the corescope user can write to. +The state directory is created privately and stores small queue/statistics files. +Database URLs never become filesystem paths. ### Minimal Config ```json { - "dbPath": "data/meshcore.db", + "databaseURL": "postgres://telemetry_writer@localhost/telemetry", + "stateDir": "data", "mqttSources": [ { "name": "local", @@ -81,7 +82,7 @@ the corescope user can write to. } ``` -### Full Config (same as Node.js) +### Full Config The ingestor reads these fields from the existing `config.json`: @@ -93,13 +94,14 @@ The ingestor reads these fields from the existing `config.json`: - `iataFilter` — optional regional filter - `clientId`: optional MQTT ClientID. Default `corescope--`, new suffix per ingestor start. Two running ingestors must not share a value on one broker - `mqtt` — legacy single-broker config (auto-converted to `mqttSources`) -- `dbPath` — SQLite DB path (default: `data/meshcore.db`) +- `databaseURL` — restricted telemetry writer URL; schema-owner credentials are refused +- `stateDir` — local queue/statistics directory (default: `data`) ## Test ```bash cd cmd/ingestor -go test -v ./... +CORESCOPE_TEST_POSTGRES_URL=postgres://test_admin@localhost/postgres go test -v ./... ``` ## What It Does @@ -109,7 +111,7 @@ go test -v ./... 3. Receives raw hex packets via JSON messages (`{ "raw": "...", "SNR": ..., "RSSI": ... }`) 4. Decodes MeshCore packet headers, paths, and payloads (ported from `decoder.js`) 5. Computes content hashes (path-independent, SHA-256-based) -6. Writes to SQLite: `transmissions` + `observations` tables +6. Writes atomically to PostgreSQL: `transmissions` + `observations` tables 7. Upserts `nodes` from decoded ADVERT packets (with validation) 8. Upserts `observers` from MQTT topic metadata @@ -147,3 +149,10 @@ cmd/ingestor/ util.go — shared utilities go.mod / go.sum — Go module definition ``` + +PostgreSQL autovacuum, automatic analyze and checkpoints are managed by the +PostgreSQL service, not by runtime owner privileges. The previous SQLite +`db.vacuumOnStartup`, `db.incrementalVacuumPages` and `db.analysisLimit` +settings have no PostgreSQL effect. The migration/restore procedure must run +`ANALYZE` after bulk imports; runtime credentials cannot alter planner targets. +Packet retention remains bounded to 250 transmissions per transaction. diff --git a/cmd/ingestor/advert_evidence_migration_race_test.go b/cmd/ingestor/advert_evidence_migration_race_test.go index b7965e111..2eb6f37b1 100644 --- a/cmd/ingestor/advert_evidence_migration_race_test.go +++ b/cmd/ingestor/advert_evidence_migration_race_test.go @@ -11,18 +11,24 @@ import ( func seedUnbackfilledAdvert(t *testing.T, canonical, observed string) (*Store, string) { t.Helper() path := filepath.Join(t.TempDir(), "legacy-advert.db") - s, err := OpenStore(path) + s, err := openPostgresTestStore(t, path) if err != nil { t.Fatal(err) } if err := s.UpsertObserver("fixture-observer", "Fixture observer", "", nil); err != nil { t.Fatal(err) } - if _, err := s.db.Exec(`INSERT INTO transmissions(id,hash,raw_hex,first_seen,payload_type,route_type) VALUES(1,'legacy-advert',?,'2026-01-01T00:00:00Z',4,?); - INSERT INTO observations(id,transmission_id,observer_idx,raw_hex,path_json,timestamp) VALUES(1,1,(SELECT rowid FROM observers WHERE id='fixture-observer'),?,'[]',1)`, canonical, int(canonical[1]-'0')&3, observed); err != nil { + if _, err := s.db.Exec(`INSERT INTO transmissions(id,hash,raw_hex,first_seen,payload_type,route_type) VALUES(1,'legacy-advert',$1,'2026-01-01T00:00:00Z',4,$2)`, canonical, int(canonical[1]-'0')&3); err != nil { s.Close() t.Fatal(err) } + if _, err := s.db.Exec(`INSERT INTO observations(id,transmission_id,observer_idx,raw_hex,path_json,timestamp) VALUES(1,1,(SELECT rowid FROM observers WHERE id='fixture-observer'),$1,'[]',1)`, observed); err != nil { + t.Fatal(err) + } + if _, err := testAdmin(t, s).Exec(testNativeSQL(`UPDATE sqlite_sequence SET seq=1 WHERE name='observations'`, `SELECT setval('observations_id_seq',1,true)`)); err != nil { + t.Fatal(err) + } + return s, path } @@ -33,9 +39,9 @@ func TestAdvertRouteEvidenceLegacyProtectionErrorsDoNotDropIncomingRaw(t *testin defer s.Close() stmt := `DROP TABLE advert_evidence_backfill` if failure == "write" { - stmt = `CREATE TRIGGER fail_old_evidence BEFORE INSERT ON advert_route_evidence WHEN NEW.bit=2 BEGIN SELECT RAISE(ABORT,'fixture old evidence failure'); END` + stmt = testNativeSQL(`CREATE TRIGGER fail_old_evidence BEFORE INSERT ON advert_route_evidence WHEN NEW.bit=2 BEGIN SELECT RAISE(ABORT,'fixture old evidence failure'); END`, `CREATE OR REPLACE FUNCTION fail_old_evidence_fn() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.bit=2 THEN RAISE EXCEPTION 'fixture old evidence failure'; END IF; RETURN NEW; END $$; CREATE TRIGGER fail_old_evidence BEFORE INSERT ON advert_route_evidence FOR EACH ROW EXECUTE FUNCTION fail_old_evidence_fn()`) } - if _, err := s.db.Exec(stmt); err != nil { + if _, err := testAdmin(t, s).Exec(stmt); err != nil { t.Fatal(err) } _, err := s.InsertTransmission(&PacketData{Hash: "legacy-advert", ObserverID: "fixture-observer", PayloadType: 4, RouteType: 1, RawHex: "1100aa", PathJSON: "[]"}) @@ -75,19 +81,23 @@ func TestAdvertRouteEvidenceLegacyCoalescedPathAndIndex(t *testing.T) { if mask != 3 { t.Fatalf("coalesced path lookup lost old evidence: mask=%d", mask) } - rows, err := s.db.Query(`EXPLAIN QUERY PLAN `+legacyAdvertObservationSQL, 1, 1, "") + if testBackendValue() == "postgres" { + if _, err := s.db.Exec(`SET enable_seqscan=off`); err != nil { + t.Fatal(err) + } + } + rows, err := s.db.Query(testNativeSQL(`EXPLAIN QUERY PLAN `, `EXPLAIN (COSTS OFF) `)+legacyAdvertObservationSQL, 1, 1, "") if err != nil { t.Fatal(err) } defer rows.Close() indexed := false for rows.Next() { - var id, parent, unused int var detail string - if err := rows.Scan(&id, &parent, &unused, &detail); err != nil { + if err := scanTestPlan(rows, &detail); err != nil { t.Fatal(err) } - if strings.Contains(detail, "SEARCH observations USING INDEX idx_observations_dedup") { + if strings.Contains(detail, "idx_observations_dedup") { indexed = true } } @@ -111,7 +121,7 @@ func TestAdvertRouteEvidenceCompletionSkipsLookupAfterRestart(t *testing.T) { } s.Close() var err error - s, err = OpenStore(path) + s, err = openPostgresTestStore(t, path) if err != nil { t.Fatal(err) } @@ -120,7 +130,7 @@ func TestAdvertRouteEvidenceCompletionSkipsLookupAfterRestart(t *testing.T) { } // Any accidental legacy lookup now fails. Completed stores must incur // no extra SQL on this steady-state overwrite path. - if _, err := s.db.Exec(`DROP TABLE advert_evidence_backfill`); err != nil { + if _, err := testAdmin(t, s).Exec(`DROP TABLE advert_evidence_backfill`); err != nil { t.Fatal(err) } if _, err := s.InsertTransmission(&PacketData{Hash: "legacy-advert", ObserverID: "fixture-observer", PayloadType: 4, RouteType: 1, RawHex: "1100aa", PathJSON: "[]"}); err != nil { @@ -129,7 +139,7 @@ func TestAdvertRouteEvidenceCompletionSkipsLookupAfterRestart(t *testing.T) { } func BenchmarkAdvertEvidenceLegacyPreservation(b *testing.B) { - s, err := OpenStore(filepath.Join(b.TempDir(), "legacy-preservation.db")) + s, err := openPostgresTestStore(b, filepath.Join(b.TempDir(), "legacy-preservation.db")) if err != nil { b.Fatal(err) } @@ -151,7 +161,7 @@ func BenchmarkAdvertEvidenceLegacyPreservation(b *testing.B) { if state != "pending" { cursor = 1 } - if _, err := s.db.Exec(`INSERT INTO advert_evidence_backfill(id,obs_cursor) VALUES(1,?) ON CONFLICT(id) DO UPDATE SET obs_cursor=excluded.obs_cursor`, cursor); err != nil { + if _, err := s.db.Exec(`INSERT INTO advert_evidence_backfill(id,obs_cursor) VALUES(1,$1) ON CONFLICT(id) DO UPDATE SET obs_cursor=excluded.obs_cursor`, cursor); err != nil { b.Fatal(err) } s.advertEvidenceComplete.Store(state == "complete") @@ -160,7 +170,17 @@ func BenchmarkAdvertEvidenceLegacyPreservation(b *testing.B) { b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { - if err := s.preserveLegacyAdvertObservation(1, observerIdx, "[]"); err != nil { + if err := func() error { + tx, e := beginWrite(s.db, s.Backend()) + if e != nil { + return e + } + defer tx.Rollback() + if e = s.preserveLegacyAdvertObservation(tx, 1, observerIdx, "[]"); e != nil { + return e + } + return tx.Commit() + }(); err != nil { b.Fatal(err) } } @@ -196,7 +216,7 @@ func TestAdvertRouteEvidencePreservesLegacyConflictBeforeBackfill(t *testing.T) if restart { s.Close() var err error - s, err = OpenStore(path) + s, err = openPostgresTestStore(t, path) if err != nil { t.Fatal(err) } diff --git a/cmd/ingestor/advert_evidence_scale_test.go b/cmd/ingestor/advert_evidence_scale_test.go index bd716a373..b6aabc767 100644 --- a/cmd/ingestor/advert_evidence_scale_test.go +++ b/cmd/ingestor/advert_evidence_scale_test.go @@ -25,7 +25,7 @@ func TestAdvertBackfillScale(t *testing.T) { } else if !os.IsNotExist(err) { t.Fatal(err) } - s, err := OpenStore(path) + s, err := openPostgresTestStore(t, path) if err != nil { t.Fatal(err) } @@ -42,17 +42,17 @@ func TestAdvertBackfillScale(t *testing.T) { { raw := "1100" + strings.Repeat("ab", 118) for start := 1; start <= 1000000; start += 10000 { - _, err = s.db.Exec(`WITH RECURSIVE ids(n) AS (VALUES(?) UNION ALL SELECT n+1 FROM ids WHERE n COALESCE((SELECT obs_cursor FROM advert_evidence_backfill WHERE id=1),0)` // Caller holds writerMu, including through the subsequent observation UPSERT. // The backfill commits evidence and obs_cursor together under that same lock. -func (s *Store) preserveLegacyAdvertObservation(txID, observerIdx int64, path string) error { +func (s *Store) preserveLegacyAdvertObservation(tx *sql.Tx, txID, observerIdx int64, path string) error { if s.advertEvidenceComplete.Load() { return nil } var raw sql.NullString - err := s.stmtGetLegacyAdvertObservation.QueryRow(txID, observerIdx, path).Scan(&raw) + err := tx.Stmt(s.stmtGetLegacyAdvertObservation).QueryRow(txID, observerIdx, path).Scan(&raw) if err == sql.ErrNoRows { return nil } @@ -34,7 +34,7 @@ func (s *Store) preserveLegacyAdvertObservation(txID, observerIdx int64, path st return err } if bit := packetpath.AdvertRouteEvidence(raw.String); bit != 0 { - _, err = s.stmtInsertAdvertEvidence.Exec(txID, bit, txID, bit) + _, err = tx.Stmt(s.stmtInsertAdvertEvidence).Exec(txID, bit, txID, bit) } return err } @@ -44,12 +44,12 @@ func (s *Store) preserveLegacyAdvertObservation(txID, observerIdx int64, path st // cursors and 500-row transactions make this cancellable/resumable; new traffic // records evidence synchronously, so rows beyond either scan need no replay. func (s *Store) backfillAdvertEvidence(ctx context.Context, db *sql.DB) error { - if _, err := db.ExecContext(ctx, `INSERT OR IGNORE INTO advert_evidence_backfill(id) VALUES(1)`); err != nil { + if _, err := db.ExecContext(ctx, `INSERT INTO advert_evidence_backfill(id) VALUES(1) ON CONFLICT DO NOTHING`); err != nil { return err } for _, source := range []struct{ cursor, table, query string }{ - {"tx_cursor", "transmissions", `SELECT id,id,COALESCE(raw_hex,''),payload_type FROM transmissions WHERE id>? AND id<=? ORDER BY id LIMIT 500`}, - {"obs_cursor", "observations", `SELECT o.id,o.transmission_id,COALESCE(o.raw_hex,''),t.payload_type FROM observations o JOIN transmissions t ON t.id=o.transmission_id WHERE o.id>? AND o.id<=? ORDER BY o.id LIMIT 500`}, + {"tx_cursor", "transmissions", `SELECT id,id,COALESCE(raw_hex,''),payload_type FROM transmissions WHERE id>` + s.parameter(1) + ` AND id<=` + s.parameter(2) + ` ORDER BY id LIMIT 500`}, + {"obs_cursor", "observations", `SELECT o.id,o.transmission_id,COALESCE(o.raw_hex,''),t.payload_type FROM observations o JOIN transmissions t ON t.id=o.transmission_id WHERE o.id>` + s.parameter(1) + ` AND o.id<=` + s.parameter(2) + ` ORDER BY o.id LIMIT 500`}, } { // A finite horizon prevents live traffic from extending this scan. var upper int64 @@ -101,17 +101,17 @@ func (s *Store) backfillAdvertEvidence(ctx context.Context, db *sql.DB) error { err = func() error { writerMu.Lock() defer writerMu.Unlock() - tx, err := db.BeginTx(ctx, nil) + tx, err := beginWriteContext(ctx, db, s.Backend()) if err != nil { return err } defer tx.Rollback() for _, item := range batch { - if _, err := tx.ExecContext(ctx, insertAdvertEvidenceSQL+` AND EXISTS(SELECT 1 FROM transmissions WHERE id=?)`, item.txID, item.bit, item.txID, item.bit, item.txID); err != nil { + if _, err := tx.ExecContext(ctx, insertAdvertEvidenceSQL+` AND EXISTS(SELECT 1 FROM transmissions WHERE id=`+s.parameter(5)+`)`, item.txID, item.bit, item.txID, item.bit, item.txID); err != nil { return err } } - if _, err := tx.ExecContext(ctx, `UPDATE advert_evidence_backfill SET `+source.cursor+`=? WHERE id=1`, lastID); err != nil { + if _, err := tx.ExecContext(ctx, `UPDATE advert_evidence_backfill SET `+source.cursor+`=`+s.parameter(1)+` WHERE id=1`, lastID); err != nil { return err } return tx.Commit() diff --git a/cmd/ingestor/advert_route_evidence_test.go b/cmd/ingestor/advert_route_evidence_test.go index bccbf8a11..10494fa8a 100644 --- a/cmd/ingestor/advert_route_evidence_test.go +++ b/cmd/ingestor/advert_route_evidence_test.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "path/filepath" + "strings" "testing" ) @@ -28,7 +29,7 @@ func TestAdvertRouteEvidenceSurvivesObservationUpsert(t *testing.T) { } { t.Run(sequence.name, func(t *testing.T) { path := filepath.Join(t.TempDir(), "evidence.db") - s, err := OpenStore(path) + s, err := openPostgresTestStore(t, path) if err != nil { t.Fatal(err) } @@ -64,7 +65,7 @@ func TestAdvertRouteEvidenceSurvivesObservationUpsert(t *testing.T) { if canonical != sequence.raws[0] { t.Fatalf("canonical raw changed: %s", canonical) } - if err := s.db.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='advert_route_evidence'`).Scan(&count); err != nil { + if err := s.db.QueryRow(testNativeSQL(`SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='advert_route_evidence'`, `SELECT COUNT(*) FROM information_schema.tables WHERE table_schema=current_schema() AND table_name='advert_route_evidence'`)).Scan(&count); err != nil { t.Fatal(err) } if count != 1 { @@ -94,14 +95,14 @@ func TestAdvertRouteEvidenceSurvivesObservationUpsert(t *testing.T) { t.Fatal("duplicate traffic appended route evidence") } var sequenceID int64 - if err := s.db.QueryRow(`SELECT seq FROM sqlite_sequence WHERE name='advert_route_evidence'`).Scan(&sequenceID); err != nil { + if err := s.db.QueryRow(testNativeSQL(`SELECT seq FROM sqlite_sequence WHERE name='advert_route_evidence'`, `SELECT last_value FROM advert_route_evidence_id_seq`)).Scan(&sequenceID); err != nil { t.Fatal(err) } if sequenceID != maxID { t.Fatalf("duplicate traffic advanced evidence sequence: %d, want %d", sequenceID, maxID) } s.Close() - s, err = OpenStore(path) + s, err = openPostgresTestStore(t, path) if err != nil { t.Fatal(err) } @@ -121,7 +122,7 @@ func TestAdvertRouteEvidenceSurvivesObservationUpsert(t *testing.T) { } func TestAdvertRouteEvidenceBackfillResumeAndLiveUnion(t *testing.T) { - s, err := OpenStore(filepath.Join(t.TempDir(), "backfill.db")) + s, err := openPostgresTestStore(t, filepath.Join(t.TempDir(), "backfill.db")) if err != nil { t.Fatal(err) } @@ -136,10 +137,10 @@ func TestAdvertRouteEvidenceBackfillResumeAndLiveUnion(t *testing.T) { t.Fatal(err) } for id := 1; id <= 1200; id++ { - if _, err := tx.Exec(`INSERT INTO transmissions(id,hash,raw_hex,first_seen,payload_type,route_type) VALUES(?,?, '1100aa','2026-01-01T00:00:00Z',4,1)`, id, fmt.Sprintf("history-%d", id)); err != nil { + if _, err := tx.Exec(`INSERT INTO transmissions(id,hash,raw_hex,first_seen,payload_type,route_type) VALUES($1,$2, '1100aa','2026-01-01T00:00:00Z',4,1)`, id, fmt.Sprintf("history-%d", id)); err != nil { t.Fatal(err) } - if _, err := tx.Exec(`INSERT INTO observations(transmission_id,observer_idx,raw_hex,path_json,timestamp) VALUES(?,(SELECT rowid FROM observers WHERE id='fixture-observer'),'1200aa','[]',1)`, id); err != nil { + if _, err := tx.Exec(`INSERT INTO observations(transmission_id,observer_idx,raw_hex,path_json,timestamp) VALUES($1,(SELECT rowid FROM observers WHERE id='fixture-observer'),'1200aa','[]',1)`, id); err != nil { t.Fatal(err) } } @@ -153,7 +154,7 @@ func TestAdvertRouteEvidenceBackfillResumeAndLiveUnion(t *testing.T) { } // Abort after one committed batch; the failed batch must not move its // persisted cursor, so a retry can recover every remaining frame. - if _, err := s.db.Exec(`CREATE TRIGGER fail_evidence_batch BEFORE INSERT ON advert_route_evidence WHEN NEW.tx_id=501 BEGIN SELECT RAISE(ABORT,'fixture failure'); END`); err != nil { + if _, err := testAdmin(t, s).Exec(testNativeSQL(`CREATE TRIGGER fail_evidence_batch BEFORE INSERT ON advert_route_evidence WHEN NEW.tx_id=501 BEGIN SELECT RAISE(ABORT,'fixture failure'); END`, `CREATE OR REPLACE FUNCTION fail_evidence_batch_fn() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.tx_id=501 THEN RAISE EXCEPTION 'fixture failure'; END IF; RETURN NEW; END $$; CREATE TRIGGER fail_evidence_batch BEFORE INSERT ON advert_route_evidence FOR EACH ROW EXECUTE FUNCTION fail_evidence_batch_fn()`)); err != nil { t.Fatal(err) } if err := s.backfillAdvertEvidence(context.Background(), s.db); err == nil { @@ -166,7 +167,7 @@ func TestAdvertRouteEvidenceBackfillResumeAndLiveUnion(t *testing.T) { if cursor != 500 { t.Fatalf("cursor=%d after failure, want committed batch boundary 500", cursor) } - if _, err := s.db.Exec(`DROP TRIGGER fail_evidence_batch`); err != nil { + if _, err := testAdmin(t, s).Exec(testNativeSQL(`DROP TRIGGER fail_evidence_batch`, `DROP TRIGGER fail_evidence_batch ON advert_route_evidence`)); err != nil { t.Fatal(err) } done := make(chan error, 1) @@ -187,13 +188,13 @@ func TestAdvertRouteEvidenceBackfillResumeAndLiveUnion(t *testing.T) { t.Fatalf("got %d evidence rows, want two per history transmission", count) } var before, after int64 - if err := s.db.QueryRow(`SELECT seq FROM sqlite_sequence WHERE name='advert_route_evidence'`).Scan(&before); err != nil { + if err := s.db.QueryRow(testNativeSQL(`SELECT seq FROM sqlite_sequence WHERE name='advert_route_evidence'`, `SELECT last_value FROM advert_route_evidence_id_seq`)).Scan(&before); err != nil { t.Fatal(err) } if err := s.backfillAdvertEvidence(context.Background(), s.db); err != nil { t.Fatal(err) } - if err := s.db.QueryRow(`SELECT seq FROM sqlite_sequence WHERE name='advert_route_evidence'`).Scan(&after); err != nil { + if err := s.db.QueryRow(testNativeSQL(`SELECT seq FROM sqlite_sequence WHERE name='advert_route_evidence'`, `SELECT last_value FROM advert_route_evidence_id_seq`)).Scan(&after); err != nil { t.Fatal(err) } if before != after { @@ -206,7 +207,7 @@ func TestAdvertRouteEvidenceBackfillResumeAndLiveUnion(t *testing.T) { func TestAdvertRouteEvidenceFailureKeepsCoreIngestion(t *testing.T) { for _, failure := range []string{"incoming-write", "legacy-read", "legacy-write"} { t.Run(failure, func(t *testing.T) { - s, err := OpenStore(filepath.Join(t.TempDir(), "write-failure.db")) + s, err := openPostgresTestStore(t, filepath.Join(t.TempDir(), "write-failure.db")) if err != nil { t.Fatal(err) } @@ -226,14 +227,14 @@ func TestAdvertRouteEvidenceFailureKeepsCoreIngestion(t *testing.T) { if _, err := s.db.Exec(`DELETE FROM advert_route_evidence; UPDATE observations SET raw_hex='1200aa',resolved_path=NULL`); err != nil { t.Fatal(err) } - stmt := `CREATE TRIGGER fail_evidence BEFORE INSERT ON advert_route_evidence WHEN NEW.bit=1 BEGIN SELECT RAISE(ABORT,'fixture evidence failure'); END` + stmt := testNativeSQL(`CREATE TRIGGER fail_evidence BEFORE INSERT ON advert_route_evidence WHEN NEW.bit=1 BEGIN SELECT RAISE(ABORT,'fixture evidence failure'); END`, `CREATE OR REPLACE FUNCTION fail_evidence_fn() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.bit=1 THEN RAISE EXCEPTION 'fixture evidence failure'; END IF; RETURN NEW; END $$; CREATE TRIGGER fail_evidence BEFORE INSERT ON advert_route_evidence FOR EACH ROW EXECUTE FUNCTION fail_evidence_fn()`) if failure == "legacy-read" { stmt = `DROP TABLE advert_evidence_backfill` } if failure == "legacy-write" { - stmt = `CREATE TRIGGER fail_evidence BEFORE INSERT ON advert_route_evidence WHEN NEW.bit=2 BEGIN SELECT RAISE(ABORT,'fixture evidence failure'); END` + stmt = testNativeSQL(`CREATE TRIGGER fail_evidence BEFORE INSERT ON advert_route_evidence WHEN NEW.bit=2 BEGIN SELECT RAISE(ABORT,'fixture evidence failure'); END`, `CREATE OR REPLACE FUNCTION fail_evidence_fn() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.bit=2 THEN RAISE EXCEPTION 'fixture evidence failure'; END IF; RETURN NEW; END $$; CREATE TRIGGER fail_evidence BEFORE INSERT ON advert_route_evidence FOR EACH ROW EXECUTE FUNCTION fail_evidence_fn()`) } - if _, err := s.db.Exec(stmt); err != nil { + if _, err := testAdmin(t, s).Exec(stmt); err != nil { t.Fatal(err) } before := s.Stats.WriteErrors.Load() @@ -275,7 +276,7 @@ func TestAdvertRouteEvidenceFailureKeepsCoreIngestion(t *testing.T) { } func TestAdvertRouteEvidenceConstraintsAndFeedIndex(t *testing.T) { - s, err := OpenStore(filepath.Join(t.TempDir(), "constraints.db")) + s, err := openPostgresTestStore(t, filepath.Join(t.TempDir(), "constraints.db")) if err != nil { t.Fatal(err) } @@ -288,17 +289,28 @@ func TestAdvertRouteEvidenceConstraintsAndFeedIndex(t *testing.T) { `INSERT INTO advert_route_evidence(tx_id,bit) VALUES(1,3)`, `INSERT INTO advert_route_evidence(tx_id,bit) VALUES(999,2)`, } { - if _, err := s.db.Exec(stmt); err == nil { + if _, err := testAdmin(t, s).Exec(stmt); err == nil { t.Errorf("constraint accepted: %s", stmt) } } - var id, parent, unused int - var detail string - if err := s.db.QueryRow(`EXPLAIN QUERY PLAN SELECT id,tx_id,bit FROM advert_route_evidence WHERE id>1 ORDER BY id LIMIT 500`).Scan(&id, &parent, &unused, &detail); err != nil { + if s.Backend() == "postgres" { + s.db.Exec(`SET enable_seqscan=off`) + } + rows, err := s.db.Query(testNativeSQL(`EXPLAIN QUERY PLAN SELECT id,tx_id,bit FROM advert_route_evidence WHERE id>1 ORDER BY id LIMIT 500`, `EXPLAIN (COSTS OFF) SELECT id,tx_id,bit FROM advert_route_evidence WHERE id>1 ORDER BY id LIMIT 500`)) + if err != nil { t.Fatal(err) } - if detail != "SEARCH advert_route_evidence USING INTEGER PRIMARY KEY (rowid>?)" { - t.Fatalf("feed must seek by primary key: %s", detail) + var plan string + for rows.Next() { + var line string + if err := scanTestPlan(rows, &line); err != nil { + t.Fatal(err) + } + plan += line + } + rows.Close() + if !strings.Contains(plan, testNativeSQL("INTEGER PRIMARY KEY", "advert_route_evidence_pkey")) { + t.Fatalf("feed must seek by primary key: %s", plan) } if _, err := s.db.Exec(`DELETE FROM observations; DELETE FROM transmissions`); err != nil { t.Fatal(err) @@ -316,7 +328,7 @@ func TestAdvertRouteEvidenceConstraintsAndFeedIndex(t *testing.T) { } func BenchmarkAdvertEvidenceRepeatedWrite(b *testing.B) { - s, err := OpenStore(filepath.Join(b.TempDir(), "repeat.db")) + s, err := openPostgresTestStore(b, filepath.Join(b.TempDir(), "repeat.db")) if err != nil { b.Fatal(err) } @@ -335,7 +347,7 @@ func BenchmarkAdvertEvidenceRepeatedWrite(b *testing.B) { } func TestAdvertRouteEvidenceRejectsUnprovenFrames(t *testing.T) { - s, err := OpenStore(filepath.Join(t.TempDir(), "evidence.db")) + s, err := openPostgresTestStore(t, filepath.Join(t.TempDir(), "evidence.db")) if err != nil { t.Fatal(err) } @@ -347,7 +359,7 @@ func TestAdvertRouteEvidenceRejectsUnprovenFrames(t *testing.T) { } } var count int - if err := s.db.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='advert_route_evidence'`).Scan(&count); err != nil { + if err := s.db.QueryRow(testNativeSQL(`SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='advert_route_evidence'`, `SELECT COUNT(*) FROM information_schema.tables WHERE table_schema=current_schema() AND table_name='advert_route_evidence'`)).Scan(&count); err != nil { t.Fatal(err) } if count != 1 { diff --git a/cmd/ingestor/approved_channels.go b/cmd/ingestor/approved_channels.go index c95a43da9..487b73d50 100644 --- a/cmd/ingestor/approved_channels.go +++ b/cmd/ingestor/approved_channels.go @@ -1,20 +1,23 @@ package main import ( + "context" "database/sql" "errors" - "fmt" "log" - "os" - "path/filepath" + "net/url" + "strings" "sync/atomic" "time" + _ "github.com/mattn/go-sqlite3" "github.com/meshcore-analyzer/channel" + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/pgutil" ) // Approved hashtag channels (docs/specs/2026-10-07-channel-proposals-design.md). -// The server owns users.db; the ingestor only reads it, read-only, with raw +// The server owns account data; the ingestor only reads its approved view // SQL, once per approvedChannelsRefresh, and never writes it (AGENTS.md, // read/write separation). @@ -22,9 +25,11 @@ const approvedChannelsRefresh = time.Minute // approvedChannelsQuery is pinned in internal/users/proposals_test.go // (ingestorApprovedQuery), which checks it against ApprovedSubjects. -const approvedChannelsQuery = `SELECT subject FROM proposals WHERE kind = 'hashtag_channel' AND status = 'approved' ORDER BY decided_at, id LIMIT ?` +const approvedChannelsQuery = `SELECT subject FROM approved_channels WHERE kind = 'hashtag_channel' ORDER BY decided_at, id LIMIT $1` -var errUsersDBMissing = errors.New("users.db not found") +const approvedChannelsSQLiteQuery = `SELECT subject FROM proposals WHERE kind = 'hashtag_channel' AND status = 'approved' ORDER BY decided_at, id LIMIT ?1` + +var errUsersDBMissing = errors.New("approved channel account storage is not configured") // channelKeySet hands each message the current channel key map. Snapshots // are never mutated after they are stored, so decoders read them without a @@ -32,6 +37,7 @@ var errUsersDBMissing = errors.New("users.db not found") type channelKeySet struct { configured map[string]string path string + backend dbconfig.Backend max int cur atomic.Pointer[map[string]string] @@ -42,8 +48,24 @@ type channelKeySet struct { // newChannelKeySet starts with the configured keys (the very map // loadChannelKeys built, so the feature off changes nothing). -func newChannelKeySet(configured map[string]string, usersDBPath string, max int) *channelKeySet { - s := &channelKeySet{configured: configured, path: usersDBPath, max: max} +// newChannelKeySet retains compatibility for an unambiguous native target. +func newChannelKeySet(configured map[string]string, target string, max int) *channelKeySet { + storage := dbconfig.Storage{Backend: dbconfig.SQLite, UsersDBPath: target} + if strings.HasPrefix(strings.ToLower(target), "postgres:") || strings.HasPrefix(strings.ToLower(target), "postgresql:") { + storage.Backend = dbconfig.Postgres + storage.ApprovedChannelsDatabaseURL = target + } + return newChannelKeySetStorage(configured, storage, max) +} + +// newChannelKeySetStorage consumes the same resolved storage choice as startup. +// Inactive addresses never cause fallback to a different account store. +func newChannelKeySetStorage(configured map[string]string, storage dbconfig.Storage, max int) *channelKeySet { + target := storage.UsersDBPath + if storage.Backend == dbconfig.Postgres { + target = storage.ApprovedChannelsDatabaseURL + } + s := &channelKeySet{configured: configured, path: target, backend: storage.Backend, max: max} s.cur.Store(&configured) return s } @@ -58,7 +80,7 @@ func (s *channelKeySet) current() *map[string]string { return s.cur.Load() } // refresh re-reads the approved names and swaps in configured + approved. // A failed read keeps the current snapshot: only a successful read changes -// the set, so a locked or missing users.db never drops a key. +// the set, so an unavailable or incomplete account store never drops a key. func (s *channelKeySet) refresh() { names, err := s.readApproved() if err != nil { @@ -69,7 +91,7 @@ func (s *channelKeySet) refresh() { return } if s.failing { - log.Printf("[channels] approved channels readable again from %s", s.path) + log.Print("[channels] approved channels readable again") s.failing = false } next, added := mergeApprovedKeys(s.configured, names) @@ -80,22 +102,96 @@ func (s *channelKeySet) refresh() { s.cur.Store(&next) } -func (s *channelKeySet) readApproved() ([]string, error) { +// readAccountSnapshot opens only the selected restricted projection target. +// Both approved channels and linked companions use this readiness/grants fence. +func (s *channelKeySet) readAccountSnapshot(ctx context.Context) (*sql.Tx, error) { + if s.backend != dbconfig.SQLite && s.backend != dbconfig.Postgres { + return nil, errors.New("approved channel storage backend must be selected") + } if s.db == nil { - if _, err := os.Stat(s.path); err != nil { - if os.IsNotExist(err) { - return nil, fmt.Errorf("%w at %s", errUsersDBMissing, s.path) + if s.path == "" { + return nil, errUsersDBMissing + } + var db *sql.DB + var err error + if s.backend == dbconfig.SQLite { + uri, e := dbconfig.SQLiteURI(s.path, url.Values{"mode": {"ro"}, "_busy_timeout": {"5000"}}) + if e != nil { + return nil, e + } + db, err = sql.Open("sqlite3", uri) + } else { + db, err = pgutil.Open(s.path, true) + if err == nil { + err = pgutil.AssertReadOnly(db) + } + if err == nil { + var credentialAccess bool + err = db.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname=current_schema() AND c.relkind IN ('r','p','m','f') AND c.relname <> 'corescope_schema' AND has_table_privilege(c.oid,'SELECT'))`).Scan(&credentialAccess) + if err == nil && credentialAccess { + err = errors.New("approved-channel reader has account-table access") + } } - return nil, err } - db, err := sql.Open("sqlite3", usersDBReadOnlyDSN(s.path)) if err != nil { + if db != nil { + db.Close() + } return nil, err } db.SetMaxOpenConns(1) + db.SetMaxIdleConns(1) s.db = db } - rows, err := s.db.Query(approvedChannelsQuery, s.max) + if s.backend == dbconfig.SQLite { + if err := dbconfig.AssertSQLiteImportComplete(s.db); err != nil { + return nil, err + } + } + // Check schema and rows within one native read-only snapshot. PostgreSQL's + // projection role can read only the readiness marker and approved view. + opts := &sql.TxOptions{ReadOnly: true} + if s.backend == dbconfig.Postgres { + opts.Isolation = sql.LevelRepeatableRead + } + tx, err := s.db.BeginTx(ctx, opts) + if err != nil { + return nil, err + } + var version int + if s.backend == dbconfig.Postgres { + var ready bool + if err := tx.QueryRowContext(ctx, `SELECT version,ready FROM corescope_schema WHERE kind='accounts'`).Scan(&version, &ready); err != nil { + tx.Rollback() + return nil, errors.New("approved-channel account schema is not initialized") + } + if version != 1 || !ready { + tx.Rollback() + return nil, errors.New("approved-channel account schema is incomplete or unsupported") + } + } else { + var count int + if err := tx.QueryRowContext(ctx, `SELECT count(*),COALESCE(MAX(version),0) FROM schema_version`).Scan(&count, &version); err != nil || count != 1 || version < 4 || version > 7 { + tx.Rollback() + return nil, errors.New("approved-channel SQLite account schema is incomplete or unsupported") + } + } + return tx, nil +} + +func (s *channelKeySet) readApproved() ([]string, error) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + tx, err := s.readAccountSnapshot(ctx) + if err != nil { + return nil, err + } + defer tx.Rollback() + query := approvedChannelsSQLiteQuery + if s.backend == dbconfig.Postgres { + query = approvedChannelsQuery + } + rows, err := tx.QueryContext(ctx, query, s.max) if err != nil { return nil, err } @@ -108,13 +204,16 @@ func (s *channelKeySet) readApproved() ([]string, error) { } out = append(out, name) } - return out, rows.Err() -} - -// usersDBReadOnlyDSN opens users.db read-only: this process must never -// take a write lock on the server's file. -func usersDBReadOnlyDSN(path string) string { - return "file:" + path + "?mode=ro&_busy_timeout=5000" + if err := rows.Err(); err != nil { + return nil, err + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := tx.Commit(); err != nil { + return nil, err + } + return out, nil } // mergeApprovedKeys returns a new map: the configured keys plus a derived key @@ -148,12 +247,7 @@ func (s *channelKeySet) Close() { } } -// logApprovedChannelsSource logs the users.db file the approved channels are -// read from, resolved to an absolute path so a relative one shows which file -// it means; the path as given when it cannot be resolved. -func logApprovedChannelsSource(path string) { - if abs, err := filepath.Abs(path); err == nil { - path = abs - } - log.Printf("[proposals] reading approved channels from %s", path) +// Connection strings contain credentials and must never appear in logs. +func logApprovedChannelsSource(databaseURL string) { + log.Print("[proposals] reading approved channels from the selected account store") } diff --git a/cmd/ingestor/approved_channels_import_test.go b/cmd/ingestor/approved_channels_import_test.go new file mode 100644 index 000000000..718168ba1 --- /dev/null +++ b/cmd/ingestor/approved_channels_import_test.go @@ -0,0 +1,31 @@ +package main + +import ( + "errors" + "path/filepath" + "testing" + + "github.com/meshcore-analyzer/dbconfig" +) + +func TestApprovedChannelsPendingSQLiteImportKeepsLastGoodKeys(t *testing.T) { + t.Setenv("CORESCOPE_TEST_BACKEND", "sqlite") + path := filepath.Join(t.TempDir(), "accounts.db") + writeUsersDB(t, path, testProposal{"#complete", "approved", 1}) + keys := newApprovedTestKeySet(t, configuredKeys(), path, 128) + defer keys.Close() + keys.refresh() + want := keys.Snapshot()["#complete"] + if want == "" { + t.Fatal("complete initial approval was not loaded") + } + execUsersDB(t, path, `CREATE TABLE corescope_reverse_progress (table_name TEXT)`) + execUsersDB(t, path, `UPDATE proposals SET subject='#partial'`) + if _, err := keys.readApproved(); !errors.Is(err, dbconfig.ErrSQLiteImportIncomplete) { + t.Fatalf("partial import was accepted: %v", err) + } + keys.refresh() + if keys.Snapshot()["#complete"] != want || keys.Snapshot()["#partial"] != "" { + t.Fatal("partial import replaced the last verified approval set") + } +} diff --git a/cmd/ingestor/approved_channels_test.go b/cmd/ingestor/approved_channels_test.go index 239b78dcc..e3b80c759 100644 --- a/cmd/ingestor/approved_channels_test.go +++ b/cmd/ingestor/approved_channels_test.go @@ -6,7 +6,11 @@ import ( "database/sql" "encoding/hex" "fmt" + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/pgutil" + "github.com/meshcore-analyzer/pgutil/pgtest" "log" + "net/url" "os" "path/filepath" "reflect" @@ -18,7 +22,7 @@ import ( // The columns cmd/server's users.db (internal/users schema v4) has; the // ingestor reads kind, subject, status, decided_at and id. const testProposalsDDL = `CREATE TABLE IF NOT EXISTS proposals ( - id INTEGER PRIMARY KEY, + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, kind TEXT NOT NULL, subject TEXT NOT NULL, status TEXT NOT NULL, @@ -36,16 +40,39 @@ type testProposal struct { func execUsersDB(t testing.TB, path string, stmts ...string) { t.Helper() - db, err := sql.Open("sqlite3", path) + db, err := openApprovedFixture(t, approvedFixture(t, path).owner) if err != nil { t.Fatal(err) } defer db.Close() for _, s := range stmts { + if approvedTestBackend(t) == dbconfig.SQLite && s == testProposalsDDL { + s = strings.Replace(s, "BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY", "INTEGER PRIMARY KEY", 1) + } if _, err := db.Exec(s); err != nil { t.Fatalf("%s: %v", s, err) } } + if stmts[0] == testProposalsDDL { + if approvedTestBackend(t) == dbconfig.SQLite { + if _, err := db.Exec(`CREATE TABLE IF NOT EXISTS schema_version(version INTEGER NOT NULL); DELETE FROM schema_version; INSERT INTO schema_version VALUES(5)`); err != nil { + t.Fatal(err) + } + return + } + f := approvedFixture(t, path) + for _, stmt := range []string{ + `CREATE TABLE IF NOT EXISTS corescope_schema(version INTEGER NOT NULL, kind TEXT PRIMARY KEY, ready BOOLEAN NOT NULL)`, + `INSERT INTO corescope_schema VALUES(1,'accounts',true) ON CONFLICT(kind) DO NOTHING`, + `CREATE OR REPLACE VIEW approved_channels AS SELECT kind,subject,decided_at,id FROM proposals WHERE status='approved'`, + `GRANT SELECT ON approved_channels,corescope_schema TO ` + f.role, + } { + if _, err := db.Exec(stmt); err != nil { + t.Fatal(err) + } + } + } + } func writeUsersDB(t *testing.T, path string, rows ...testProposal) { @@ -62,19 +89,45 @@ func configuredKeys() map[string]string { return map[string]string{"Public": "8b3387e9c5cdea6ac9e5edbaa115cd72", "#mesh": deriveHashtagChannelKey("#mesh")} } -func TestApprovedChannelsQueryMatchesUsersPackage(t *testing.T) { - src, err := os.ReadFile(filepath.Join("..", "..", "internal", "users", "proposals_test.go")) - if err != nil { +func TestApprovedChannelsQueryMatchesProjection(t *testing.T) { + path := filepath.Join(t.TempDir(), "projection.db") + writeUsersDB(t, path, testProposal{"#fixture", "approved", 1}) + reader := newApprovedTestKeySet(t, configuredKeys(), path, 128) + defer reader.Close() + if _, err := reader.readApproved(); err != nil { t.Fatal(err) } - if !strings.Contains(string(src), approvedChannelsQuery) { - t.Fatalf("internal/users/proposals_test.go no longer pins %q; keep the two in step", approvedChannelsQuery) +} + +func TestApprovedChannelsRejectsIncompleteAccountImport(t *testing.T) { + if approvedTestBackend(t) != dbconfig.Postgres { + t.Skip("PostgreSQL import readiness marker; SQLite schema completeness is tested separately") + } + path := filepath.Join(t.TempDir(), "account-import-gate.db") + writeUsersDB(t, path, testProposal{"#fixture", "approved", 1}) + reader := newApprovedTestKeySet(t, configuredKeys(), path, 128) + defer reader.Close() + reader.refresh() + if _, ok := reader.Snapshot()["#fixture"]; !ok { + t.Fatal("projection reader with readiness metadata could not load a verified account store") + } + execUsersDB(t, path, `UPDATE corescope_schema SET ready=false WHERE kind='accounts'`, `DELETE FROM proposals`) + if _, err := reader.readApproved(); err == nil { + t.Fatal("partially imported account data was accepted") + } + reader.refresh() + if _, ok := reader.Snapshot()["#fixture"]; !ok { + t.Fatal("incomplete account import discarded the last good approved keys") + } + execUsersDB(t, path, `UPDATE corescope_schema SET ready=true,version=2 WHERE kind='accounts'`) + if _, err := reader.readApproved(); err == nil { + t.Fatal("future account schema version was accepted") } } func TestChannelKeySetOffReturnsConfiguredMap(t *testing.T) { cfgd := configuredKeys() - s := newChannelKeySet(cfgd, filepath.Join(t.TempDir(), "users.db"), 128) + s := newApprovedTestKeySet(t, cfgd, filepath.Join(t.TempDir(), "users.db"), 128) if reflect.ValueOf(s.Snapshot()).Pointer() != reflect.ValueOf(cfgd).Pointer() { t.Fatal("before any refresh the decoder must get the very map loadChannelKeys built") } @@ -85,7 +138,7 @@ func TestChannelKeySetLoadsApproved(t *testing.T) { writeUsersDB(t, path, testProposal{"#mycity", "approved", 10}, testProposal{"#waiting", "pending", 0}, testProposal{"#nope", "rejected", 11}, testProposal{"#gone", "revoked", 12}) - s := newChannelKeySet(configuredKeys(), path, 128) + s := newApprovedTestKeySet(t, configuredKeys(), path, 128) defer s.Close() s.refresh() snap := s.Snapshot() @@ -107,7 +160,7 @@ func TestChannelKeySetConfiguredNameWins(t *testing.T) { writeUsersDB(t, path, testProposal{"#mesh", "approved", 1}) cfgd := configuredKeys() cfgd["#mesh"] = "00112233445566778899aabbccddeeff" - s := newChannelKeySet(cfgd, path, 128) + s := newApprovedTestKeySet(t, cfgd, path, 128) defer s.Close() s.refresh() if got := s.Snapshot()["#mesh"]; got != "00112233445566778899aabbccddeeff" { @@ -124,7 +177,7 @@ func TestChannelKeySetConfiguredNameWins(t *testing.T) { func TestChannelKeySetRevokeRemovesKeyOnNextRefresh(t *testing.T) { path := filepath.Join(t.TempDir(), "users.db") writeUsersDB(t, path, testProposal{"#mycity", "approved", 1}) - s := newChannelKeySet(configuredKeys(), path, 128) + s := newApprovedTestKeySet(t, configuredKeys(), path, 128) defer s.Close() s.refresh() before := s.Snapshot() @@ -143,7 +196,7 @@ func TestChannelKeySetMissingFileKeepsConfiguredAndLogsOnce(t *testing.T) { log.SetOutput(&buf) defer log.SetOutput(os.Stderr) path := filepath.Join(t.TempDir(), "users.db") - s := newChannelKeySet(configuredKeys(), path, 128) + s := newApprovedTestKeySet(t, configuredKeys(), path, 128) defer s.Close() s.refresh() s.refresh() @@ -166,10 +219,14 @@ func TestChannelKeySetMissingFileKeepsConfiguredAndLogsOnce(t *testing.T) { func TestChannelKeySetReadErrorKeepsLastGoodSet(t *testing.T) { path := filepath.Join(t.TempDir(), "users.db") writeUsersDB(t, path, testProposal{"#mycity", "approved", 1}) - s := newChannelKeySet(configuredKeys(), path, 128) + s := newApprovedTestKeySet(t, configuredKeys(), path, 128) defer s.Close() s.refresh() - execUsersDB(t, path, `DROP TABLE proposals`) + statement := `DROP VIEW approved_channels` + if approvedTestBackend(t) == dbconfig.SQLite { + statement = `DROP TABLE proposals` + } + execUsersDB(t, path, statement) s.refresh() if _, ok := s.Snapshot()["#mycity"]; !ok { t.Fatal("a failing read removed an approved key") @@ -181,7 +238,7 @@ func TestChannelKeySetSkipsNamesThatFailTheRules(t *testing.T) { writeUsersDB(t, path, testProposal{"#public", "approved", 1}, testProposal{"nohash", "approved", 2}, testProposal{"#" + strings.Repeat("a", 31), "approved", 3}, testProposal{"#ok", "approved", 4}) - s := newChannelKeySet(configuredKeys(), path, 128) + s := newApprovedTestKeySet(t, configuredKeys(), path, 128) defer s.Close() s.refresh() snap := s.Snapshot() @@ -193,7 +250,7 @@ func TestChannelKeySetSkipsNamesThatFailTheRules(t *testing.T) { func TestChannelKeySetCapKeepsOldestApprovals(t *testing.T) { path := filepath.Join(t.TempDir(), "users.db") writeUsersDB(t, path, testProposal{"#c", "approved", 30}, testProposal{"#a", "approved", 10}, testProposal{"#b", "approved", 20}) - s := newChannelKeySet(configuredKeys(), path, 2) + s := newApprovedTestKeySet(t, configuredKeys(), path, 2) defer s.Close() s.refresh() snap := s.Snapshot() @@ -205,35 +262,39 @@ func TestChannelKeySetCapKeepsOldestApprovals(t *testing.T) { func TestChannelKeySetIsReadOnly(t *testing.T) { path := filepath.Join(t.TempDir(), "users.db") writeUsersDB(t, path, testProposal{"#mycity", "approved", 1}) - s := newChannelKeySet(configuredKeys(), path, 128) + s := newApprovedTestKeySet(t, configuredKeys(), path, 128) defer s.Close() s.refresh() _, err := s.db.Exec(`INSERT INTO proposals (kind, subject, status, created_at) VALUES ('hashtag_channel', '#x', 'approved', 1)`) - if err == nil || !strings.Contains(strings.ToLower(err.Error()), "readonly") { + if err == nil || !strings.Contains(strings.ToLower(err.Error()), "read") { t.Fatalf("write through the ingestor's users.db handle: err = %v; want a read-only refusal", err) } - if !strings.Contains(usersDBReadOnlyDSN(path), "mode=ro") { - t.Fatal("DSN without mode=ro") + if approvedTestBackend(t) == dbconfig.Postgres { + if _, err := s.db.Exec(`SET default_transaction_read_only=off`); err != nil { + t.Fatal(err) + } + if _, err := s.db.Query(`SELECT * FROM proposals`); err == nil { + t.Fatal("projection reader can read proposals table") + } } + } // The server keeps users.db in WAL mode with a writer connection open. func TestChannelKeySetReadsWALWithOpenWriter(t *testing.T) { path := filepath.Join(t.TempDir(), "users.db") writeUsersDB(t, path, testProposal{"#mycity", "approved", 1}) - w, err := sql.Open("sqlite3", path) + w, err := openApprovedFixture(t, approvedFixture(t, path).owner) if err != nil { t.Fatal(err) } defer w.Close() w.SetMaxOpenConns(1) - if _, err := w.Exec(`PRAGMA journal_mode=WAL`); err != nil { - t.Fatal(err) - } + if _, err := w.Exec(`INSERT INTO proposals (kind, subject, status, created_at, decided_at) VALUES ('hashtag_channel', '#walcity', 'approved', 9, 9)`); err != nil { t.Fatal(err) } - s := newChannelKeySet(configuredKeys(), path, 128) + s := newApprovedTestKeySet(t, configuredKeys(), path, 128) defer s.Close() s.refresh() if s.Snapshot()["#walcity"] == "" || s.Snapshot()["#mycity"] == "" { @@ -256,7 +317,7 @@ func grpTxtFor(t testing.TB, channelName, text string) []byte { func TestChannelKeySetConcurrentDecodeWhileSwapping(t *testing.T) { path := filepath.Join(t.TempDir(), "users.db") writeUsersDB(t, path, testProposal{"#mycity", "approved", 1}) - s := newChannelKeySet(configuredKeys(), path, 128) + s := newApprovedTestKeySet(t, configuredKeys(), path, 128) defer s.Close() s.refresh() buf := grpTxtFor(t, "#mycity", "alice: hi") @@ -338,8 +399,8 @@ func TestApprovedChannelsConfig(t *testing.T) { return cfg } off := load(``) - if off.ApprovedChannelsEnabled() || off.ApprovedChannelsMax() != 128 || off.UsersDBPath() != filepath.Join(dir, "data", "users.db") { - t.Fatalf("no block: enabled=%v max=%d path=%q", off.ApprovedChannelsEnabled(), off.ApprovedChannelsMax(), off.UsersDBPath()) + if off.ApprovedChannelsEnabled() || off.ApprovedChannelsMax() != 128 || off.ApprovedChannelsURL() != "" { + t.Fatalf("no block: enabled=%v max=%d path=%q", off.ApprovedChannelsEnabled(), off.ApprovedChannelsMax(), off.ApprovedChannelsURL()) } on := load(`, "userManagement": {"enabled": true, "adminEmails": ["a@example.org"], "mail": {"provider": "brevo"}, "channelProposals": {"enabled": true}}`) if !on.ApprovedChannelsEnabled() || on.ApprovedChannelsMax() != 128 { @@ -354,9 +415,9 @@ func TestApprovedChannelsConfig(t *testing.T) { t.Errorf("%s resolved as on", um) } } - custom := load(`, "userManagement": {"enabled": true, "dbPath": "accounts/users.db", "channelProposals": {"enabled": true, "maxApproved": 7}}`) - if custom.UsersDBPath() != "accounts/users.db" || custom.ApprovedChannelsMax() != 7 { - t.Fatalf("custom: path=%q max=%d", custom.UsersDBPath(), custom.ApprovedChannelsMax()) + custom := load(`, "userManagement": {"enabled": true, "approvedChannelsDatabaseURL": "postgres://reader@localhost/accounts", "channelProposals": {"enabled": true, "maxApproved": 7}}`) + if custom.ApprovedChannelsURL() != "postgres://reader@localhost/accounts" || custom.ApprovedChannelsMax() != 7 { + t.Fatalf("custom: path=%q max=%d", custom.ApprovedChannelsURL(), custom.ApprovedChannelsMax()) } if load(`, "userManagement": {"enabled": true, "channelProposals": {"enabled": true, "maxApproved": -3}}`).ApprovedChannelsMax() != 128 { t.Fatal("negative maxApproved must fall back to 128") @@ -365,17 +426,133 @@ func TestApprovedChannelsConfig(t *testing.T) { // The startup line shows the absolute users.db path, so a relative path // resolved against another working directory than the server's is visible. -func TestApprovedChannelsSourceLogIsAbsolute(t *testing.T) { +func TestApprovedChannelsSourceLogRedactsCredentials(t *testing.T) { var buf bytes.Buffer prev := log.Writer() log.SetOutput(&buf) defer log.SetOutput(prev) - logApprovedChannelsSource("users.db") - want, err := filepath.Abs("users.db") + logApprovedChannelsSource("postgres://reader:secret-password@localhost/accounts") + if strings.Contains(buf.String(), "secret-password") || strings.Contains(buf.String(), "postgres://") { + t.Fatal("database URL leaked in log") + } +} + +type approvedFixtureDB struct{ owner, reader, role string } + +var approvedFixtures sync.Map + +func approvedFixture(t testing.TB, key string) approvedFixtureDB { + t.Helper() + if approvedTestBackend(t) == dbconfig.SQLite { + return approvedFixtureDB{owner: key, reader: key} + } + key = t.Name() + "|" + key + if v, ok := approvedFixtures.Load(key); ok { + return v.(approvedFixtureDB) + } + owner := pgtest.NewSchema(t) + u, _ := url.Parse(owner) + schema := u.Query().Get("search_path") + role := schema + "_reader" + db, err := pgutil.Open(owner, false) if err != nil { t.Fatal(err) } - if !strings.Contains(buf.String(), "[proposals] reading approved channels from "+want+"\n") { - t.Fatalf("log line = %q, want the absolute path %q", buf.String(), want) + for _, stmt := range []string{`CREATE ROLE ` + role + ` LOGIN PASSWORD '` + role + `'`, `GRANT USAGE ON SCHEMA ` + schema + ` TO ` + role} { + if _, err := db.Exec(stmt); err != nil { + t.Fatal(err) + } + } + u.User = url.UserPassword(role, role) + f := approvedFixtureDB{owner, u.String(), role} + approvedFixtures.Store(key, f) + t.Cleanup(func() { + approvedFixtures.Delete(key) + db.Exec(`DROP OWNED BY ` + role) + db.Exec(`DROP ROLE ` + role) + db.Close() + }) + return f +} +func newApprovedTestKeySet(t testing.TB, configured map[string]string, key string, max int) *channelKeySet { + return newChannelKeySet(configured, approvedFixture(t, key).reader, max) +} + +func approvedTestBackend(t testing.TB) dbconfig.Backend { + t.Helper() + switch os.Getenv("CORESCOPE_TEST_BACKEND") { + case "", "sqlite": + return dbconfig.SQLite + case "postgres": + return dbconfig.Postgres + default: + t.Fatal("invalid CORESCOPE_TEST_BACKEND") + return "" + } +} +func openApprovedFixture(t testing.TB, path string) (*sql.DB, error) { + t.Helper() + if approvedTestBackend(t) == dbconfig.Postgres { + return pgutil.Open(path, false) + } + uri, err := dbconfig.SQLiteURI(path, url.Values{"_journal_mode": {"WAL"}, "_busy_timeout": {"5000"}}) + if err != nil { + return nil, err + } + return sql.Open("sqlite3", uri) +} + +func TestApprovedChannelsStorageChoiceDoesNotFallback(t *testing.T) { + path := filepath.Join(t.TempDir(), "accounts.db") + // A selected PostgreSQL target cannot fall back even when a local file exists. + if approvedTestBackend(t) == dbconfig.SQLite { + writeUsersDB(t, path, testProposal{"#local", "approved", 1}) + } + s := newChannelKeySetStorage(configuredKeys(), dbconfig.Storage{Backend: dbconfig.Postgres, UsersDBPath: path}, 128) + defer s.Close() + if _, err := s.readApproved(); err != errUsersDBMissing { + t.Fatalf("missing PostgreSQL target error=%v", err) + } + s.refresh() + if len(s.Snapshot()) != 2 { + t.Fatal("invalid backend selection added account keys") + } +} + +func TestApprovedChannelsSQLiteVersionsAndLiteralPath(t *testing.T) { + if approvedTestBackend(t) != dbconfig.SQLite { + t.Skip("native SQLite contract") + } + path := filepath.Join(t.TempDir(), "account # % ü.db") + writeUsersDB(t, path, testProposal{"#local", "approved", 1}) + s := newChannelKeySetStorage(configuredKeys(), dbconfig.Storage{Backend: dbconfig.SQLite, UsersDBPath: path, ApprovedChannelsDatabaseURL: "postgres://unused.invalid/accounts"}, 128) + defer s.Close() + for _, version := range []int{4, 5, 6, 7} { + execUsersDB(t, path, fmt.Sprintf(`UPDATE schema_version SET version=%d`, version)) + names, err := s.readApproved() + if err != nil || len(names) != 1 || names[0] != "#local" { + t.Fatalf("version%d names=%v,%v", version, names, err) + } + } + s.refresh() + for _, version := range []int{3, 8} { + execUsersDB(t, path, fmt.Sprintf(`UPDATE schema_version SET version=%d`, version)) + if _, err := s.readApproved(); err == nil { + t.Fatalf("unsupported version%d accepted", version) + } + s.refresh() + if s.Snapshot()["#local"] == "" { + t.Fatal("unsupported schema removed last good key") + } + } + if s.db.Stats().MaxOpenConnections != 1 { + t.Fatal("account reader pool is not bounded") + } + // mode=ro remains authoritative even when a caller disables query_only. + if _, err := s.db.Exec(`PRAGMA query_only=OFF`); err != nil { + t.Fatal(err) + } + if _, err := s.db.Exec(`DELETE FROM proposals`); err == nil { + t.Fatal("SQLite account reader became writable") } } diff --git a/cmd/ingestor/async_migration.go b/cmd/ingestor/async_migration.go index 6b10501ee..40070f8bc 100644 --- a/cmd/ingestor/async_migration.go +++ b/cmd/ingestor/async_migration.go @@ -1,31 +1,7 @@ -// Async migration helper — runs schema/backfill work that may take minutes on -// large prod tables WITHOUT blocking ingestor startup. -// -// MIGRATION ANNOTATION CONVENTION (read this before touching migrations): -// -// Sync schema/data migrations (CREATE INDEX, ALTER TABLE, UPDATE ... WHERE) -// that run inline during OpenStore() block the ingestor from accepting -// packets until they finish. On an empty dev DB they return in milliseconds; -// at prod scale (1.9M+ observations, 80K+ adverts) they can pin the boot -// for minutes and trigger restart loops. This regression class has bitten us -// repeatedly (#791 resolved_path backfill, #1483 obs_observer_ts_idx_v1). -// -// ANY new CREATE INDEX / ALTER TABLE / data-rewrite migration MUST EITHER: -// 1. Run via Store.RunAsyncMigration(...) below (preferred for backfills -// and any work that may touch >1K rows). The migration is recorded as -// `pending_async` immediately, returns to the caller (boot proceeds), -// and completes in a goroutine. Status flips to `done`, `cancelled`, -// or `failed` when fn returns; interruptions retain the error message. -// 2. Carry the preflight annotation comment immediately above the -// migration block, e.g. -// // PREFLIGHT: async=true reason="" -// Use this for migrations that are genuinely cheap at any scale -// (e.g. ALTER TABLE ADD COLUMN, CREATE INDEX on a known-bounded -// table). The annotation is grepped by -// ~/.openclaw/skills/pr-preflight/scripts/check-async-migrations.sh -// — its absence on a touched migration block is a hard-fail gate. -// -// See MIGRATIONS.md in the repo root for the full policy and examples. +// Async data backfills resume through durable bookkeeping. Schema DDL belongs +// to the SQLite writer or PostgreSQL owner; PostgreSQL callbacks must not +// create or alter tables/indexes. Long backfills use bounded transactions and +// yield between batches so live ingestion can continue. package main @@ -34,22 +10,21 @@ import ( "database/sql" "errors" "fmt" + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/dbschema" "log" ) -// ensureAsyncMigrationsTable creates the bookkeeping table used by -// RunAsyncMigration / AsyncMigrationStatus. Idempotent. -func ensureAsyncMigrationsTable(db *sql.DB) error { - _, err := db.Exec(` - CREATE TABLE IF NOT EXISTS _async_migrations ( - name TEXT PRIMARY KEY, - status TEXT NOT NULL, -- pending_async | done | cancelled | failed - started_at TEXT NOT NULL DEFAULT (datetime('now')), - ended_at TEXT, - error TEXT - ) - `) - return err +// SQLite owns its ledger; PostgreSQL checks the bootstrap-owned ledger without DDL. +func ensureAsyncMigrationsTable(db *sql.DB, backend dbconfig.Backend) error { + if backend == dbconfig.SQLite { + return dbschema.EnsureSQLiteAsyncMigrations(db) + } + rows, err := db.Query(`SELECT name FROM _async_migrations LIMIT 0`) + if err != nil { + return err + } + return rows.Close() } // RunAsyncMigration registers `name` as a pending async migration and @@ -67,27 +42,30 @@ func ensureAsyncMigrationsTable(db *sql.DB) error { // - The caller's WaitGroup tracks the goroutine so tests/shutdown can // wait via Store.WaitForAsyncMigrations(). func (s *Store) RunAsyncMigration(ctx context.Context, name string, fn func(context.Context, *sql.DB) error) error { - if err := ensureAsyncMigrationsTable(s.db); err != nil { + if err := ensureAsyncMigrationsTable(s.db, s.Backend()); err != nil { return fmt.Errorf("ensure _async_migrations: %w", err) } var existing string - row := s.db.QueryRow(`SELECT status FROM _async_migrations WHERE name = ?`, name) + row := s.db.QueryRow(`SELECT status FROM _async_migrations WHERE name = `+s.parameter(1), name) switch err := row.Scan(&existing); err { case nil: if existing == "done" { return nil // already complete, nothing to do } // pending_async, cancelled, or failed → reset and retry. - if _, err := s.db.Exec(` + if _, err := s.db.Exec(s.nativeSQL(` UPDATE _async_migrations SET status = 'pending_async', started_at = datetime('now'), ended_at = NULL, error = NULL - WHERE name = ?`, name); err != nil { + WHERE name = ?1`, ` + UPDATE _async_migrations + SET status = 'pending_async', started_at = to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC','YYYY-MM-DD HH24:MI:SS'), ended_at = NULL, error = NULL + WHERE name = `+s.parameter(1)), name); err != nil { return fmt.Errorf("reset async migration %q: %w", name, err) } case sql.ErrNoRows: if _, err := s.db.Exec(` - INSERT INTO _async_migrations (name, status) VALUES (?, 'pending_async')`, + INSERT INTO _async_migrations (name, status) VALUES (`+s.parameter(1)+`, 'pending_async')`, name); err != nil { return fmt.Errorf("register async migration %q: %w", name, err) } @@ -109,19 +87,21 @@ func (s *Store) RunAsyncMigration(ctx context.Context, name string, fn func(cont if errors.Is(runErr, context.Canceled) { status, label = "cancelled", "cancelled" } - if _, err := s.db.Exec(` - UPDATE _async_migrations - SET status = ?, ended_at = datetime('now'), error = ? - WHERE name = ?`, status, runErr.Error(), name); err != nil { + if _, err := s.db.Exec(s.nativeSQL(` + UPDATE _async_migrations SET status = ?1, ended_at = datetime('now'), error = ?2 WHERE name = ?3`, ` + UPDATE _async_migrations SET status = $1, ended_at = to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC','YYYY-MM-DD HH24:MI:SS'), error = $2 WHERE name = $3`), status, runErr.Error(), name); err != nil { log.Printf("[async-migration] failed to record %s for %q: %v", status, name, err) } log.Printf("[async-migration] %q %s: %v", name, label, runErr) return } - if _, err := s.db.Exec(` + if _, err := s.db.Exec(s.nativeSQL(` UPDATE _async_migrations SET status = 'done', ended_at = datetime('now'), error = NULL - WHERE name = ?`, name); err != nil { + WHERE name = ?1`, ` + UPDATE _async_migrations + SET status = 'done', ended_at = to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC','YYYY-MM-DD HH24:MI:SS'), error = NULL + WHERE name = `+s.parameter(1)), name); err != nil { log.Printf("[async-migration] failed to mark %q done: %v", name, err) return } @@ -138,11 +118,11 @@ func (s *Store) RunAsyncMigration(ctx context.Context, name string, fn func(cont // (one of "pending_async", "done", "cancelled", "failed") or sql.ErrNoRows if no such // migration has been registered. func (s *Store) AsyncMigrationStatus(name string) (string, error) { - if err := ensureAsyncMigrationsTable(s.db); err != nil { + if err := ensureAsyncMigrationsTable(s.db, s.Backend()); err != nil { return "", err } var status string - err := s.db.QueryRow(`SELECT status FROM _async_migrations WHERE name = ?`, name).Scan(&status) + err := s.db.QueryRow(`SELECT status FROM _async_migrations WHERE name = `+s.parameter(1), name).Scan(&status) return status, err } diff --git a/cmd/ingestor/async_migration_test.go b/cmd/ingestor/async_migration_test.go index 75974778f..1431f0db3 100644 --- a/cmd/ingestor/async_migration_test.go +++ b/cmd/ingestor/async_migration_test.go @@ -113,7 +113,7 @@ func TestRunAsyncMigration_PanicCapture(t *testing.T) { } var errMsg sql.NullString - if err := s.db.QueryRow(`SELECT error FROM _async_migrations WHERE name = ?`, name).Scan(&errMsg); err != nil { + if err := s.db.QueryRow(`SELECT error FROM _async_migrations WHERE name = $1`, name).Scan(&errMsg); err != nil { t.Fatalf("error column lookup: %v", err) } if !errMsg.Valid || errMsg.String == "" { @@ -158,7 +158,7 @@ func TestRunAsyncMigration_RestartSafetyInterruptedIsRetried(t *testing.T) { t.Run(previous, func(t *testing.T) { s := newTestStore(t) const name = "test_restart_interrupted_v1" - if _, err := s.db.Exec(`INSERT INTO _async_migrations (name, status, ended_at, error) VALUES (?, ?, '2026-01-01 00:00:00', 'prior interruption')`, name, previous); err != nil { + if _, err := s.db.Exec(`INSERT INTO _async_migrations (name, status, ended_at, error) VALUES ($1, $2, '2026-01-01 00:00:00', 'prior interruption')`, name, previous); err != nil { t.Fatalf("seed interrupted row: %v", err) } @@ -180,7 +180,7 @@ func TestRunAsyncMigration_RestartSafetyInterruptedIsRetried(t *testing.T) { var status string var endedAt, errCol sql.NullString - if err := s.db.QueryRow(`SELECT status, ended_at, error FROM _async_migrations WHERE name = ?`, name).Scan(&status, &endedAt, &errCol); err != nil { + if err := s.db.QueryRow(`SELECT status, ended_at, error FROM _async_migrations WHERE name = $1`, name).Scan(&status, &endedAt, &errCol); err != nil { t.Fatal(err) } if status != "pending_async" || endedAt.Valid || errCol.Valid { @@ -188,7 +188,7 @@ func TestRunAsyncMigration_RestartSafetyInterruptedIsRetried(t *testing.T) { } close(release) s.WaitForAsyncMigrations() - if err := s.db.QueryRow(`SELECT status, ended_at, error FROM _async_migrations WHERE name = ?`, name).Scan(&status, &endedAt, &errCol); err != nil { + if err := s.db.QueryRow(`SELECT status, ended_at, error FROM _async_migrations WHERE name = $1`, name).Scan(&status, &endedAt, &errCol); err != nil { t.Fatal(err) } if status != "done" || !endedAt.Valid || endedAt.String == "" || errCol.Valid || atomic.LoadInt32(&calls) != 1 { @@ -206,10 +206,10 @@ func TestRunAsyncMigration_RestartSafetyPendingIsRetried(t *testing.T) { s := newTestStore(t) const name = "test_restart_pending_v1" - if err := ensureAsyncMigrationsTable(s.db); err != nil { + if err := ensureAsyncMigrationsTable(s.db, s.Backend()); err != nil { t.Fatalf("ensure table: %v", err) } - if _, err := s.db.Exec(`INSERT INTO _async_migrations (name, status) VALUES (?, 'pending_async')`, name); err != nil { + if _, err := s.db.Exec(`INSERT INTO _async_migrations (name, status) VALUES ($1, 'pending_async')`, name); err != nil { t.Fatalf("seed pending row: %v", err) } @@ -266,7 +266,7 @@ func TestRunAsyncMigration_CompletionStatus(t *testing.T) { var status string var endedAt, errCol sql.NullString - if err := s.db.QueryRow(`SELECT status, ended_at, error FROM _async_migrations WHERE name = ?`, name).Scan(&status, &endedAt, &errCol); err != nil { + if err := s.db.QueryRow(`SELECT status, ended_at, error FROM _async_migrations WHERE name = $1`, name).Scan(&status, &endedAt, &errCol); err != nil { t.Fatal(err) } if status != tt.want || !endedAt.Valid || endedAt.String == "" { diff --git a/cmd/ingestor/client_reception.go b/cmd/ingestor/client_reception.go index f3e73609d..ecdc7f39d 100644 --- a/cmd/ingestor/client_reception.go +++ b/cmd/ingestor/client_reception.go @@ -128,7 +128,7 @@ func (s *Store) UpsertClientObserver(pubkey, name, ts string) error { return nil } _, err := s.db.Exec(` - INSERT INTO client_observers (pubkey, name, last_seen) VALUES (?,?,?) + INSERT INTO client_observers (pubkey, name, last_seen) VALUES (`+s.parameter(1)+`,`+s.parameter(2)+`,`+s.parameter(3)+`) ON CONFLICT(pubkey) DO UPDATE SET name = excluded.name, last_seen = excluded.last_seen`, strings.ToLower(pubkey), name, ts) return err @@ -258,7 +258,7 @@ func (s *Store) InsertClientReception(r *ClientReception) (bool, error) { res, err := s.db.Exec(` INSERT INTO client_receptions (rx_pubkey, heard_key, heard_keylen, rssi, snr, lat, lon, pos_acc_m, rx_at, ingested_at, src) - VALUES (?,?,?,?,?,?,?,?,?,?,?) + VALUES (`+s.parameter(1)+`,`+s.parameter(2)+`,`+s.parameter(3)+`,`+s.parameter(4)+`,`+s.parameter(5)+`,`+s.parameter(6)+`,`+s.parameter(7)+`,`+s.parameter(8)+`,`+s.parameter(9)+`,`+s.parameter(10)+`,`+s.parameter(11)+`) ON CONFLICT(rx_pubkey, heard_key, rx_at) DO NOTHING`, r.RxPubkey, r.HeardKey, r.HeardKeyLen, r.RSSI, r.SNR, r.Lat, r.Lon, r.PosAccM, r.RxAt, r.IngestedAt, r.Src) if err != nil { @@ -299,7 +299,7 @@ func (s *Store) InsertClientRfSample(o *ClientRfSample) (bool, error) { uptime_secs, battery_mv, queue_len, errors, noise_floor, last_rssi, last_snr, tx_air_secs, rx_air_secs, recv, sent, flood_rx, direct_rx, flood_tx, direct_tx, recv_errors) - VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) + VALUES (`+s.parameter(1)+`,`+s.parameter(2)+`,`+s.parameter(3)+`,`+s.parameter(4)+`,`+s.parameter(5)+`,`+s.parameter(6)+`,`+s.parameter(7)+`,`+s.parameter(8)+`,`+s.parameter(9)+`,`+s.parameter(10)+`,`+s.parameter(11)+`,`+s.parameter(12)+`,`+s.parameter(13)+`,`+s.parameter(14)+`,`+s.parameter(15)+`,`+s.parameter(16)+`,`+s.parameter(17)+`,`+s.parameter(18)+`,`+s.parameter(19)+`,`+s.parameter(20)+`,`+s.parameter(21)+`,`+s.parameter(22)+`,`+s.parameter(23)+`) ON CONFLICT(rx_pubkey, sampled_at) DO NOTHING`, o.RxPubkey, o.SampledAt, o.IngestedAt, o.Lat, o.Lon, o.PosAccM, boolToInt(o.Stationary), o.UptimeSecs, o.BatteryMV, o.QueueLen, o.Errors, o.NoiseFloor, o.LastRSSI, @@ -418,7 +418,7 @@ func (s *Store) InsertClientRxObservation(o *ClientRxObservation) (bool, error) (rx_pubkey, rx_at, ingested_at, pkt_hash, route_type, payload_type, code1, code2, scope_name, hash_size, hop_count, path_json, forwarder, snr, rssi, lat, lon, pos_acc_m) - VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) + VALUES (`+s.parameter(1)+`,`+s.parameter(2)+`,`+s.parameter(3)+`,`+s.parameter(4)+`,`+s.parameter(5)+`,`+s.parameter(6)+`,`+s.parameter(7)+`,`+s.parameter(8)+`,`+s.parameter(9)+`,`+s.parameter(10)+`,`+s.parameter(11)+`,`+s.parameter(12)+`,`+s.parameter(13)+`,`+s.parameter(14)+`,`+s.parameter(15)+`,`+s.parameter(16)+`,`+s.parameter(17)+`,`+s.parameter(18)+`) ON CONFLICT(rx_pubkey, pkt_hash, rx_at) DO NOTHING`, o.RxPubkey, o.RxAt, o.IngestedAt, o.PktHash, o.RouteType, o.PayloadType, o.Code1, o.Code2, o.ScopeName, o.HashSize, o.HopCount, o.PathJSON, o.Forwarder, @@ -458,7 +458,7 @@ func (s *Store) InsertClientDeclaredRegions(o *ClientDeclaredRegions) (bool, err res, err := s.db.Exec(` INSERT INTO node_declared_regions (target, rx_pubkey, observed_at, ingested_at, regions_csv, truncated, lat, lon, pos_acc_m, repeater_clock) - VALUES (?,?,?,?,?,?,?,?,?,?) + VALUES (`+s.parameter(1)+`,`+s.parameter(2)+`,`+s.parameter(3)+`,`+s.parameter(4)+`,`+s.parameter(5)+`,`+s.parameter(6)+`,`+s.parameter(7)+`,`+s.parameter(8)+`,`+s.parameter(9)+`,`+s.parameter(10)+`) ON CONFLICT(target, rx_pubkey, observed_at) DO NOTHING`, o.Target, o.RxPubkey, o.ObservedAt, o.IngestedAt, o.RegionsCSV, boolToInt(o.Truncated), o.Lat, o.Lon, o.PosAccM, o.RepeaterClock) if err != nil { @@ -478,7 +478,7 @@ func (s *Store) CurrentDeclaredRegions(target string) (*ClientDeclaredRegions, e row := s.db.QueryRow(` SELECT target, rx_pubkey, observed_at, ingested_at, regions_csv, truncated, lat, lon, pos_acc_m, repeater_clock FROM node_declared_regions - WHERE target = ? + WHERE target = `+s.parameter(1)+` ORDER BY observed_at DESC LIMIT 1`, target) var o ClientDeclaredRegions var truncated int diff --git a/cmd/ingestor/client_reception_test.go b/cmd/ingestor/client_reception_test.go index c29662bf4..65c5d3d02 100644 --- a/cmd/ingestor/client_reception_test.go +++ b/cmd/ingestor/client_reception_test.go @@ -54,16 +54,20 @@ func TestPruneOldClientReceptions(t *testing.T) { // this is a regression guard. func TestPruneClientRxObservationsUsesIndex(t *testing.T) { s := newTestStore(t) - rows, err := s.db.Query(`EXPLAIN QUERY PLAN DELETE FROM client_rx_observations WHERE rx_at < ?`, "2026-01-01T00:00:00Z") + if testBackendValue() == "postgres" { + if _, err := s.db.Exec(`SET enable_seqscan=off`); err != nil { + t.Fatal(err) + } + } + rows, err := s.db.Query(testNativeSQL(`EXPLAIN QUERY PLAN DELETE FROM client_rx_observations WHERE rx_at < $1`, `EXPLAIN (COSTS OFF) DELETE FROM client_rx_observations WHERE rx_at < $1`), "2026-01-01T00:00:00Z") if err != nil { t.Fatal(err) } defer rows.Close() plan := "" for rows.Next() { - var id, parent, notused int var detail string - if err := rows.Scan(&id, &parent, ¬used, &detail); err != nil { + if err := scanTestPlan(rows, &detail); err != nil { t.Fatal(err) } plan += detail + "\n" @@ -125,12 +129,12 @@ func TestPruneOldClientRxObservations(t *testing.T) { t.Fatalf("expected 2 observations remaining (recent + boundary), got %d", remaining) } var recentSurvived int - s.db.QueryRow(`SELECT COUNT(*) FROM client_rx_observations WHERE pkt_hash = ?`, "hash-recent").Scan(&recentSurvived) + s.db.QueryRow(`SELECT COUNT(*) FROM client_rx_observations WHERE pkt_hash = $1`, "hash-recent").Scan(&recentSurvived) if recentSurvived != 1 { t.Fatalf("the recent row must survive the prune; got %d", recentSurvived) } var boundarySurvived int - s.db.QueryRow(`SELECT COUNT(*) FROM client_rx_observations WHERE pkt_hash = ?`, "hash-boundary").Scan(&boundarySurvived) + s.db.QueryRow(`SELECT COUNT(*) FROM client_rx_observations WHERE pkt_hash = $1`, "hash-boundary").Scan(&boundarySurvived) if boundarySurvived != 1 { t.Fatalf("boundary row (500ms after cutoff instant) must survive; an RFC3339 (no-ms) cutoff would wrongly delete it because '.' < 'Z' lexicographically — got %d", boundarySurvived) } @@ -139,7 +143,7 @@ func TestPruneOldClientRxObservations(t *testing.T) { func TestClientReceptionsTableExists(t *testing.T) { s := newTestStore(t) cols := map[string]bool{} - rows, err := s.db.Query(`PRAGMA table_info(client_receptions)`) + rows, err := s.db.Query(testNativeSQL(`SELECT cid,name,type,"notnull",dflt_value,pk FROM pragma_table_info('client_receptions') ORDER BY cid`, `SELECT ordinal_position::int,column_name,CASE data_type WHEN 'bigint' THEN 'INTEGER' WHEN 'double precision' THEN 'REAL' ELSE upper(data_type) END,CASE is_nullable WHEN 'NO' THEN 1 ELSE 0 END,column_default,0 FROM information_schema.columns WHERE table_schema=current_schema() AND table_name='client_receptions' ORDER BY ordinal_position`)) if err != nil { t.Fatalf("PRAGMA failed: %v", err) } @@ -168,12 +172,15 @@ func crI(i int) *int { return &i } // per-node coverage query (sargable heard_key IN-list + bbox, mirroring // cmd/server coverageHeardKeyCandidates) seeks the heard_key composite index // rather than scanning the table. Without idx_client_recept_heard_geo the plan -// is "SCAN client_receptions". +// is "Seq Scan on client_receptions". func TestClientReceptionsCoverageQueryUsesIndex(t *testing.T) { s := newTestStore(t) - q := `EXPLAIN QUERY PLAN SELECT lat, lon, snr, rssi, heard_key, rx_at + q := testNativeSQL(`EXPLAIN QUERY PLAN SELECT lat, lon, snr, rssi, heard_key, rx_at + FROM client_receptions + WHERE heard_key IN ($1,$2,$3) AND lat BETWEEN $4 AND $5 AND lon BETWEEN $6 AND $7`, `EXPLAIN (COSTS OFF) SELECT lat, lon, snr, rssi, heard_key, rx_at FROM client_receptions - WHERE heard_key IN (?,?,?) AND lat BETWEEN ? AND ? AND lon BETWEEN ? AND ?` + WHERE heard_key IN ($1,$2,$3) AND lat BETWEEN $4 AND $5 AND lon BETWEEN $6 AND $7`) + s.db.Exec(`SET enable_seqscan=off`) rows, err := s.db.Query(q, "aabbccddeeff00112233", "aabbcc", "aabb", 50.0, 52.0, 3.0, 4.0) if err != nil { t.Fatal(err) @@ -181,17 +188,16 @@ func TestClientReceptionsCoverageQueryUsesIndex(t *testing.T) { defer rows.Close() plan := "" for rows.Next() { - var id, parent, notused int var detail string - if err := rows.Scan(&id, &parent, ¬used, &detail); err != nil { + if err := scanTestPlan(rows, &detail); err != nil { t.Fatal(err) } plan += detail + "\n" } - if !strings.Contains(plan, "USING INDEX idx_client_recept") { + if !strings.Contains(plan, "idx_client_recept") { t.Fatalf("coverage query should use a client_recept index, plan was:\n%s", plan) } - if strings.Contains(plan, "SCAN client_receptions") { + if strings.Contains(plan, "Seq Scan on client_receptions") { t.Fatalf("coverage query should not full-scan, plan was:\n%s", plan) } } @@ -201,16 +207,20 @@ func TestClientReceptionsCoverageQueryUsesIndex(t *testing.T) { // index rather than full-scanning under the writer lock (polish review). func TestClientReceptionsRetentionUsesRxAtIndex(t *testing.T) { s := newTestStore(t) - rows, err := s.db.Query(`EXPLAIN QUERY PLAN DELETE FROM client_receptions WHERE rx_at < ?`, "2026-01-01T00:00:00Z") + if testBackendValue() == "postgres" { + if _, err := s.db.Exec(`SET enable_seqscan=off`); err != nil { + t.Fatal(err) + } + } + rows, err := s.db.Query(testNativeSQL(`EXPLAIN QUERY PLAN DELETE FROM client_receptions WHERE rx_at < $1`, `EXPLAIN (COSTS OFF) DELETE FROM client_receptions WHERE rx_at < $1`), "2026-01-01T00:00:00Z") if err != nil { t.Fatal(err) } defer rows.Close() plan := "" for rows.Next() { - var id, parent, notused int var detail string - if err := rows.Scan(&id, &parent, ¬used, &detail); err != nil { + if err := scanTestPlan(rows, &detail); err != nil { t.Fatal(err) } plan += detail + "\n" @@ -230,22 +240,32 @@ func TestClientReceptionsRetentionUsesRxAtIndex(t *testing.T) { // index-backed). func TestRxLeaderboardQueryIsIndexBacked(t *testing.T) { s := newTestStore(t) - rows, err := s.db.Query(`EXPLAIN QUERY PLAN + if testBackendValue() == "postgres" { + if _, err := s.db.Exec(`SET enable_seqscan=off`); err != nil { + t.Fatal(err) + } + } + rows, err := s.db.Query(testNativeSQL(`EXPLAIN QUERY PLAN + SELECT cr.rx_pubkey, COUNT(*), COUNT(DISTINCT cr.heard_key) + FROM client_receptions cr + WHERE cr.rx_at >= $1 + GROUP BY cr.rx_pubkey + ORDER BY COUNT(*) DESC + LIMIT $2`, `EXPLAIN (COSTS OFF) SELECT cr.rx_pubkey, COUNT(*), COUNT(DISTINCT cr.heard_key) FROM client_receptions cr - WHERE cr.rx_at >= ? + WHERE cr.rx_at >= $1 GROUP BY cr.rx_pubkey ORDER BY COUNT(*) DESC - LIMIT ?`, "2026-01-01T00:00:00Z", 100) + LIMIT $2`), "2026-01-01T00:00:00Z", 100) if err != nil { t.Fatal(err) } defer rows.Close() plan := "" for rows.Next() { - var id, parent, notused int var detail string - if err := rows.Scan(&id, &parent, ¬used, &detail); err != nil { + if err := scanTestPlan(rows, &detail); err != nil { t.Fatal(err) } plan += detail + "\n" @@ -254,7 +274,7 @@ func TestRxLeaderboardQueryIsIndexBacked(t *testing.T) { // The concern is a bare table-heap scan, not which specific index wins. The // plan must stay index-backed (covering or search) — a regression to a bare // "SCAN cr" without an index fails here. - if !strings.Contains(plan, "INDEX") { + if !strings.Contains(strings.ToLower(plan), "index") { t.Fatalf("leaderboard SELECT must stay index-backed (no full table-heap scan), plan was:\n%s", plan) } } @@ -380,7 +400,7 @@ func TestHandleClientPacketRelayedAdvertWritesReception(t *testing.T) { handleClientPacket(s, &Config{}, "test", testCompanionPK, msg, nil, nil) var obsName string - s.db.QueryRow(`SELECT name FROM client_observers WHERE pubkey=?`, testCompanionPK).Scan(&obsName) + s.db.QueryRow(`SELECT name FROM client_observers WHERE pubkey=$1`, testCompanionPK).Scan(&obsName) if obsName != "MyMob" { t.Fatalf("expected client_observers name 'MyMob', got %q", obsName) } @@ -390,7 +410,7 @@ func TestHandleClientPacketRelayedAdvertWritesReception(t *testing.T) { // The 0-hop advert→full-pubkey branch is covered by TestDeriveHeardKey. var n, keylen int var src string - if err := s.db.QueryRow(`SELECT COUNT(*), COALESCE(MAX(heard_keylen),0), COALESCE(MAX(src),'') FROM client_receptions WHERE rx_pubkey=?`, testCompanionPK).Scan(&n, &keylen, &src); err != nil { + if err := s.db.QueryRow(`SELECT COUNT(*), COALESCE(MAX(heard_keylen),0), COALESCE(MAX(src),'') FROM client_receptions WHERE rx_pubkey=$1`, testCompanionPK).Scan(&n, &keylen, &src); err != nil { t.Fatal(err) } if n != 1 || keylen < 2 || src != "rxlog" { @@ -401,7 +421,7 @@ func TestHandleClientPacketRelayedAdvertWritesReception(t *testing.T) { const companion2 = "b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3" handleClientPacket(s, &Config{}, "test", companion2, map[string]interface{}{"raw": advertHex, "direction": "rx"}, nil, nil) var n2 int - s.db.QueryRow(`SELECT COUNT(*) FROM client_receptions WHERE rx_pubkey=?`, companion2).Scan(&n2) + s.db.QueryRow(`SELECT COUNT(*) FROM client_receptions WHERE rx_pubkey=$1`, companion2).Scan(&n2) if n2 != 0 { t.Fatalf("packet without gps must be dropped, got %d rows", n2) } @@ -432,7 +452,7 @@ func TestHandleClientPacketZeroHopAdvertWritesReception(t *testing.T) { var keylen int var snr sql.NullFloat64 var lat, lon float64 - if err := s.db.QueryRow(`SELECT heard_key, heard_keylen, src, snr, lat, lon FROM client_receptions WHERE rx_pubkey=?`, testCompanionPK). + if err := s.db.QueryRow(`SELECT heard_key, heard_keylen, src, snr, lat, lon FROM client_receptions WHERE rx_pubkey=$1`, testCompanionPK). Scan(&heardKey, &keylen, &src, &snr, &lat, &lon); err != nil { t.Fatalf("expected a 0-hop advert reception: %v", err) } @@ -560,7 +580,7 @@ func TestHandleClientPacketDiscoverRespWritesReception(t *testing.T) { var heardKey, src string var keylen int - if err := s.db.QueryRow(`SELECT heard_key, heard_keylen, src FROM client_receptions WHERE rx_pubkey=?`, "aa11"). + if err := s.db.QueryRow(`SELECT heard_key, heard_keylen, src FROM client_receptions WHERE rx_pubkey=$1`, "aa11"). Scan(&heardKey, &keylen, &src); err != nil { t.Fatalf("expected a discover-response reception: %v", err) } @@ -709,7 +729,7 @@ func TestClientObservationScopeNameFromTransportCode(t *testing.T) { // same content dedups across scopes), so raw and raw2 below share one // pkt_hash — distinguish the two inserted rows by rx_at, not pkt_hash. var code1, code2, scopeName sql.NullString - if err := s.db.QueryRow(`SELECT code1, code2, scope_name FROM client_rx_observations WHERE rx_at = ?`, ts1). + if err := s.db.QueryRow(`SELECT code1, code2, scope_name FROM client_rx_observations WHERE rx_at = $1`, ts1). Scan(&code1, &code2, &scopeName); err != nil { t.Fatal(err) } @@ -735,7 +755,7 @@ func TestClientObservationScopeNameFromTransportCode(t *testing.T) { handleClientPacket(s, cfgWithObservations(), "test", "aa11", msg2, nil, regionKeySetFromKeys(regionKeys)) var code1b, scopeNameB sql.NullString - if err := s.db.QueryRow(`SELECT code1, scope_name FROM client_rx_observations WHERE rx_at = ?`, ts2). + if err := s.db.QueryRow(`SELECT code1, scope_name FROM client_rx_observations WHERE rx_at = $1`, ts2). Scan(&code1b, &scopeNameB); err != nil { t.Fatal(err) } diff --git a/cmd/ingestor/client_regions_test.go b/cmd/ingestor/client_regions_test.go index f8ee7e1b0..7ceb1daa9 100644 --- a/cmd/ingestor/client_regions_test.go +++ b/cmd/ingestor/client_regions_test.go @@ -442,7 +442,7 @@ func TestDeclaredRegionsDifferentReportersPersistBoth(t *testing.T) { } var n int - if err := s.db.QueryRow(`SELECT COUNT(*) FROM node_declared_regions WHERE target = ? AND observed_at = ?`, "bb11", at).Scan(&n); err != nil { + if err := s.db.QueryRow(`SELECT COUNT(*) FROM node_declared_regions WHERE target = $1 AND observed_at = $2`, "bb11", at).Scan(&n); err != nil { t.Fatal(err) } if n != 2 { @@ -480,16 +480,20 @@ func TestCurrentDeclaredRegionsNormalizesTargetCase(t *testing.T) { // index seek on idx_ndr_prune rather than a full table scan. func TestPruneClientDeclaredRegionsUsesIndex(t *testing.T) { s := newTestStore(t) - rows, err := s.db.Query(`EXPLAIN QUERY PLAN DELETE FROM node_declared_regions WHERE observed_at < ?`, "2026-01-01T00:00:00Z") + if testBackendValue() == "postgres" { + if _, err := s.db.Exec(`SET enable_seqscan=off`); err != nil { + t.Fatal(err) + } + } + rows, err := s.db.Query(testNativeSQL(`EXPLAIN QUERY PLAN DELETE FROM node_declared_regions WHERE observed_at < $1`, `EXPLAIN (COSTS OFF) DELETE FROM node_declared_regions WHERE observed_at < $1`), "2026-01-01T00:00:00Z") if err != nil { t.Fatal(err) } defer rows.Close() plan := "" for rows.Next() { - var id, parent, notused int var detail string - if err := rows.Scan(&id, &parent, ¬used, &detail); err != nil { + if err := scanTestPlan(rows, &detail); err != nil { t.Fatal(err) } plan += detail + "\n" @@ -592,7 +596,7 @@ func TestPruneOldClientDeclaredRegionsAtMillisecondBoundary(t *testing.T) { t.Fatalf("expected 1 row pruned (1ms before the cutoff), got %d", n) } var boundarySurvived int - s.db.QueryRow(`SELECT COUNT(*) FROM node_declared_regions WHERE observed_at = ?`, boundary).Scan(&boundarySurvived) + s.db.QueryRow(`SELECT COUNT(*) FROM node_declared_regions WHERE observed_at = $1`, boundary).Scan(&boundarySurvived) if boundarySurvived != 1 { t.Fatalf("boundary row (1ms after a cutoff with a non-zero millisecond component) must survive; an RFC3339 (no-ms) cutoff would wrongly delete it because '.' < 'Z' lexicographically — got %d", boundarySurvived) } diff --git a/cmd/ingestor/client_rf_sample.go b/cmd/ingestor/client_rf_sample.go index 46a8f5a26..ad307f112 100644 --- a/cmd/ingestor/client_rf_sample.go +++ b/cmd/ingestor/client_rf_sample.go @@ -134,7 +134,7 @@ func (s *Store) ClientRfDeltas(rxPubkey, from, to string) ([]ClientRfDelta, erro LAG(recv) OVER w AS prev_recv, LAG(recv_errors) OVER w AS prev_errs FROM client_rf_samples - WHERE rx_pubkey = ? AND sampled_at >= ? AND sampled_at < ? + WHERE rx_pubkey = `+s.parameter(1)+` AND sampled_at >= `+s.parameter(2)+` AND sampled_at < `+s.parameter(3)+` WINDOW w AS (ORDER BY sampled_at) ORDER BY sampled_at`, rxPubkey, from, to) if err != nil { diff --git a/cmd/ingestor/config.go b/cmd/ingestor/config.go index 0279e18d4..2aa65c895 100644 --- a/cmd/ingestor/config.go +++ b/cmd/ingestor/config.go @@ -6,7 +6,6 @@ import ( "fmt" "log" "os" - "path/filepath" "strings" "sync" "time" @@ -46,8 +45,10 @@ type MQTTLegacy struct { Topic string `json:"topic"` } -// Config holds the ingestor configuration, compatible with the Node.js config.json format. +// Config holds the shared Go-service configuration and ingestor-only settings. type Config struct { + DatabaseURL string `json:"databaseURL"` + StateDir string `json:"stateDir"` DBPath string `json:"dbPath"` MQTT *MQTTLegacy `json:"mqtt,omitempty"` MQTTSources []MQTTSource `json:"mqttSources,omitempty"` @@ -107,7 +108,7 @@ type Config struct { NeighborEdgesMaxAgeDays int `json:"neighborEdgesMaxAgeDays,omitempty"` // IngestBufferSize caps the in-memory queue (number of MQTT messages) held - // while the single SQLite writer is blocked by startup migrations/prunes + // while the single serialized writer is blocked by startup migrations/prunes // (#1608). Received messages are drained once the write path is ready. // 0 / unset => default. Bounded memory. IngestBufferSize int `json:"ingestBufferSize,omitempty"` @@ -254,14 +255,15 @@ type RetentionConfig struct { // region-discovery stream would otherwise grow without limit. Consumed by // Task 6's pruner. ClientRegionsDays int `json:"clientRegionsDays"` - // HopIndexDays turns on the observation_hops inverted index (the + // HopIndexDays turns on the SQLite observation_hops inverted index (the // accelerator behind /api/nodes/{pk}/reach) and bounds it to this many // days. 0 or unset = off: no table, no triggers, and a previously installed // index is removed at startup. The index costs roughly 200 bytes per // observation (about 30% of the observations table, measured on a real // 10M-observation database; real paths average 10 to 12 hops) plus a // trigger on every observation insert. Reach windows longer than this fall - // back to the substring scan. + // back to the substring scan. PostgreSQL uses its native scan regardless + // of this value; ConfigureObservationHops logs that backend limitation. HopIndexDays int `json:"hopIndexDays"` } @@ -340,10 +342,10 @@ type RuntimeConfig struct { MaxMemoryMB int `json:"maxMemoryMB"` } -// DBConfig is the shared SQLite vacuum/maintenance config (#919, #921). +// DBConfig shares startup-load settings and accepts obsolete SQLite config keys. type DBConfig = dbconfig.DBConfig -// IncrementalVacuumPages returns the configured pages per vacuum or 1024 default. +// IncrementalVacuumPages preserves legacy config decoding; PostgreSQL does not use it. func (c *Config) IncrementalVacuumPages() int { if c.DB != nil && c.DB.IncrementalVacuumPages > 0 { return c.DB.IncrementalVacuumPages @@ -351,16 +353,8 @@ func (c *Config) IncrementalVacuumPages() int { return 1024 } -// AnalysisLimit returns the per-index row cap for the planner stats refresh -// (#2058). A negative setting disables the refresh; zero means unset, matching -// IncrementalVacuumPages above. -// -// 10000 rather than the 400 SQLite's documentation offers: measured on the -// 9.4 GB staging database, 400 and 1000 leave the channel-query plan exactly as -// it was, 10000 produces the same plan as an unbounded ANALYZE, and it costs 2.0s -// against that ANALYZE's 242.9s, both timed warm. The full ladder, and the cold -// figure that matters at startup, are in Store.RefreshPlannerStats and -// Store.EnsurePlannerStats. +// AnalysisLimit preserves the old SQLite setting for configuration compatibility. +// PostgreSQL owns planner statistics; runtime maintenance does not use this limit. func (c *Config) AnalysisLimit() int { if c.DB != nil && c.DB.AnalysisLimit != 0 { return c.DB.AnalysisLimit @@ -581,10 +575,23 @@ func LoadConfig(path string) (*Config, error) { }} } - // Default DB path - if cfg.DBPath == "" { - cfg.DBPath = "data/meshcore.db" + if v := os.Getenv("CORESCOPE_DATABASE_URL"); v != "" { + cfg.DatabaseURL = v + } + if v := os.Getenv("CORESCOPE_WRITER_DATABASE_URL"); v != "" { + cfg.DatabaseURL = v + } + if v := os.Getenv("CORESCOPE_STATE_DIR"); v != "" { + cfg.StateDir = v } + if v := os.Getenv("CORESCOPE_APPROVED_CHANNELS_DATABASE_URL"); v != "" { + if cfg.UserManagement == nil { + cfg.UserManagement = &UserManagementConfig{} + } + cfg.UserManagement.ApprovedChannelsDatabaseURL = v + } + // Legacy DBPath is never interpreted as a connection string. + // Default DB path // Normalize: convert legacy single mqtt config to mqttSources if len(cfg.MQTTSources) == 0 && cfg.MQTT != nil && cfg.MQTT.Broker != "" { @@ -661,9 +668,11 @@ const autoRegionKeysDefaultRefreshMinutes = 15 // UserManagementConfig is the ingestor's view of userManagement. type UserManagementConfig struct { - Enabled bool `json:"enabled"` - DBPath string `json:"dbPath,omitempty"` - ChannelProposals *ChannelProposalsConfig `json:"channelProposals,omitempty"` + DatabaseURL string `json:"databaseURL,omitempty"` + Enabled bool `json:"enabled"` + ApprovedChannelsDatabaseURL string `json:"approvedChannelsDatabaseURL,omitempty"` + DBPath string `json:"dbPath,omitempty"` + ChannelProposals *ChannelProposalsConfig `json:"channelProposals,omitempty"` } // ChannelProposalsConfig is the ingestor's view of @@ -692,15 +701,12 @@ func (c *Config) ApprovedChannelsMax() int { return u.ChannelProposals.MaxApproved } -// UsersDBPath resolves users.db with the server's rule: userManagement.dbPath, -// else users.db next to the analyzer database. -func (c *Config) UsersDBPath() string { - if c.UserManagement != nil { - if p := strings.TrimSpace(c.UserManagement.DBPath); p != "" { - return p - } +// ApprovedChannelsURL is a separate restricted account-reader credential. +func (c *Config) ApprovedChannelsURL() string { + if c.UserManagement == nil { + return "" } - return filepath.Join(filepath.Dir(c.DBPath), "users.db") + return strings.TrimSpace(c.UserManagement.ApprovedChannelsDatabaseURL) } // AutoRegionKeysEnabled reports whether region keys may be derived from diff --git a/cmd/ingestor/config_test.go b/cmd/ingestor/config_test.go index e5d79e3dc..52657c9c0 100644 --- a/cmd/ingestor/config_test.go +++ b/cmd/ingestor/config_test.go @@ -40,8 +40,8 @@ func TestLoadConfigMissingFile(t *testing.T) { if err != nil { t.Fatalf("missing config should not error (zero-config mode), got: %v", err) } - if cfg.DBPath != "data/meshcore.db" { - t.Errorf("dbPath=%s, want data/meshcore.db", cfg.DBPath) + if cfg.DBPath != "" || cfg.DatabaseURL != "" || cfg.StateDir != "" { + t.Errorf("unexpected database/state defaults") } // Should default to localhost MQTT if len(cfg.MQTTSources) != 1 { @@ -193,8 +193,8 @@ func TestLoadConfigDefaultDBPath(t *testing.T) { if err != nil { t.Fatal(err) } - if cfg.DBPath != "data/meshcore.db" { - t.Errorf("dbPath=%s, want data/meshcore.db", cfg.DBPath) + if cfg.DBPath != "" || cfg.DatabaseURL != "" || cfg.StateDir != "" { + t.Errorf("unexpected database/state defaults") } } diff --git a/cmd/ingestor/coverage_boost_test.go b/cmd/ingestor/coverage_boost_test.go index fc2daa889..33d86958c 100644 --- a/cmd/ingestor/coverage_boost_test.go +++ b/cmd/ingestor/coverage_boost_test.go @@ -635,7 +635,9 @@ func TestDecryptChannelMessageBadCiphertextHex(t *testing.T) { func TestCheckpointDoesNotPanic(t *testing.T) { store := newTestStore(t) - store.Checkpoint() + if _, err := store.db.Exec(`CHECKPOINT`); err == nil { + t.Fatal("runtime role may force server checkpoints") + } } func TestLogStatsDoesNotPanic(t *testing.T) { @@ -762,7 +764,7 @@ func TestOpenStoreInvalidPath(t *testing.T) { t.Fatalf("setup: %v", err) } f.Close() - _, err = OpenStore(filepath.Join(f.Name(), "db.sqlite")) + _, err = openPostgresTestStore(t, filepath.Join(f.Name(), "db.sqlite")) if err == nil { t.Error("should error on impossible path") } @@ -1025,28 +1027,28 @@ func TestApplySchemaMigrationsOnFreshDB(t *testing.T) { // Check that migrations were recorded var count int - if err := store.db.QueryRow("SELECT COUNT(*) FROM _migrations").Scan(&count); err != nil { + if err := store.db.QueryRow(testNativeSQL(`SELECT COUNT(*) FROM _migrations WHERE name='observers_identity_autoincrement_v1'`, "SELECT COUNT(*) FROM corescope_schema WHERE kind='telemetry' AND ready=true")).Scan(&count); err != nil { t.Fatal(err) } - if count < 3 { - t.Errorf("expected at least 3 migrations recorded, got %d", count) + if count != 1 { + t.Errorf("expected one ready telemetry schema, got %d", count) } // Check observations table exists with dedup index var tblName string - err := store.db.QueryRow("SELECT name FROM sqlite_master WHERE type='table' AND name='observations'").Scan(&tblName) + err := store.db.QueryRow(testNativeSQL(`SELECT name FROM sqlite_master WHERE type='table' AND name='observations'`, "SELECT table_name FROM information_schema.tables WHERE table_schema=current_schema() AND table_name='observations'")).Scan(&tblName) if err != nil { t.Error("observations table should exist") } // Check inactive_nodes table exists - err = store.db.QueryRow("SELECT name FROM sqlite_master WHERE type='table' AND name='inactive_nodes'").Scan(&tblName) + err = store.db.QueryRow(testNativeSQL(`SELECT name FROM sqlite_master WHERE type='table' AND name='inactive_nodes'`, "SELECT table_name FROM information_schema.tables WHERE table_schema=current_schema() AND table_name='inactive_nodes'")).Scan(&tblName) if err != nil { t.Error("inactive_nodes table should exist") } // Check packets_v view exists - err = store.db.QueryRow("SELECT name FROM sqlite_master WHERE type='view' AND name='packets_v'").Scan(&tblName) + err = store.db.QueryRow(testNativeSQL(`SELECT name FROM sqlite_master WHERE type='view' AND name='packets_v'`, "SELECT table_name FROM information_schema.views WHERE table_schema=current_schema() AND table_name='packets_v'")).Scan(&tblName) if err != nil { t.Error("packets_v view should exist") } @@ -1058,14 +1060,14 @@ func TestOpenStoreExistingDB(t *testing.T) { dbPath := dir + "/test.db" // Open and close - s1, err := OpenStore(dbPath) + s1, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatal(err) } s1.Close() // Re-open — should skip migrations (already applied) - s2, err := OpenStore(dbPath) + s2, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatal(err) } @@ -1078,7 +1080,7 @@ func TestMoveStaleNodesReplace(t *testing.T) { pk := "stale_node_replace_0000000000000000000000000001" // Insert into inactive_nodes first - if _, err := store.db.Exec("INSERT INTO inactive_nodes (public_key, name, role, last_seen, first_seen) VALUES (?, 'Old', 'companion', '2019-01-01T00:00:00Z', '2019-01-01T00:00:00Z')", pk); err != nil { + if _, err := store.db.Exec("INSERT INTO inactive_nodes (public_key, name, role, last_seen, first_seen) VALUES ($1, 'Old', 'companion', '2019-01-01T00:00:00Z', '2019-01-01T00:00:00Z')", pk); err != nil { t.Fatal(err) } @@ -1095,7 +1097,7 @@ func TestMoveStaleNodesReplace(t *testing.T) { // Should have replaced the inactive node var name string - if err := store.db.QueryRow("SELECT name FROM inactive_nodes WHERE public_key = ?", pk).Scan(&name); err != nil { + if err := store.db.QueryRow("SELECT name FROM inactive_nodes WHERE public_key = $1", pk).Scan(&name); err != nil { t.Fatal(err) } if name != "StaleNode" { @@ -1165,7 +1167,7 @@ func TestRemoveStaleObservers(t *testing.T) { } // Override last_seen to 30 days ago cutoff := time.Now().UTC().AddDate(0, 0, -30).Format(time.RFC3339) - _, err = store.db.Exec("UPDATE observers SET last_seen = ? WHERE id = ?", cutoff, "obs-old") + _, err = store.db.Exec("UPDATE observers SET last_seen = $1 WHERE id = $2", cutoff, "obs-old") if err != nil { t.Fatal(err) } @@ -1195,7 +1197,7 @@ func TestRemoveStaleObservers(t *testing.T) { // Check that the old observer is marked inactive var inactive int - if err := store.db.QueryRow("SELECT inactive FROM observers WHERE id = ?", "obs-old").Scan(&inactive); err != nil { + if err := store.db.QueryRow("SELECT inactive FROM observers WHERE id = $1", "obs-old").Scan(&inactive); err != nil { t.Fatal(err) } if inactive != 1 { @@ -1204,7 +1206,7 @@ func TestRemoveStaleObservers(t *testing.T) { // Check that the recent observer is still active var newInactive int - if err := store.db.QueryRow("SELECT inactive FROM observers WHERE id = ?", "obs-new").Scan(&newInactive); err != nil { + if err := store.db.QueryRow("SELECT inactive FROM observers WHERE id = $1", "obs-new").Scan(&newInactive); err != nil { t.Fatal(err) } if newInactive != 0 { @@ -1233,7 +1235,7 @@ func TestRemoveStaleObserversKeepForever(t *testing.T) { t.Fatal(err) } cutoff := time.Now().UTC().AddDate(0, 0, -365).Format(time.RFC3339) - _, err = store.db.Exec("UPDATE observers SET last_seen = ? WHERE id = ?", cutoff, "obs-ancient") + _, err = store.db.Exec("UPDATE observers SET last_seen = $1 WHERE id = $2", cutoff, "obs-ancient") if err != nil { t.Fatal(err) } @@ -1257,7 +1259,7 @@ func TestRemoveStaleObserversKeepForever(t *testing.T) { // Observer should NOT be marked inactive var inactive int - if err := store.db.QueryRow("SELECT inactive FROM observers WHERE id = ?", "obs-ancient").Scan(&inactive); err != nil { + if err := store.db.QueryRow("SELECT inactive FROM observers WHERE id = $1", "obs-ancient").Scan(&inactive); err != nil { t.Fatal(err) } if inactive != 0 { @@ -1274,7 +1276,7 @@ func TestRemoveStaleObserversReactivation(t *testing.T) { t.Fatal(err) } cutoff := time.Now().UTC().AddDate(0, 0, -30).Format(time.RFC3339) - _, err = store.db.Exec("UPDATE observers SET last_seen = ? WHERE id = ?", cutoff, "obs-test") + _, err = store.db.Exec("UPDATE observers SET last_seen = $1 WHERE id = $2", cutoff, "obs-test") if err != nil { t.Fatal(err) } @@ -1289,7 +1291,7 @@ func TestRemoveStaleObserversReactivation(t *testing.T) { // Verify it's inactive var inactive int - if err := store.db.QueryRow("SELECT inactive FROM observers WHERE id = ?", "obs-test").Scan(&inactive); err != nil { + if err := store.db.QueryRow("SELECT inactive FROM observers WHERE id = $1", "obs-test").Scan(&inactive); err != nil { t.Fatal(err) } if inactive != 1 { @@ -1302,7 +1304,7 @@ func TestRemoveStaleObserversReactivation(t *testing.T) { t.Fatal(err) } - if err := store.db.QueryRow("SELECT inactive FROM observers WHERE id = ?", "obs-test").Scan(&inactive); err != nil { + if err := store.db.QueryRow("SELECT inactive FROM observers WHERE id = $1", "obs-test").Scan(&inactive); err != nil { t.Fatal(err) } if inactive != 0 { diff --git a/cmd/ingestor/coverage_gate_test.go b/cmd/ingestor/coverage_gate_test.go index 0ba2f3cbb..4ef5eade2 100644 --- a/cmd/ingestor/coverage_gate_test.go +++ b/cmd/ingestor/coverage_gate_test.go @@ -91,7 +91,7 @@ func TestClientRxCoverageGateOffDoesNotFallThroughToObserverPath(t *testing.T) { t.Fatalf("feature OFF: expected 0 client_receptions rows, got %d", n) } var observerRows int - if err := store.db.QueryRow(`SELECT COUNT(*) FROM observers WHERE id = ?`, testCompanionPK).Scan(&observerRows); err != nil { + if err := store.db.QueryRow(`SELECT COUNT(*) FROM observers WHERE id = $1`, testCompanionPK).Scan(&observerRows); err != nil { t.Fatal(err) } if observerRows != 0 { diff --git a/cmd/ingestor/coverage_query_bench_test.go b/cmd/ingestor/coverage_query_bench_test.go index 078ec8643..e8a928816 100644 --- a/cmd/ingestor/coverage_query_bench_test.go +++ b/cmd/ingestor/coverage_query_bench_test.go @@ -12,9 +12,9 @@ import ( // match on the heard node. const coverageBenchSQL = `SELECT lat, lon, snr, rssi, heard_key, rx_at FROM client_receptions - WHERE lat BETWEEN ? AND ? AND lon BETWEEN ? AND ? - AND ( (heard_keylen = 32 AND heard_key = ?) - OR (heard_keylen IN (2,3) AND substr(?, 1, heard_keylen*2) = heard_key) )` + WHERE lat BETWEEN $1 AND $2 AND lon BETWEEN $3 AND $4 + AND ( (heard_keylen = 32 AND heard_key = $5) + OR (heard_keylen IN (2,3) AND substr($6, 1, heard_keylen*2) = heard_key) )` // BenchmarkCoverageQuery seeds ~1M receptions across a metro-area bbox and times // the coverage query with the indexes (#5/#18) versus a forced full table scan. @@ -24,7 +24,7 @@ func BenchmarkCoverageQuery(b *testing.B) { const prefixPool = 2000 // distinct 3-byte heard_key prefixes dir := b.TempDir() - s, err := OpenStore(dir + "/bench.db") + s, err := openPostgresTestStore(b, dir+"/bench.db") if err != nil { b.Fatal(err) } @@ -37,7 +37,7 @@ func BenchmarkCoverageQuery(b *testing.B) { } stmt, err := tx.Prepare(`INSERT INTO client_receptions (rx_pubkey,heard_key,heard_keylen,snr,lat,lon,rx_at,ingested_at,src) - VALUES (?,?,?,?,?,?,?,?,?)`) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)`) if err != nil { b.Fatal(err) } @@ -78,7 +78,7 @@ func BenchmarkCoverageQuery(b *testing.B) { // full pubkey and its 2/3-byte prefixes, so an IN-list seeks them via the // heard_key-leading composite instead of scanning the bbox. inListSQL := `SELECT lat, lon, snr, rssi, heard_key, rx_at FROM client_receptions - WHERE heard_key IN (?,?,?) AND lat BETWEEN ? AND ? AND lon BETWEEN ? AND ?` + WHERE heard_key IN ($1,$2,$3) AND lat BETWEEN $4 AND $5 AND lon BETWEEN $6 AND $7` runIN := func(b *testing.B) { for i := 0; i < b.N; i++ { rows, err := s.db.Query(inListSQL, target, target[:4], target[:6], 51.1, 51.3, 3.6, 3.8) diff --git a/cmd/ingestor/db.go b/cmd/ingestor/db.go index f69c815f8..cefce9909 100644 --- a/cmd/ingestor/db.go +++ b/cmd/ingestor/db.go @@ -15,8 +15,10 @@ import ( "time" _ "github.com/mattn/go-sqlite3" + "github.com/meshcore-analyzer/dbconfig" "github.com/meshcore-analyzer/dbschema" "github.com/meshcore-analyzer/packetpath" + "github.com/meshcore-analyzer/pgutil" ) // DBStats tracks operational metrics for the ingestor database. @@ -28,9 +30,7 @@ type DBStats struct { ObserverUpserts atomic.Int64 WriteErrors atomic.Int64 SignatureDrops atomic.Int64 - // ClientUnlinkedDropped counts meshcore/client//... messages - // dropped by the linked-only filter (clientRxCoverage.requireLinkedCompanion). - ClientUnlinkedDropped atomic.Int64 + ClientUnlinkedDropped atomic.Int64 // RelayTouches counts nodes.last_seen refreshes driven by relay // participation rather than an ADVERT (#1598). RelayTouches atomic.Int64 @@ -68,17 +68,14 @@ func (s *DBStats) SnapshotBackfills() map[string]int64 { return out } -// Store wraps the SQLite database for packet ingestion. +// Store wraps the selected native database for packet ingestion. type Store struct { - db *sql.DB - path string // filesystem path to the SQLite DB (used to resolve queue dirs) - Stats DBStats - // plannerStatsAt is when RefreshPlannerStats last succeeded in this - // process, in Unix nanoseconds (0: not yet). - plannerStatsAt atomic.Int64 - - // linkedCompanions is the linked-only ingest filter - // (linked_companions.go); nil means off. + backend dbconfig.Backend + db *sql.DB + path string // compatibility anchor for local file queues; never a database URL + stateDir string + Stats DBStats + plannerStatsAt atomic.Int64 linkedCompanions *linkedCompanionSet stmtGetTxByHash *sql.Stmt @@ -135,799 +132,150 @@ const relayTouchDebounce = 5 * time.Minute // an extra UPDATE, never a missed one. const relayTouchedMaxEntries = 50000 -// OpenStore opens or creates a SQLite DB at the given path, applying the -// v3 schema that is compatible with the Node.js server. -func OpenStore(dbPath string) (*Store, error) { - return OpenStoreWithInterval(dbPath, 300) +// OpenStore preserves native path/URL callers; runtime uses selected Storage. +func OpenStore(databaseURL string) (*Store, error) { return OpenStoreWithInterval(databaseURL, 300) } +func OpenStoreWithInterval(databaseURL string, sampleIntervalSec int) (*Store, error) { + return OpenStoreWithState(databaseURL, "", sampleIntervalSec) } - -// OpenStoreWithInterval opens or creates a SQLite DB with a configurable sample interval. -func OpenStoreWithInterval(dbPath string, sampleIntervalSec int) (*Store, error) { - dir := filepath.Dir(dbPath) - if err := os.MkdirAll(dir, 0o755); err != nil { - return nil, fmt.Errorf("creating data dir: %w", err) - } - - db, err := sql.Open("sqlite3", dbschema.WriterDSN(dbPath)) - if err != nil { - return nil, fmt.Errorf("opening db: %w", err) - } - - if err := db.Ping(); err != nil { - return nil, fmt.Errorf("pinging db: %w", err) - } - - db.SetMaxOpenConns(1) - db.SetMaxIdleConns(1) - log.Printf("SQLite config: busy_timeout=5000ms, max_open_conns=1, max_idle_conns=1, journal=WAL, synchronous=FULL") - - if err := applySchema(db); err != nil { - return nil, fmt.Errorf("applying schema: %w", err) - } - - // Apply the additional server-originated migrations (now owned by - // the ingestor per #1287). Adds the indexes/columns that used to live - // in cmd/server/ensure_*.go: server now ASSERTS these exist. - if err := dbschema.Apply(db, log.Printf); err != nil { - return nil, fmt.Errorf("dbschema.Apply: %w", err) - } - - s := &Store{db: db, path: dbPath, sampleIntervalSec: sampleIntervalSec} - if err := s.prepareStatements(); err != nil { - return nil, fmt.Errorf("preparing statements: %w", err) - } - - // Continue the scope-match tally from where the previous process left - // off. Not fatal: a tally that cannot be read costs an observability - // number, and refusing to ingest over it would be the worse trade. - if err := s.LoadScopeMatchTotals(); err != nil { - log.Printf("[regions] restoring scope-match tally: %v", err) - } - - // Schedule async migrations. These must NOT block boot. See - // async_migration.go for the convention. - // PREFLIGHT: async=true reason="composite index build on observations (1.9M+ rows in prod) — converted from sync after v3.8.3" - var idxDone int - if s.db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'obs_observer_ts_idx_v1'").Scan(&idxDone) != nil { - if err := s.RunAsyncMigration(context.Background(), "obs_observer_ts_idx_v1", - func(ctx context.Context, d *sql.DB) error { - log.Println("[migration/async] Building (observer_idx, timestamp) composite index on observations...") - if _, err := d.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_observations_observer_idx_timestamp ON observations(observer_idx, timestamp)`); err != nil { - return err - } - if _, err := d.ExecContext(ctx, `INSERT OR IGNORE INTO _migrations (name) VALUES ('obs_observer_ts_idx_v1')`); err != nil { - return err - } - log.Println("[migration/async] observations(observer_idx, timestamp) index created") - return nil - }); err != nil { - log.Printf("[migration/async] scheduling obs_observer_ts_idx_v1 failed: %v", err) - } - } - - // #1690: backfill transmissions.last_seen from MAX(observations.timestamp) - // per transmission. The column is added inline by dbschema.Apply (cheap - // metadata-only ALTER); the populate query is potentially expensive - // (full obs scan + group) so we run it async. Subsequent observation - // inserts maintain the column inline (see InsertTransmission below). - // PREFLIGHT: async=true reason="full-table backfill JOIN (1.9M+ obs × 86k+ tx in prod) — must not block ingestor boot" - if err := s.RunAsyncMigration(context.Background(), "tx_last_seen_backfill_v1", - func(ctx context.Context, d *sql.DB) error { - log.Println("[migration/async] Backfilling transmissions.last_seen from MAX(observations.timestamp)...") - res, err := d.ExecContext(ctx, ` - UPDATE transmissions - SET last_seen = COALESCE(( - SELECT MAX(timestamp) FROM observations WHERE transmission_id = transmissions.id - ), last_seen) - WHERE last_seen = 0 - `) - if err != nil { - return err - } - n, _ := res.RowsAffected() - log.Printf("[migration/async] transmissions.last_seen backfill complete: %d rows updated", n) - return nil - }); err != nil { - log.Printf("[migration/async] scheduling tx_last_seen_backfill_v1 failed: %v", err) +func OpenStoreWithState(target, stateDir string, sampleIntervalSec int) (*Store, error) { + storage := dbconfig.Storage{Backend: dbconfig.SQLite, DBPath: target, StateDir: stateDir} + if strings.HasPrefix(target, "postgres:") || strings.HasPrefix(target, "postgresql:") { + storage = dbconfig.Storage{Backend: dbconfig.Postgres, WriterDatabaseURL: target, StateDir: stateDir} } - - // A missing/failed completion lookup leaves preservation enabled. This - // lifecycle state is restored on restart, independently of main's startup. - var evidenceStatus string - _ = db.QueryRow(`SELECT status FROM _async_migrations WHERE name='advert_route_evidence_v1'`).Scan(&evidenceStatus) - s.advertEvidenceComplete.Store(evidenceStatus == "done") - // The observation_hops backfill is deliberately NOT scheduled here; see - // ScheduleObservationHopsBackfill (called from main once ingest is live). - - return s, nil + return openStore(storage, nil, sampleIntervalSec, true) } - -func applySchema(db *sql.DB) error { - // auto_vacuum=INCREMENTAL is set via DSN pragma (must be before journal_mode). - // Logging of current mode is handled by CheckAutoVacuum — no duplicate log here. - - schema := ` - CREATE TABLE IF NOT EXISTS nodes ( - public_key TEXT PRIMARY KEY, - name TEXT, - role TEXT, - lat REAL, - lon REAL, - last_seen TEXT, - first_seen TEXT, - advert_count INTEGER DEFAULT 0, - battery_mv INTEGER, - temperature_c REAL, - foreign_advert INTEGER DEFAULT 0 - ); - - CREATE TABLE IF NOT EXISTS scope_match_totals ( - id INTEGER PRIMARY KEY CHECK (id = 1), - since_unix INTEGER NOT NULL, - unique_matches INTEGER NOT NULL, - explicit_over_derived INTEGER NOT NULL, - ambiguous INTEGER NOT NULL, - none_matches INTEGER NOT NULL, - updated_unix INTEGER NOT NULL - ); - - CREATE TABLE IF NOT EXISTS observers ( - id TEXT PRIMARY KEY, - name TEXT, - iata TEXT, - last_seen TEXT, - first_seen TEXT, - packet_count INTEGER DEFAULT 0, - model TEXT, - firmware TEXT, - client_version TEXT, - radio TEXT, - battery_mv INTEGER, - uptime_secs INTEGER, - noise_floor REAL, - inactive INTEGER DEFAULT 0, - last_packet_at TEXT DEFAULT NULL, - clock_skew_seconds INTEGER DEFAULT NULL, - clock_skew_count_24h INTEGER DEFAULT 0, - clock_last_naive_at TEXT DEFAULT NULL, - can_relay INTEGER DEFAULT 1, - can_relay_seen INTEGER DEFAULT 0 - ); - - CREATE INDEX IF NOT EXISTS idx_nodes_last_seen ON nodes(last_seen); - CREATE INDEX IF NOT EXISTS idx_observers_last_seen ON observers(last_seen); - - CREATE TABLE IF NOT EXISTS inactive_nodes ( - public_key TEXT PRIMARY KEY, - name TEXT, - role TEXT, - lat REAL, - lon REAL, - last_seen TEXT, - first_seen TEXT, - advert_count INTEGER DEFAULT 0, - battery_mv INTEGER, - temperature_c REAL, - foreign_advert INTEGER DEFAULT 0 - ); - - CREATE INDEX IF NOT EXISTS idx_inactive_nodes_last_seen ON inactive_nodes(last_seen); - - CREATE TABLE IF NOT EXISTS transmissions ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - raw_hex TEXT NOT NULL, - hash TEXT NOT NULL UNIQUE, - first_seen TEXT NOT NULL, - route_type INTEGER, - payload_type INTEGER, - payload_version INTEGER, - decoded_json TEXT, - from_pubkey TEXT, - last_seen INTEGER NOT NULL DEFAULT 0, - created_at TEXT DEFAULT (datetime('now')), - code1 TEXT, - code2 TEXT - ); - - CREATE INDEX IF NOT EXISTS idx_transmissions_hash ON transmissions(hash); - CREATE INDEX IF NOT EXISTS idx_transmissions_first_seen ON transmissions(first_seen); - CREATE INDEX IF NOT EXISTS idx_transmissions_payload_type ON transmissions(payload_type); - -- idx_tx_code1 is created by the transport_codes_v1 migration below, - -- after the ALTER runs — same reasoning as idx_transmissions_from_pubkey - -- and idx_tx_last_seen_zero above: a legacy DB (table-exists, - -- column-missing) would trip on this CREATE INDEX referencing code1 - -- before the column exists, since CREATE TABLE IF NOT EXISTS is a - -- no-op against a pre-existing table. - -- idx_transmissions_from_pubkey is created by the from_pubkey_v1 - -- migration after the column is added on legacy DBs (#1143). - -- idx_tx_last_seen_zero (partial, WHERE last_seen=0) is created by - -- dbschema.Apply after ensuring the last_seen column exists (#1690, - -- partial-index swap #1740) — keep it OUT of this base schema block - -- so legacy DBs (table-exists, column-missing) don't trip on the - -- CREATE INDEX before the ALTER runs. - - -- Mobile client RX coverage: a roaming companion = a mobile observer - -- with a moving GPS position, so it gets its own table rather than - -- observations (which assumes a fixed observer/location). - CREATE TABLE IF NOT EXISTS client_receptions ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - rx_pubkey TEXT NOT NULL, - heard_key TEXT NOT NULL, - heard_keylen INTEGER NOT NULL, - rssi INTEGER, - snr REAL, - lat REAL NOT NULL, - lon REAL NOT NULL, - pos_acc_m REAL, - rx_at TEXT NOT NULL, - ingested_at TEXT NOT NULL, - src TEXT NOT NULL, - UNIQUE(rx_pubkey, heard_key, rx_at) - ); - -- Coverage queries filter by bbox AND match the heard node either by full - -- key (heard_keylen=32 AND heard_key=?) or by 2-3 byte prefix. The composite - -- (heard_key, heard_keylen, lat, lon) serves the heard_key-equality seek and - -- carries lat/lon so the bbox range is satisfied from the index; it also - -- supersedes the old single-column heard_key index. idx_client_recept_latlon - -- lets the planner instead drive from a selective bbox. (#5, #18) - CREATE INDEX IF NOT EXISTS idx_client_recept_heard_geo ON client_receptions(heard_key, heard_keylen, lat, lon); - CREATE INDEX IF NOT EXISTS idx_client_recept_latlon ON client_receptions(lat, lon); - -- rx_at backs both the retention reaper (DELETE WHERE rx_at < ?) and the - -- leaderboard, which range-scans WHERE rx_at >= ? and aggregates per - -- rx_pubkey in Go (see rxLeaderboard's frontier-weighted scoring). Without - -- this index either would full-scan the table under the writer lock - -- (verified by an EXPLAIN test). A dedicated rx_pubkey index stays - -- redundant — the leaderboard no longer groups by rx_pubkey in SQL. - CREATE INDEX IF NOT EXISTS idx_client_recept_rxat ON client_receptions(rx_at); - DROP INDEX IF EXISTS idx_client_recept_rxpk; - - -- Self-reported name of each mobile client (companion), from the SELF_INFO - -- name the app sends as "origin". Lets the leaderboard show a name even - -- when the companion never advertised (so it isn't in the nodes table). - CREATE TABLE IF NOT EXISTS client_observers ( - pubkey TEXT PRIMARY KEY, - name TEXT, - last_seen TEXT - ); - - -- Diagnostic RF observations from mobile clients. Unlike client_receptions - -- this holds EVERY decodable packet, attributable or not, so it must never - -- be used for coverage. pkt_hash is ComputeContentHash() — identical to - -- transmissions.hash — so dark-traffic queries are a plain equality join. - CREATE TABLE IF NOT EXISTS client_rx_observations ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - rx_pubkey TEXT NOT NULL, - rx_at TEXT NOT NULL, - ingested_at TEXT NOT NULL, - pkt_hash TEXT NOT NULL, - route_type INTEGER NOT NULL, - payload_type INTEGER NOT NULL, - code1 TEXT, - code2 TEXT, - scope_name TEXT, - hash_size INTEGER NOT NULL, - hop_count INTEGER NOT NULL, - path_json TEXT, - forwarder TEXT, - snr REAL, - rssi INTEGER, - lat REAL NOT NULL, - lon REAL NOT NULL, - pos_acc_m REAL, - UNIQUE(rx_pubkey, pkt_hash, rx_at) - ); - CREATE INDEX IF NOT EXISTS idx_cro_prune ON client_rx_observations(rx_at); - CREATE INDEX IF NOT EXISTS idx_cro_hash ON client_rx_observations(pkt_hash, rx_at); - CREATE INDEX IF NOT EXISTS idx_cro_forwarder ON client_rx_observations(forwarder, rx_at); - CREATE INDEX IF NOT EXISTS idx_cro_scope ON client_rx_observations(scope_name, rx_at); - - -- RF environment samples from mobile clients: radio counters paired with - -- a GPS point. Absolutes only — deltas are computed at query time, and a - -- decrease in uptime_secs (reboot) or any counter (wrap) breaks the chain. - CREATE TABLE IF NOT EXISTS client_rf_samples ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - rx_pubkey TEXT NOT NULL, - sampled_at TEXT NOT NULL, - ingested_at TEXT NOT NULL, - lat REAL NOT NULL, - lon REAL NOT NULL, - pos_acc_m REAL, - stationary INTEGER NOT NULL DEFAULT 0, - uptime_secs INTEGER NOT NULL, - battery_mv INTEGER, - queue_len INTEGER, - errors INTEGER, - noise_floor INTEGER, - last_rssi INTEGER, - last_snr REAL, - tx_air_secs INTEGER, - rx_air_secs INTEGER, - recv INTEGER, - sent INTEGER, - flood_rx INTEGER, - direct_rx INTEGER, - flood_tx INTEGER, - direct_tx INTEGER, - recv_errors INTEGER, - UNIQUE(rx_pubkey, sampled_at) - ); - CREATE INDEX IF NOT EXISTS idx_crf_prune ON client_rf_samples(sampled_at); - CREATE INDEX IF NOT EXISTS idx_crf_track ON client_rf_samples(rx_pubkey, sampled_at); - - -- Declared region lists reported by repeaters via ANON_REQ_TYPE_REGIONS. - -- Observations, never state: "current" is the greatest observed_at for a - -- target, NOT the greatest ingested_at — a drive buffered offline can - -- arrive days late and must not overwrite a fresher reading. - CREATE TABLE IF NOT EXISTS node_declared_regions ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - target TEXT NOT NULL, - rx_pubkey TEXT NOT NULL, - observed_at TEXT NOT NULL, - ingested_at TEXT NOT NULL, - regions_csv TEXT NOT NULL, - truncated INTEGER NOT NULL DEFAULT 0, - lat REAL, - lon REAL, - pos_acc_m REAL, - repeater_clock INTEGER, - UNIQUE(target, rx_pubkey, observed_at) - ); - CREATE INDEX IF NOT EXISTS idx_ndr_target ON node_declared_regions(target, observed_at); - CREATE INDEX IF NOT EXISTS idx_ndr_prune ON node_declared_regions(observed_at); - ` - if _, err := db.Exec(schema); err != nil { - return fmt.Errorf("base schema: %w", err) - } - - // Create observations table (v3 schema) - obsExists := false - row := db.QueryRow("SELECT name FROM sqlite_master WHERE type='table' AND name='observations'") - var dummy string - if row.Scan(&dummy) == nil { - obsExists = true - } - - if !obsExists { - obs := ` - CREATE TABLE observations ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - transmission_id INTEGER NOT NULL REFERENCES transmissions(id), - observer_idx INTEGER, - direction TEXT, - snr REAL, - rssi REAL, - score INTEGER, - path_json TEXT, - timestamp INTEGER NOT NULL - ); - CREATE INDEX idx_observations_transmission_id ON observations(transmission_id); - CREATE INDEX idx_observations_observer_idx ON observations(observer_idx); - CREATE INDEX idx_observations_timestamp ON observations(timestamp); - CREATE UNIQUE INDEX IF NOT EXISTS idx_observations_dedup ON observations(transmission_id, observer_idx, COALESCE(path_json, '')); - ` - if _, err := db.Exec(obs); err != nil { - return fmt.Errorf("observations schema: %w", err) - } - } - - // Create/rebuild packets_v view (v3 schema: observer_idx → observers.rowid) - // The Go server reads this view; without it fresh installs get "no such table: packets_v". - db.Exec(`DROP VIEW IF EXISTS packets_v`) - _, vErr := db.Exec(` - CREATE VIEW packets_v AS - SELECT o.id, COALESCE(o.raw_hex, t.raw_hex) AS raw_hex, - datetime(o.timestamp, 'unixepoch') AS timestamp, - obs.id AS observer_id, obs.name AS observer_name, - o.direction, o.snr, o.rssi, o.score, t.hash, t.route_type, - t.payload_type, t.payload_version, o.path_json, t.decoded_json, - t.created_at - FROM observations o - JOIN transmissions t ON t.id = o.transmission_id - LEFT JOIN observers obs ON obs.rowid = o.observer_idx AND (obs.inactive IS NULL OR obs.inactive = 0) - `) - if vErr != nil { - return fmt.Errorf("packets_v view: %w", vErr) - } - - // One-time migration: recalculate advert_count to count unique transmissions only - db.Exec(`CREATE TABLE IF NOT EXISTS _migrations (name TEXT PRIMARY KEY)`) - var migDone int - row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'advert_count_unique_v1'") - if row.Scan(&migDone) != nil { - log.Println("[migration] Recalculating advert_count (unique transmissions only)...") - // Note: this migration is gated on a one-shot _migrations row, so it - // runs at most once per DB. The historical version used a LIKE-on-JSON - // substring match (#1143). Switching to from_pubkey here is safe even - // though the column may not yet be backfilled on legacy DBs: the - // migration is already marked done on those DBs and won't re-run. - db.Exec(` - UPDATE nodes SET advert_count = ( - SELECT COUNT(*) FROM transmissions t - WHERE t.payload_type = 4 - AND t.from_pubkey = nodes.public_key - ) - `) - db.Exec(`INSERT INTO _migrations (name) VALUES ('advert_count_unique_v1')`) - log.Println("[migration] advert_count recalculated") - } - - // One-time migration: change noise_floor from INTEGER to REAL affinity. - // SQLite doesn't support ALTER COLUMN, but existing float values are stored - // as REAL regardless of column affinity. New table definition already uses REAL. - // This migration casts any integer-stored noise_floor values to real. - row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'noise_floor_real_v1'") - if row.Scan(&migDone) != nil { - log.Println("[migration] Ensuring noise_floor values are stored as REAL...") - db.Exec(`UPDATE observers SET noise_floor = CAST(noise_floor AS REAL) WHERE noise_floor IS NOT NULL AND typeof(noise_floor) = 'integer'`) - db.Exec(`INSERT INTO _migrations (name) VALUES ('noise_floor_real_v1')`) - log.Println("[migration] noise_floor migration complete") - } - - // One-time migration: add telemetry columns to nodes and inactive_nodes tables. - row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'node_telemetry_v1'") - if row.Scan(&migDone) != nil { - log.Println("[migration] Adding telemetry columns to nodes/inactive_nodes...") - - // checkAndAddColumn checks whether `column` already exists in `table` - // using PRAGMA table_info, and adds it if missing. All call sites pass - // hardcoded table/column/type literals so there is no SQL injection risk. - checkAndAddColumn := func(table, column, colType string) error { - rows, err := db.Query(fmt.Sprintf("PRAGMA table_info(%s)", table)) - if err != nil { - return fmt.Errorf("querying table info for %s: %w", table, err) - } - defer rows.Close() - - exists := false - for rows.Next() { - var cid int - var name, ctype string - var notnull, pk int - var dfltValue sql.NullString - if err := rows.Scan(&cid, &name, &ctype, ¬null, &dfltValue, &pk); err != nil { - return fmt.Errorf("scanning table info for %s: %w", table, err) - } - if name == column { - exists = true - break - } - } - if err := rows.Err(); err != nil { - return fmt.Errorf("iterating table info for %s: %w", table, err) - } - if exists { - return nil - } - if _, err := db.Exec(fmt.Sprintf("ALTER TABLE %s ADD COLUMN %s %s", table, column, colType)); err != nil { - return fmt.Errorf("adding column %s to %s: %w", column, table, err) +func openStoreStorage(storage dbconfig.Storage, cfg *DBConfig, sampleIntervalSec int) (*Store, error) { + return openStore(storage, cfg, sampleIntervalSec, false) +} +func openStore(storage dbconfig.Storage, cfg *DBConfig, sampleIntervalSec int, create bool) (*Store, error) { + var db *sql.DB + var err error + stateDir := storage.StateDir + switch storage.Backend { + case dbconfig.SQLite: + if storage.DBPath == "" || strings.Contains(storage.DBPath, "://") || strings.HasPrefix(storage.DBPath, "postgres:") || strings.HasPrefix(storage.DBPath, "postgresql:") { + return nil, fmt.Errorf("SQLite telemetry requires a filesystem path") + } + if create { + if err = os.MkdirAll(filepath.Dir(storage.DBPath), 0700); err != nil { + return nil, err } - return nil - } - - if err := checkAndAddColumn("nodes", "battery_mv", "INTEGER"); err != nil { - return err - } - if err := checkAndAddColumn("nodes", "temperature_c", "REAL"); err != nil { - return err - } - if err := checkAndAddColumn("inactive_nodes", "battery_mv", "INTEGER"); err != nil { - return err - } - if err := checkAndAddColumn("inactive_nodes", "temperature_c", "REAL"); err != nil { - return err } - if _, err := db.Exec(`INSERT INTO _migrations (name) VALUES ('node_telemetry_v1')`); err != nil { - return fmt.Errorf("recording node_telemetry_v1 migration: %w", err) + var dsn string + dsn, err = sqliteTelemetryWriterDSN(storage.DBPath, create) + if err == nil { + db, err = sql.Open("sqlite3", dsn) } - log.Println("[migration] node telemetry columns added") - } - - // One-time migration: add timestamp index on observations for fast stats queries. - // Older databases created before this index was added suffer from full table scans - // on COUNT(*) WHERE timestamp > ?, causing /api/stats to take 30s+. - row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'obs_timestamp_index_v1'") - if row.Scan(&migDone) != nil { - log.Println("[migration] Adding timestamp index on observations...") - db.Exec(`CREATE INDEX IF NOT EXISTS idx_observations_timestamp ON observations(timestamp)`) - db.Exec(`INSERT INTO _migrations (name) VALUES ('obs_timestamp_index_v1')`) - log.Println("[migration] observations timestamp index created") - } - - // #1481 P0-3: covering index for GetObserverPacketCounts. The query - // joins observations → observers and GROUP BYs observer_idx with a - // timestamp WHERE filter; a composite (observer_idx, timestamp) - // index lets SQLite resolve the grouping + range filter from the - // index alone instead of a 1.9M-row scan. - // - // CONVERTED TO ASYNC (preflight-async-migration-gate). Scheduling - // happens in OpenStore() once the real *Store exists so the - // backfill WaitGroup is shared with the rest of the ingestor. - // The legacy `_migrations` gate is preserved by the async fn so - // DBs that already completed the sync build stay no-op. - - // #1483: normalize nodes.public_key to lowercase. The server's - // GetNodeLocationsByKeys lookup dropped LOWER(public_key) for perf - // (#1481 P0-3) and now relies on stored keys being lowercase. The - // decoder writes lowercase today, but legacy/admin/API inserts may - // have left mixed-case rows. Idempotent: counts and lowers any - // non-lowercase rows on every boot, runs once via _migrations gate - // for the bulk fix. Re-running stays cheap because subsequent - // passes match zero rows. - if r := db.QueryRow("SELECT COUNT(*) FROM nodes WHERE public_key != lower(public_key)"); r != nil { - var n int64 - _ = r.Scan(&n) - if n > 0 { - log.Printf("[migration] Normalizing %d nodes.public_key row(s) to lowercase (#1483)...", n) - if _, err := db.Exec(`UPDATE nodes SET public_key = lower(public_key) WHERE public_key != lower(public_key)`); err != nil { - log.Printf("[migration] public_key lowercase normalize failed: %v", err) - } else { - log.Printf("[migration] public_key lowercase normalize complete (%d rows)", n) - } + if stateDir == "" { + stateDir = filepath.Dir(storage.DBPath) } - } - - // observer_metrics table for RF health dashboard - row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'observer_metrics_v1'") - if row.Scan(&migDone) != nil { - log.Println("[migration] Creating observer_metrics table...") - _, err := db.Exec(` - CREATE TABLE IF NOT EXISTS observer_metrics ( - observer_id TEXT NOT NULL, - timestamp TEXT NOT NULL, - noise_floor REAL, - tx_air_secs INTEGER, - rx_air_secs INTEGER, - recv_errors INTEGER, - battery_mv INTEGER, - PRIMARY KEY (observer_id, timestamp) - ) - `) - if err != nil { - return fmt.Errorf("observer_metrics schema: %w", err) + case dbconfig.Postgres: + db, err = pgutil.Open(storage.WriterDatabaseURL, false) + if stateDir == "" { + stateDir = "data" } - db.Exec(`INSERT INTO _migrations (name) VALUES ('observer_metrics_v1')`) - log.Println("[migration] observer_metrics table created") + default: + return nil, fmt.Errorf("telemetry requires an explicitly selected storage backend") } - - // Migration: add timestamp index for cross-observer time-range queries - row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'observer_metrics_ts_idx'") - if row.Scan(&migDone) != nil { - log.Println("[migration] Creating observer_metrics timestamp index...") - _, err := db.Exec(`CREATE INDEX IF NOT EXISTS idx_observer_metrics_timestamp ON observer_metrics(timestamp)`) - if err != nil { - return fmt.Errorf("observer_metrics timestamp index: %w", err) - } - db.Exec(`INSERT INTO _migrations (name) VALUES ('observer_metrics_ts_idx')`) - log.Println("[migration] observer_metrics timestamp index created") + if err != nil { + return nil, err } - - // Migration: add inactive column to observers for soft-delete retention - row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'observers_inactive_v1'") - if row.Scan(&migDone) != nil { - log.Println("[migration] Adding inactive column to observers...") - _, err := db.Exec(`ALTER TABLE observers ADD COLUMN inactive INTEGER DEFAULT 0`) - if err != nil { - // Column may already exist (e.g. fresh install with schema above) - log.Printf("[migration] observers.inactive: %v (may already exist)", err) - } - db.Exec(`INSERT INTO _migrations (name) VALUES ('observers_inactive_v1')`) - log.Println("[migration] observers.inactive column added") - } - - // Migration: add packets_sent and packets_recv columns to observer_metrics - row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'observer_metrics_packets_v1'") - if row.Scan(&migDone) != nil { - log.Println("[migration] Adding packets_sent/packets_recv columns to observer_metrics...") - db.Exec(`ALTER TABLE observer_metrics ADD COLUMN packets_sent INTEGER`) - db.Exec(`ALTER TABLE observer_metrics ADD COLUMN packets_recv INTEGER`) - db.Exec(`INSERT INTO _migrations (name) VALUES ('observer_metrics_packets_v1')`) - log.Println("[migration] packets_sent/packets_recv columns added") - } - - // Migration: add channel_hash column for fast channel queries (#762) - row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'channel_hash_v1'") - if row.Scan(&migDone) != nil { - log.Println("[migration] Adding channel_hash column to transmissions...") - db.Exec(`ALTER TABLE transmissions ADD COLUMN channel_hash TEXT DEFAULT NULL`) - db.Exec(`CREATE INDEX IF NOT EXISTS idx_tx_channel_hash ON transmissions(channel_hash) WHERE payload_type = 5`) - // Backfill: extract channel name for decrypted (CHAN) packets - res, err := db.Exec(`UPDATE transmissions SET channel_hash = json_extract(decoded_json, '$.channel') WHERE payload_type = 5 AND channel_hash IS NULL AND json_extract(decoded_json, '$.type') = 'CHAN'`) - if err == nil { - n, _ := res.RowsAffected() - log.Printf("[migration] Backfilled channel_hash for %d CHAN packets", n) - } - // Backfill: extract channelHashHex for encrypted (GRP_TXT) packets, prefixed with 'enc_' - res, err = db.Exec(`UPDATE transmissions SET channel_hash = 'enc_' || json_extract(decoded_json, '$.channelHashHex') WHERE payload_type = 5 AND channel_hash IS NULL AND json_extract(decoded_json, '$.type') = 'GRP_TXT'`) - if err == nil { - n, _ := res.RowsAffected() - log.Printf("[migration] Backfilled channel_hash for %d GRP_TXT packets", n) - } - db.Exec(`INSERT INTO _migrations (name) VALUES ('channel_hash_v1')`) - log.Println("[migration] channel_hash column added and backfilled") - } - - // Migration: dropped_packets table for signature validation failures (#793) - row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'dropped_packets_v1'") - if row.Scan(&migDone) != nil { - log.Println("[migration] Creating dropped_packets table...") - _, err := db.Exec(` - CREATE TABLE IF NOT EXISTS dropped_packets ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - hash TEXT, - raw_hex TEXT, - reason TEXT NOT NULL, - observer_id TEXT, - observer_name TEXT, - node_pubkey TEXT, - node_name TEXT, - dropped_at DATETIME DEFAULT CURRENT_TIMESTAMP - ); - CREATE INDEX IF NOT EXISTS idx_dropped_observer ON dropped_packets(observer_id); - CREATE INDEX IF NOT EXISTS idx_dropped_node ON dropped_packets(node_pubkey); - `) - if err != nil { - return fmt.Errorf("dropped_packets schema: %w", err) + ok := false + defer func() { + if !ok { + db.Close() } - db.Exec(`INSERT INTO _migrations (name) VALUES ('dropped_packets_v1')`) - log.Println("[migration] dropped_packets table created") - } - - // Migration: observations.raw_hex (#881) is now owned by - // internal/dbschema/dbschema.go (#1321). The server PRAGMA-detects - // this column as hasObsRawHex; keeping a single canonical Apply - // path closes the startup race where the server's detector ran - // before this ALTER finished. - - // Migration: transmissions.scope_name (#899) is now owned by - // internal/dbschema/dbschema.go (#1321). See above. - - // Migration: add last_packet_at column to observers (#last-packet-at) - row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'observers_last_packet_at_v1'") - if row.Scan(&migDone) != nil { - log.Println("[migration] Adding last_packet_at column to observers...") - _, alterErr := db.Exec(`ALTER TABLE observers ADD COLUMN last_packet_at TEXT DEFAULT NULL`) - if alterErr != nil && !strings.Contains(alterErr.Error(), "duplicate column") { - return fmt.Errorf("observers last_packet_at ALTER: %w", alterErr) + }() + db.SetMaxOpenConns(1) + db.SetMaxIdleConns(1) + if storage.Backend == dbconfig.SQLite { + if !create { + if err = validateExistingTelemetry(db); err != nil { + return nil, err + } } - // Backfill: set last_packet_at = last_seen only for observers that actually have - // observation rows (packet_count alone is unreliable — UpsertObserver sets it to 1 - // on INSERT even for status-only observers). - res, err := db.Exec(`UPDATE observers SET last_packet_at = last_seen - WHERE last_packet_at IS NULL - AND rowid IN (SELECT DISTINCT observer_idx FROM observations WHERE observer_idx IS NOT NULL)`) - if err == nil { - n, _ := res.RowsAffected() - log.Printf("[migration] Backfilled last_packet_at for %d observers with packets", n) + if err = dbschema.ApplySQLite(db, log.Printf); err != nil { + return nil, err } - db.Exec(`INSERT INTO _migrations (name) VALUES ('observers_last_packet_at_v1')`) - log.Println("[migration] observers.last_packet_at column added") - } - - // Migration: per-observer naive-clock skew tracking (#1478). - // When the ingestor clamps a packet's envelope timestamp because the - // observer emitted a zone-less local-time string off from UTC by >15min - // (resolveRxTime in main.go), we record the event here so the UI can - // surface a ⚠️ chip + banner. Decays after 24h via server-side read sweep. - row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'observers_clock_naive_v1'") - if row.Scan(&migDone) != nil { - log.Println("[migration] Adding clock-naive columns to observers (#1478)...") - // Each ALTER is independent — ignore "duplicate column" so reruns are safe. - for _, stmt := range []string{ - `ALTER TABLE observers ADD COLUMN clock_skew_seconds INTEGER DEFAULT NULL`, - `ALTER TABLE observers ADD COLUMN clock_skew_count_24h INTEGER DEFAULT 0`, - `ALTER TABLE observers ADD COLUMN clock_last_naive_at TEXT DEFAULT NULL`, - } { - if _, err := db.Exec(stmt); err != nil && !strings.Contains(err.Error(), "duplicate column") { - return fmt.Errorf("clock_naive migration: %w", err) + if !create { + if _, err = db.Exec(`PRAGMA auto_vacuum=INCREMENTAL; PRAGMA journal_mode=WAL`); err != nil { + return nil, err } } - db.Exec(`INSERT INTO _migrations (name) VALUES ('observers_clock_naive_v1')`) - log.Println("[migration] observers.clock_naive columns added") - } - // Migration: backfill observations.path_json from raw_hex (#888) - // NOTE: This runs ASYNC via BackfillPathJSONAsync() to avoid blocking MQTT startup. - // See staging outage where ~502K rows blocked ingest for 15+ hours. - - // One-time cleanup: delete legacy packets with empty hash or empty first_seen (#994) - row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'cleanup_legacy_null_hash_ts'") - if row.Scan(&migDone) != nil { - log.Println("[migration] Cleaning up legacy packets with empty hash/timestamp...") - db.Exec(`DELETE FROM observations WHERE transmission_id IN (SELECT id FROM transmissions WHERE hash = '' OR first_seen = '')`) - res, err := db.Exec(`DELETE FROM transmissions WHERE hash = '' OR first_seen = ''`) - if err == nil { - deleted, _ := res.RowsAffected() - log.Printf("[migration] deleted %d legacy packets with empty hash/timestamp", deleted) - } - db.Exec(`INSERT INTO _migrations (name) VALUES ('cleanup_legacy_null_hash_ts')`) - } - - // Migration: foreign_advert column on nodes/inactive_nodes (#730) - // Marks nodes whose ADVERT GPS lies outside the configured geofilter polygon. - // Default 0; set to 1 by the ingestor when GeoFilter is configured and - // PassesFilter() returns false. Allows operators to surface bridged/leaked - // adverts without silently dropping them. - row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'foreign_advert_v1'") - if row.Scan(&migDone) != nil { - log.Println("[migration] Adding foreign_advert column to nodes/inactive_nodes...") - if _, err := db.Exec(`ALTER TABLE nodes ADD COLUMN foreign_advert INTEGER DEFAULT 0`); err != nil { - log.Printf("[migration] nodes.foreign_advert: %v (may already exist)", err) + } else { + var synchronous string + if err = db.QueryRow(`SHOW synchronous_commit`).Scan(&synchronous); err != nil { + return nil, err } - if _, err := db.Exec(`ALTER TABLE inactive_nodes ADD COLUMN foreign_advert INTEGER DEFAULT 0`); err != nil { - log.Printf("[migration] inactive_nodes.foreign_advert: %v (may already exist)", err) + if synchronous != "on" { + return nil, fmt.Errorf("telemetry writer requires synchronous_commit=on") } - db.Exec(`CREATE INDEX IF NOT EXISTS idx_nodes_foreign_advert ON nodes(foreign_advert) WHERE foreign_advert = 1`) - db.Exec(`INSERT INTO _migrations (name) VALUES ('foreign_advert_v1')`) - log.Println("[migration] foreign_advert column added") - } - - // Migration: from_pubkey column on transmissions (#1143). - // Replaces the unsound `decoded_json LIKE '%pubkey%'` attribution path with - // an exact-match indexed column. Synchronously adds the column + index; - // row-level backfill is run by the SERVER asynchronously - // (cmd/server/from_pubkey_migration.go) so we don't block ingestor boot. - row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'from_pubkey_v1'") - if row.Scan(&migDone) != nil { - log.Println("[migration] Adding from_pubkey column + index to transmissions (#1143)...") - if _, err := db.Exec(`ALTER TABLE transmissions ADD COLUMN from_pubkey TEXT`); err != nil { - log.Printf("[migration] transmissions.from_pubkey: %v (may already exist)", err) + + if err = dbschema.AssertReady(db); err != nil { + return nil, err } - if _, err := db.Exec(`CREATE INDEX IF NOT EXISTS idx_transmissions_from_pubkey ON transmissions(from_pubkey)`); err != nil { - log.Printf("[migration] idx_transmissions_from_pubkey: %v", err) + if err = dbschema.AssertWriter(db); err != nil { + return nil, err } - db.Exec(`INSERT INTO _migrations (name) VALUES ('from_pubkey_v1')`) - log.Println("[migration] from_pubkey column + index added") - } - - // Migration: nodes.default_scope (#899 Feature 3) is now owned by - // internal/dbschema/dbschema.go (#1321). The server PRAGMA-detects - // this column as hasDefaultScope; keeping a single canonical Apply - // path closes the startup race that #1321 documented. - - // Migration: normalize known channel_hash values for existing rows. - // Before this PR, config key "public" was stored as channel_hash="public". - // After this PR, new rows use channel_hash="Public". Without backfill, - // channel grouping queries split into two buckets across the upgrade boundary. - row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'channel_hash_casing_v1'") - if row.Scan(&migDone) != nil { - log.Println("[migration] Normalizing known channel_hash values...") - res, err := db.Exec(`UPDATE transmissions SET channel_hash = 'Public' WHERE channel_hash = 'public' AND payload_type = 5`) - if err != nil { - log.Printf("[migration] ERROR: failed to normalize channel_hash: %v", err) - return fmt.Errorf("migration channel_hash_casing_v1 UPDATE failed: %w", err) + } + if stateDir == "" { + stateDir = "data" + } + if err = os.MkdirAll(stateDir, 0700); err != nil { + return nil, fmt.Errorf("create state directory: %w", err) + } + anchor := filepath.Join(stateDir, "meshcore") + if storage.Backend == dbconfig.SQLite { + anchor = filepath.Join(stateDir, filepath.Base(storage.DBPath)) + } + s := &Store{db: db, path: anchor, stateDir: stateDir, backend: storage.Backend, sampleIntervalSec: sampleIntervalSec} + if err = s.prepareStatements(); err != nil { + return nil, fmt.Errorf("prepare writer statements: %w", err) + } + if err = s.LoadScopeMatchTotals(); err != nil { + log.Printf("[regions] restoring scope-match tally: %v", err) + } + + if s.Backend() == dbconfig.SQLite { + var present int + if err := db.QueryRow(`SELECT count(*) FROM sqlite_master WHERE type='index' AND name='idx_observations_observer_idx_timestamp'`).Scan(&present); err != nil { + return nil, err } - n, _ := res.RowsAffected() - log.Printf("[migration] Normalized %d channel_hash rows from 'public' to 'Public'", n) - if _, err := db.Exec(`INSERT OR IGNORE INTO _migrations (name) VALUES ('channel_hash_casing_v1')`); err != nil { - log.Printf("[migration] WARNING: failed to record migration: %v", err) + // Completion metadata must not hide a missing physical index after recovery. + if present == 0 { + if _, err := db.Exec(`UPDATE _async_migrations SET status='pending_async' WHERE name='obs_observer_ts_idx_v1' AND status='done'`); err != nil { + return nil, err + } } - log.Println("[migration] channel_hash casing normalization complete") - } - - // Migration: transmissions.code1/code2 — the transport codes were decoded - // and discarded; storing them makes scope forwarding queryable per repeater. - row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'transport_codes_v1'") - if row.Scan(new(int)) != nil { - log.Println("[migration] Adding code1/code2 columns to transmissions...") - // Each ALTER is independent — ignore "duplicate column" so reruns (and - // the fresh-database path, where the base schema already created the - // columns) are safe. Any other failure must stop before the guard row - // is inserted, or prepareStatements' code1/code2 INSERT fails forever - // on every subsequent restart with no way to retry the migration. - for _, stmt := range []string{ - `ALTER TABLE transmissions ADD COLUMN code1 TEXT DEFAULT NULL`, - `ALTER TABLE transmissions ADD COLUMN code2 TEXT DEFAULT NULL`, - } { - if _, err := db.Exec(stmt); err != nil && !strings.Contains(err.Error(), "duplicate column") { - return fmt.Errorf("transport_codes_v1 migration: %w", err) + // PREFLIGHT: async=true reason="upstream composite observation index; keep its potentially large build off the startup goroutine" + if err := s.RunAsyncMigration(context.Background(), "obs_observer_ts_idx_v1", func(ctx context.Context, d *sql.DB) error { + if err := dbschema.EnsureSQLiteObserverTimeIndex(ctx, d); err != nil { + return err } + _, err := d.ExecContext(ctx, `INSERT INTO _migrations(name) VALUES('obs_observer_ts_idx_v1') ON CONFLICT DO NOTHING`) + return err + }); err != nil { + return nil, err } - db.Exec(`CREATE INDEX IF NOT EXISTS idx_tx_code1 ON transmissions(code1) WHERE code1 IS NOT NULL`) - db.Exec(`INSERT INTO _migrations (name) VALUES ('transport_codes_v1')`) - log.Println("[migration] code1/code2 columns added") + } + var evidenceStatus string + _ = db.QueryRow(`SELECT status FROM _async_migrations WHERE name='advert_route_evidence_v1'`).Scan(&evidenceStatus) + s.advertEvidenceComplete.Store(evidenceStatus == "done") + // The data backfill is resumable bookkeeping, never runtime DDL. It does + // not block readiness, and only rows not maintained by live ingest match. + if err = s.RunAsyncMigration(context.Background(), "tx_last_seen_backfill_v1", func(ctx context.Context, d *sql.DB) error { + _, e := d.ExecContext(ctx, `UPDATE transmissions SET last_seen=COALESCE((SELECT MAX(timestamp) FROM observations WHERE transmission_id=transmissions.id),last_seen) WHERE last_seen=0`) + return e + }); err != nil { + return nil, err } - return nil + // The observation_hops backfill is deliberately NOT scheduled here; see + // ScheduleObservationHopsBackfill (called from main once ingest is live). + + ok = true + return s, nil } func (s *Store) prepareStatements() error { @@ -941,20 +289,20 @@ func (s *Store) prepareStatements() error { return err } - s.stmtGetTxByHash, err = s.db.Prepare("SELECT id, first_seen FROM transmissions WHERE hash = ?") + s.stmtGetTxByHash, err = s.db.Prepare("SELECT id, first_seen FROM transmissions WHERE hash = " + s.parameter(1)) if err != nil { return err } s.stmtInsertTransmission, err = s.db.Prepare(` INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json, channel_hash, scope_name, from_pubkey, last_seen, code1, code2) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + VALUES (` + s.parameter(1) + `, ` + s.parameter(2) + `, ` + s.parameter(3) + `, ` + s.parameter(4) + `, ` + s.parameter(5) + `, ` + s.parameter(6) + `, ` + s.parameter(7) + `, ` + s.parameter(8) + `, ` + s.parameter(9) + `, ` + s.parameter(10) + `, ` + s.parameter(11) + `, ` + s.parameter(12) + `, ` + s.parameter(13) + `) RETURNING id `) if err != nil { return err } - s.stmtUpdateTxFirstSeen, err = s.db.Prepare("UPDATE transmissions SET first_seen = ? WHERE id = ?") + s.stmtUpdateTxFirstSeen, err = s.db.Prepare("UPDATE transmissions SET first_seen = " + s.parameter(1) + " WHERE id = " + s.parameter(2)) if err != nil { return err } @@ -969,74 +317,100 @@ func (s *Store) prepareStatements() error { // "tx_last_seen_backfill_v1". // PREFLIGHT: async=true reason="prepared-statement row-level UPDATE BY PRIMARY KEY (transmissions.id) — single-row touch per observation, indexed by PK, constant-time at any scale. Not a migration." s.stmtTouchNodeLastSeen, err = s.db.Prepare( - "UPDATE nodes SET last_seen = ? WHERE public_key = ? AND (last_seen IS NULL OR last_seen < ?)") + "UPDATE nodes SET last_seen = " + s.parameter(1) + " WHERE public_key = " + s.parameter(2) + " AND (last_seen IS NULL OR last_seen < " + s.parameter(3) + ")") if err != nil { return fmt.Errorf("preparing touch node last_seen: %w", err) } - s.stmtBumpTxLastSeen, err = s.db.Prepare("UPDATE transmissions SET last_seen = ? WHERE id = ? AND last_seen < ?") + s.stmtBumpTxLastSeen, err = s.db.Prepare("UPDATE transmissions SET last_seen = " + s.parameter(1) + " WHERE id = " + s.parameter(2) + " AND last_seen < " + s.parameter(3)) if err != nil { return err } s.stmtInsertObservation, err = s.db.Prepare(` INSERT INTO observations (transmission_id, observer_idx, direction, snr, rssi, score, path_json, timestamp, raw_hex, resolved_path) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + VALUES (` + s.parameter(1) + `, ` + s.parameter(2) + `, ` + s.parameter(3) + `, ` + s.parameter(4) + `, ` + s.parameter(5) + `, ` + s.parameter(6) + `, ` + s.parameter(7) + `, ` + s.parameter(8) + `, ` + s.parameter(9) + `, ` + s.parameter(10) + `) ON CONFLICT(transmission_id, observer_idx, COALESCE(path_json, '')) DO UPDATE SET - snr = COALESCE(excluded.snr, snr), - rssi = COALESCE(excluded.rssi, rssi), - score = COALESCE(excluded.score, score), - raw_hex = COALESCE(excluded.raw_hex, raw_hex), - resolved_path = COALESCE(excluded.resolved_path, resolved_path) + snr = COALESCE(excluded.snr, observations.snr), + rssi = COALESCE(excluded.rssi, observations.rssi), + score = COALESCE(excluded.score, observations.score), + raw_hex = COALESCE(excluded.raw_hex, observations.raw_hex), + resolved_path = COALESCE(excluded.resolved_path, observations.resolved_path) `) if err != nil { return err } - s.stmtUpsertNode, err = s.db.Prepare(` + s.stmtUpsertNode, err = s.db.Prepare(s.nativeSQL(` INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen) - VALUES (?, ?, ?, ?, ?, ?, ?) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7) ON CONFLICT(public_key) DO UPDATE SET - name = COALESCE(?, name), - role = COALESCE(?, role), - lat = COALESCE(?, lat), - lon = COALESCE(?, lon), - last_seen = MAX(MIN(COALESCE(last_seen, ''), ?), ?) - `) + name = COALESCE(?8, nodes.name), + role = COALESCE(?9, nodes.role), + lat = COALESCE(?10, nodes.lat), + lon = COALESCE(?11, nodes.lon), + last_seen = MAX(MIN(COALESCE(nodes.last_seen, ''), ?12), ?13) + `, ` + INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen) + VALUES (`+s.parameter(1)+`, `+s.parameter(2)+`, `+s.parameter(3)+`, `+s.parameter(4)+`, `+s.parameter(5)+`, `+s.parameter(6)+`, `+s.parameter(7)+`) + ON CONFLICT(public_key) DO UPDATE SET + name = COALESCE(`+s.parameter(8)+`, nodes.name), + role = COALESCE(`+s.parameter(9)+`, nodes.role), + lat = COALESCE(`+s.parameter(10)+`, nodes.lat), + lon = COALESCE(`+s.parameter(11)+`, nodes.lon), + last_seen = GREATEST(LEAST(COALESCE(nodes.last_seen, ''), `+s.parameter(12)+`), `+s.parameter(13)+`) + `)) if err != nil { return err } s.stmtIncrementAdvertCount, err = s.db.Prepare(` - UPDATE nodes SET advert_count = advert_count + 1 WHERE public_key = ? + UPDATE nodes SET advert_count = advert_count + 1 WHERE public_key = ` + s.parameter(1) + ` `) if err != nil { return err } - s.stmtUpsertObserver, err = s.db.Prepare(` + s.stmtUpsertObserver, err = s.db.Prepare(s.nativeSQL(` INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count, model, firmware, client_version, radio, battery_mv, uptime_secs, noise_floor, can_relay, can_relay_seen) - VALUES (?, ?, ?, ?, ?, 1, ?, ?, ?, ?, ?, ?, ?, COALESCE(?, 1), CASE WHEN ? IS NULL THEN 0 ELSE 1 END) + VALUES (?1, ?2, ?3, ?4, ?5, 1, ?6, ?7, ?8, ?9, ?10, ?11, ?12, COALESCE(?13, 1), CASE WHEN CAST(?14 AS INTEGER) IS NULL THEN 0 ELSE 1 END) ON CONFLICT(id) DO UPDATE SET - name = COALESCE(?, name), - iata = COALESCE(?, iata), - last_seen = MAX(MIN(COALESCE(last_seen, ''), ?), ?), - packet_count = packet_count + 1, - model = COALESCE(?, model), - firmware = COALESCE(?, firmware), - client_version = COALESCE(?, client_version), - radio = COALESCE(?, radio), - battery_mv = COALESCE(?, battery_mv), - uptime_secs = COALESCE(?, uptime_secs), - noise_floor = COALESCE(?, noise_floor), - can_relay = COALESCE(?, can_relay), - can_relay_seen = CASE WHEN ? IS NULL THEN can_relay_seen ELSE 1 END - `) + name = COALESCE(?15, observers.name), + iata = COALESCE(?16, observers.iata), + last_seen = MAX(MIN(COALESCE(observers.last_seen, ''), ?17), ?18), + packet_count = observers.packet_count + 1, + model = COALESCE(?19, observers.model), + firmware = COALESCE(?20, observers.firmware), + client_version = COALESCE(?21, observers.client_version), + radio = COALESCE(?22, observers.radio), + battery_mv = COALESCE(?23, observers.battery_mv), + uptime_secs = COALESCE(?24, observers.uptime_secs), + noise_floor = COALESCE(?25, observers.noise_floor), + can_relay = COALESCE(?26, observers.can_relay), + can_relay_seen = CASE WHEN CAST(?27 AS INTEGER) IS NULL THEN observers.can_relay_seen ELSE 1 END + `, ` + INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count, model, firmware, client_version, radio, battery_mv, uptime_secs, noise_floor, can_relay, can_relay_seen) + VALUES (`+s.parameter(1)+`, `+s.parameter(2)+`, `+s.parameter(3)+`, `+s.parameter(4)+`, `+s.parameter(5)+`, 1, `+s.parameter(6)+`, `+s.parameter(7)+`, `+s.parameter(8)+`, `+s.parameter(9)+`, `+s.parameter(10)+`, `+s.parameter(11)+`, `+s.parameter(12)+`, COALESCE(`+s.parameter(13)+`, 1), CASE WHEN `+s.parameter(14)+`::bigint IS NULL THEN 0 ELSE 1 END) + ON CONFLICT(id) DO UPDATE SET + name = COALESCE(`+s.parameter(15)+`, observers.name), + iata = COALESCE(`+s.parameter(16)+`, observers.iata), + last_seen = GREATEST(LEAST(COALESCE(observers.last_seen, ''), `+s.parameter(17)+`), `+s.parameter(18)+`), + packet_count = observers.packet_count + 1, + model = COALESCE(`+s.parameter(19)+`, observers.model), + firmware = COALESCE(`+s.parameter(20)+`, observers.firmware), + client_version = COALESCE(`+s.parameter(21)+`, observers.client_version), + radio = COALESCE(`+s.parameter(22)+`, observers.radio), + battery_mv = COALESCE(`+s.parameter(23)+`, observers.battery_mv), + uptime_secs = COALESCE(`+s.parameter(24)+`, observers.uptime_secs), + noise_floor = COALESCE(`+s.parameter(25)+`, observers.noise_floor), + can_relay = COALESCE(`+s.parameter(26)+`, observers.can_relay), + can_relay_seen = CASE WHEN `+s.parameter(27)+`::bigint IS NULL THEN observers.can_relay_seen ELSE 1 END + `)) if err != nil { return err } - s.stmtGetObserverRowid, err = s.db.Prepare("SELECT rowid FROM observers WHERE id = ?") + s.stmtGetObserverRowid, err = s.db.Prepare("SELECT rowid FROM observers WHERE id = " + s.parameter(1)) if err != nil { return err } @@ -1044,29 +418,36 @@ func (s *Store) prepareStatements() error { // Args: ingestNow, rxTime, ingestNow, rxTime, rowid // MIN(existing, ingestNow) clamps any future value already in the DB before // taking MAX with rxTime, so the guard never locks in a past bug's stale future. - s.stmtUpdateObserverLastSeen, err = s.db.Prepare(` + s.stmtUpdateObserverLastSeen, err = s.db.Prepare(s.nativeSQL(` UPDATE observers SET - last_seen = MAX(MIN(COALESCE(last_seen, ''), ?), ?), - last_packet_at = MAX(MIN(COALESCE(last_packet_at, ''), ?), ?) - WHERE rowid = ?`) + last_seen = MAX(MIN(COALESCE(last_seen, ''), ?1), ?2), + last_packet_at = MAX(MIN(COALESCE(last_packet_at, ''), ?3), ?4) + WHERE rowid = ?5`, ` + UPDATE observers SET + last_seen = GREATEST(LEAST(COALESCE(last_seen, ''), `+s.parameter(1)+`), `+s.parameter(2)+`), + last_packet_at = GREATEST(LEAST(COALESCE(last_packet_at, ''), `+s.parameter(3)+`), `+s.parameter(4)+`) + WHERE rowid = `+s.parameter(5))) if err != nil { return err } s.stmtUpdateNodeTelemetry, err = s.db.Prepare(` UPDATE nodes SET - battery_mv = COALESCE(?, battery_mv), - temperature_c = COALESCE(?, temperature_c) - WHERE public_key = ? + battery_mv = COALESCE(` + s.parameter(1) + `, battery_mv), + temperature_c = COALESCE(` + s.parameter(2) + `, temperature_c) + WHERE public_key = ` + s.parameter(3) + ` `) if err != nil { return err } s.stmtUpsertMetrics, err = s.db.Prepare(` - INSERT OR REPLACE INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors, battery_mv, packets_sent, packets_recv) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) - `) + INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors, battery_mv, packets_sent, packets_recv) + VALUES (` + s.parameter(1) + `, ` + s.parameter(2) + `, ` + s.parameter(3) + `, ` + s.parameter(4) + `, ` + s.parameter(5) + `, ` + s.parameter(6) + `, ` + s.parameter(7) + `, ` + s.parameter(8) + `, ` + s.parameter(9) + `) + ON CONFLICT(observer_id,timestamp) DO UPDATE SET noise_floor=excluded.noise_floor, + tx_air_secs=excluded.tx_air_secs,rx_air_secs=excluded.rx_air_secs,recv_errors=excluded.recv_errors, + battery_mv=excluded.battery_mv,packets_sent=excluded.packets_sent,packets_recv=excluded.packets_recv + `) if err != nil { return err } @@ -1077,160 +458,139 @@ func (s *Store) prepareStatements() error { // InsertTransmission inserts a decoded packet into transmissions + observations. // Returns true if a new transmission was created (not a duplicate hash). func (s *Store) InsertTransmission(data *PacketData) (bool, error) { - hash := data.Hash - if hash == "" { + if data.Hash == "" { return false, nil } - - // Wait/hold instrumentation (#1340). The hot path uses prepared - // statements that auto-commit; gate the whole function under - // writerMu so concurrent mqtt_handler inserts queue behind any - // other writer (vacuum, prune, neighbor-builder) and the wait is - // Go-visible. - mqttWaitStart := time.Now() + waitStart := time.Now() writerMu.Lock() - mqttWait := time.Since(mqttWaitStart) - mqttHoldStart := time.Now() + wait := time.Since(waitStart) + holdStart := time.Now() defer func() { - mqttHold := time.Since(mqttHoldStart) writerMu.Unlock() - recordWriterTiming("mqtt_handler", mqttWait, mqttHold, "InsertTransmission") + recordWriterTiming("mqtt_handler", wait, time.Since(holdStart), "InsertTransmission") }() - - rxTime := data.Timestamp + tx, err := beginWrite(s.db, s.Backend()) + if err != nil { + return false, err + } + defer tx.Rollback() ingestNow := time.Now().UTC().Format(time.RFC3339) + rxTime := data.Timestamp if rxTime == "" { rxTime = ingestNow } - var txID int64 - isNew := false - - // Check for existing transmission - var existingID int64 - var existingFirstSeen string - err := s.stmtGetTxByHash.QueryRow(hash).Scan(&existingID, &existingFirstSeen) - if err == nil { - // Existing transmission - txID = existingID - if rxTime < existingFirstSeen { - _, _ = s.stmtUpdateTxFirstSeen.Exec(rxTime, txID) - } - } else { - // New transmission - isNew = true - result, err := s.stmtInsertTransmission.Exec( - data.RawHex, hash, rxTime, - data.RouteType, data.PayloadType, data.PayloadVersion, - data.DecodedJSON, nilIfEmpty(data.ChannelHash), - scopeNameForDB(data), - nilIfEmpty(data.FromPubkey), - epochSecondsForLastSeen(rxTime), - nilIfEmpty(data.Code1), - nilIfEmpty(data.Code2), - ) - if err != nil { - s.Stats.WriteErrors.Add(1) - return false, fmt.Errorf("insert transmission: %w", err) - } - txID, _ = result.LastInsertId() - s.Stats.TransmissionsInserted.Add(1) + var firstSeen string + err = tx.Stmt(s.stmtGetTxByHash).QueryRow(data.Hash).Scan(&txID, &firstSeen) + isNew := err == sql.ErrNoRows + if err != nil && !isNew { + return false, err } - - if !isNew { - s.Stats.DuplicateTransmissions.Add(1) + if isNew { + err = tx.Stmt(s.stmtInsertTransmission).QueryRow(data.RawHex, data.Hash, rxTime, data.RouteType, data.PayloadType, data.PayloadVersion, data.DecodedJSON, nilIfEmpty(data.ChannelHash), scopeNameForDB(data), nilIfEmpty(data.FromPubkey), epochSecondsForLastSeen(rxTime), nilIfEmpty(data.Code1), nilIfEmpty(data.Code2)).Scan(&txID) + } else if rxTime < firstSeen { + _, err = tx.Stmt(s.stmtUpdateTxFirstSeen).Exec(rxTime, txID) + } + if err != nil { + s.Stats.WriteErrors.Add(1) + return false, fmt.Errorf("write transmission: %w", err) } - // Capture route evidence BEFORE the observation conflict update can erase - // a different route. Duplicate evidence is a read-only indexed probe. - // Analytics failures must not drop core observations or liveness updates; - // known evidence is a lower bound when an evidence read/write fails. if data.PayloadType == 4 { if bit := packetpath.AdvertRouteEvidence(data.RawHex); bit != 0 { - if _, err := s.stmtInsertAdvertEvidence.Exec(txID, bit, txID, bit); err != nil { + if err := optionalWrite(tx, func() error { _, e := tx.Stmt(s.stmtInsertAdvertEvidence).Exec(txID, bit, txID, bit); return e }); err != nil { s.Stats.WriteErrors.Add(1) log.Printf("[db] record advert route evidence (non-fatal): %v", err) } } } - - // Resolve observer_idx and update last_seen var observerIdx *int64 if data.ObserverID != "" { - var rowid int64 - err := s.stmtGetObserverRowid.QueryRow(data.ObserverID).Scan(&rowid) + var id int64 + err = tx.Stmt(s.stmtGetObserverRowid).QueryRow(data.ObserverID).Scan(&id) if err == nil { - observerIdx = &rowid - // observer.last_seen and last_packet_at answer "when did the analyzer - // last hear from this observer" — both are ingest-time questions. - // Per-packet rxTime is stored separately on observations/transmissions - // using envelope time (see InsertTransmission above). See #1465. - _, _ = s.stmtUpdateObserverLastSeen.Exec(ingestNow, ingestNow, ingestNow, ingestNow, rowid) + observerIdx = &id + _, err = tx.Stmt(s.stmtUpdateObserverLastSeen).Exec(ingestNow, ingestNow, ingestNow, ingestNow, id) + } + if err != nil && err != sql.ErrNoRows { + return false, err } } - - // Until backfill commits this observation's evidence, preserve its old - // frame before UPSERT can destroy it. writerMu also guards checkpoints. - // Run even for malformed incoming raw: the surviving old frame is valid - // evidence independently of whether the new frame contributes a bit. if !isNew && data.PayloadType == 4 && observerIdx != nil && data.RawHex != "" { - if err := s.preserveLegacyAdvertObservation(txID, *observerIdx, data.PathJSON); err != nil { + if err := optionalWrite(tx, func() error { return s.preserveLegacyAdvertObservation(tx, txID, *observerIdx, data.PathJSON) }); err != nil { s.Stats.WriteErrors.Add(1) log.Printf("[db] preserve legacy advert evidence (non-fatal): %v", err) } } - - // Insert observation epochTs := time.Now().Unix() - if t, err := time.Parse(time.RFC3339, rxTime); err == nil { + if t, e := time.Parse(time.RFC3339, rxTime); e == nil { epochTs = t.Unix() } - - // Resolve hop prefixes to full pubkeys for `observations.resolved_path`. - // Per #1547: this writer was lost in the #1289 refactor and lives in - // the ingestor now. Per #1560: use the context-aware resolver so - // 1-byte prefix collisions are disambiguated via NeighborGraph - // adjacency (anchored on from_pubkey for ADVERTs, previous hop - // otherwise). Empty resolved JSON → NULL via nilIfEmpty. - resolved := resolvePathWithContext( - parsePathArray(data.PathJSON), - strings.ToLower(data.FromPubkey), - s.neighborGraph.load(), - s.prefixIdx.load(), - ) - resolvedJSON := marshalResolvedPath(resolved) - - _, err = s.stmtInsertObservation.Exec( - txID, observerIdx, data.Direction, - data.SNR, data.RSSI, data.Score, - data.PathJSON, epochTs, nilIfEmpty(data.RawHex), - nilIfEmpty(resolvedJSON), - ) + resolved := resolvePathWithContext(parsePathArray(data.PathJSON), strings.ToLower(data.FromPubkey), s.neighborGraph.load(), s.prefixIdx.load()) + _, err = tx.Stmt(s.stmtInsertObservation).Exec(txID, observerIdx, data.Direction, data.SNR, data.RSSI, data.Score, data.PathJSON, epochTs, nilIfEmpty(data.RawHex), nilIfEmpty(marshalResolvedPath(resolved))) if err != nil { s.Stats.WriteErrors.Add(1) - log.Printf("[db] observation insert (non-fatal): %v", err) + return false, fmt.Errorf("insert observation: %w", err) + } + if _, err = tx.Stmt(s.stmtBumpTxLastSeen).Exec(epochTs, txID, epochTs); err != nil { + return false, err + } + if err = tx.Commit(); err != nil { + s.Stats.WriteErrors.Add(1) + return false, fmt.Errorf("commit packet: %w", err) + } + if isNew { + s.Stats.TransmissionsInserted.Add(1) } else { - s.Stats.ObservationsInserted.Add(1) - // #1598: a resolved hop proves the node was forwarding traffic at - // rxTime. Refresh its last_seen so staleness/eviction logic sees - // relay activity, not just ADVERTs. - s.touchRelayNodesLocked(resolvedPubkeys(resolved), rxTime) - // #1690: bump transmissions.last_seen so cold-load can filter on - // effective recency. Conditional `last_seen < ?` so we never go - // backwards on out-of-order ingest. - if _, err := s.stmtBumpTxLastSeen.Exec(epochTs, txID, epochTs); err != nil { - log.Printf("[db] tx last_seen bump (non-fatal): %v", err) - } + s.Stats.DuplicateTransmissions.Add(1) } - - // Each prepared-stmt Exec auto-commits. Count one WAL commit per - // successful InsertTransmission so the perf page sees commit pressure. + s.Stats.ObservationsInserted.Add(1) s.Stats.WALCommits.Add(1) - + s.touchRelayNodesLocked(resolvedPubkeys(resolved), rxTime) return isNew, nil } +// beginWrite serializes ID allocation before any statement can consume an +// identity value. A transaction lock releases only after commit or rollback. +func beginWrite(db *sql.DB, backend ...dbconfig.Backend) (*sql.Tx, error) { + return beginWriteContext(context.Background(), db, backend...) +} +func beginWriteContext(ctx context.Context, db *sql.DB, backend ...dbconfig.Backend) (*sql.Tx, error) { + tx, err := db.BeginTx(ctx, nil) + if err != nil { + return nil, err + } + if len(backend) == 0 || backend[0] == dbconfig.Postgres { + if _, err = tx.ExecContext(ctx, `SELECT pg_advisory_xact_lock(hashtextextended(current_schema(),$1))`, dbschema.WriterLockKey); err != nil { + tx.Rollback() + return nil, err + } + } + return tx, nil +} + +// Analytics are best effort; a failed PostgreSQL statement otherwise poisons +// the packet transaction. Savepoints preserve the existing non-fatal behavior. +func optionalWrite(tx *sql.Tx, fn func() error) error { + if _, err := tx.Exec(`SAVEPOINT packet_analytics`); err != nil { + return err + } + err := fn() + if err != nil { + if _, rollbackErr := tx.Exec(`ROLLBACK TO SAVEPOINT packet_analytics`); rollbackErr != nil { + return rollbackErr + } + } + _, releaseErr := tx.Exec(`RELEASE SAVEPOINT packet_analytics`) + if err != nil { + return err + } + return releaseErr +} + // UpsertNode inserts or updates a node. func (s *Store) UpsertNode(pubKey, name, role string, lat, lon *float64, lastSeen string) error { + // Preserve the decoder's byte cap while making the name safe for SQL text. + name = strings.ToValidUTF8(name, "") ingestNow := time.Now().UTC().Format(time.RFC3339) now := lastSeen if now == "" { @@ -1262,7 +622,7 @@ func (s *Store) MarkNodeForeign(pubKey string) error { if pubKey == "" { return nil } - _, err := s.db.Exec(`UPDATE nodes SET foreign_advert = 1 WHERE public_key = ?`, pubKey) + _, err := s.db.Exec(`UPDATE nodes SET foreign_advert = 1 WHERE public_key = `+s.parameter(1), pubKey) if err != nil { s.Stats.WriteErrors.Add(1) } @@ -1340,7 +700,7 @@ func (s *Store) UpsertObserverAt(id, name, iata string, meta *ObserverMeta, last s.Stats.ObserverUpserts.Add(1) // Reactivate if this observer was previously marked inactive - s.db.Exec(`UPDATE observers SET inactive = 0 WHERE id = ? AND inactive = 1`, id) + s.db.Exec(`UPDATE observers SET inactive = 0 WHERE id = `+s.parameter(1)+` AND inactive = 1`, id) return nil } @@ -1361,20 +721,33 @@ func (s *Store) UpsertObserverRetained(id, name, iata string, meta *ObserverMeta normalizedIATA := strings.TrimSpace(strings.ToUpper(iata)) model, firmware, clientVersion, radio, batteryMv, uptimeSecs, noiseFloor, canRelay := observerMetaColumns(meta) - _, err := s.db.Exec(` + _, err := s.db.Exec(s.nativeSQL(` UPDATE observers SET - name = COALESCE(?, name), - iata = COALESCE(?, iata), - model = COALESCE(?, model), - firmware = COALESCE(?, firmware), - client_version = COALESCE(?, client_version), - radio = COALESCE(?, radio), - battery_mv = COALESCE(?, battery_mv), - uptime_secs = COALESCE(?, uptime_secs), - noise_floor = COALESCE(?, noise_floor), - can_relay = COALESCE(?, can_relay), - can_relay_seen = CASE WHEN ? IS NULL THEN can_relay_seen ELSE 1 END - WHERE id = ?`, + name = COALESCE(?1, name), + iata = COALESCE(?2, iata), + model = COALESCE(?3, model), + firmware = COALESCE(?4, firmware), + client_version = COALESCE(?5, client_version), + radio = COALESCE(?6, radio), + battery_mv = COALESCE(?7, battery_mv), + uptime_secs = COALESCE(?8, uptime_secs), + noise_floor = COALESCE(?9, noise_floor), + can_relay = COALESCE(?10, can_relay), + can_relay_seen = CASE WHEN CAST(?11 AS INTEGER) IS NULL THEN can_relay_seen ELSE 1 END + WHERE id = ?12`, ` + UPDATE observers SET + name = COALESCE(`+s.parameter(1)+`, name), + iata = COALESCE(`+s.parameter(2)+`, iata), + model = COALESCE(`+s.parameter(3)+`, model), + firmware = COALESCE(`+s.parameter(4)+`, firmware), + client_version = COALESCE(`+s.parameter(5)+`, client_version), + radio = COALESCE(`+s.parameter(6)+`, radio), + battery_mv = COALESCE(`+s.parameter(7)+`, battery_mv), + uptime_secs = COALESCE(`+s.parameter(8)+`, uptime_secs), + noise_floor = COALESCE(`+s.parameter(9)+`, noise_floor), + can_relay = COALESCE(`+s.parameter(10)+`, can_relay), + can_relay_seen = CASE WHEN `+s.parameter(11)+`::bigint IS NULL THEN can_relay_seen ELSE 1 END + WHERE id = `+s.parameter(12)), name, normalizedIATA, model, firmware, clientVersion, radio, batteryMv, uptimeSecs, noiseFloor, canRelay, canRelay, id, ) @@ -1426,7 +799,7 @@ func observerMetaColumns(meta *ObserverMeta) (model, firmware, clientVersion, ra return model, firmware, clientVersion, radio, batteryMv, uptimeSecs, noiseFloor, canRelay } -// Close checkpoints the WAL and closes the database. +// Close waits for data backfills and closes the database. func (s *Store) Close() error { // Long resumable backfills must not hold shutdown hostage; they pick up // from their stored progress on the next boot. @@ -1434,7 +807,6 @@ func (s *Store) Close() error { s.hopBackfillCancel() } s.backfillWg.Wait() - s.Checkpoint() return s.db.Close() } @@ -1501,7 +873,7 @@ func (s *Store) InsertMetrics(data *MetricsData) error { func (s *Store) PruneOldMetrics(retentionDays int) (int64, error) { cutoff := time.Now().UTC().AddDate(0, 0, -retentionDays).Format(time.RFC3339) // Tagged for /api/perf writer-lock visibility (#1340). - result, err := s.instrumentedExec("prune_metrics", `DELETE FROM observer_metrics WHERE timestamp < ?`, cutoff) + result, err := s.instrumentedExec("prune_metrics", `DELETE FROM observer_metrics WHERE timestamp < `+s.parameter(1), cutoff) if err != nil { return 0, fmt.Errorf("prune metrics: %w", err) } @@ -1512,265 +884,6 @@ func (s *Store) PruneOldMetrics(retentionDays int) (int64, error) { return n, nil } -// CheckAutoVacuum inspects the current auto_vacuum mode and logs a warning -// if not INCREMENTAL. Performs opt-in full VACUUM if db.vacuumOnStartup is set (#919). -func (s *Store) CheckAutoVacuum(cfg *Config) { - var autoVacuum int - if err := s.db.QueryRow("PRAGMA auto_vacuum").Scan(&autoVacuum); err != nil { - log.Printf("[db] warning: could not read auto_vacuum: %v", err) - return - } - - if autoVacuum == 2 { - log.Printf("[db] auto_vacuum=INCREMENTAL") - return - } - - modes := map[int]string{0: "NONE", 1: "FULL", 2: "INCREMENTAL"} - mode := modes[autoVacuum] - if mode == "" { - mode = fmt.Sprintf("UNKNOWN(%d)", autoVacuum) - } - - log.Printf("[db] auto_vacuum=%s — DB needs one-time VACUUM to enable incremental auto-vacuum. "+ - "Set db.vacuumOnStartup: true in config to migrate (will block startup for several minutes on large DBs). "+ - "See https://github.com/Kpa-clawbot/CoreScope/issues/919", mode) - - if cfg.DB != nil && cfg.DB.VacuumOnStartup { - // WARNING: Full VACUUM creates a temporary copy of the entire DB file. - // Requires ~2× the DB file size in free disk space or it will fail. - log.Printf("[db] vacuumOnStartup=true — starting one-time full VACUUM (ensure 2x DB size free disk space)...") - start := time.Now() - - if _, err := s.instrumentedExec("vacuum", "PRAGMA auto_vacuum = INCREMENTAL"); err != nil { - log.Printf("[db] VACUUM failed: could not set auto_vacuum: %v", err) - return - } - if _, err := s.instrumentedExec("vacuum", "VACUUM"); err != nil { - log.Printf("[db] VACUUM failed: %v", err) - return - } - - elapsed := time.Since(start) - log.Printf("[db] VACUUM complete in %v — auto_vacuum is now INCREMENTAL", elapsed.Round(time.Millisecond)) - } -} - -// RunIncrementalVacuum returns free pages to the OS (#919). -// Safe to call on auto_vacuum=NONE databases (noop). -func (s *Store) RunIncrementalVacuum(pages int) { - // Tagged for /api/perf writer-lock visibility (#1340). - if _, err := s.instrumentedExec("vacuum", fmt.Sprintf("PRAGMA incremental_vacuum(%d)", pages)); err != nil { - log.Printf("[vacuum] incremental_vacuum error: %v", err) - } -} - -// RefreshPlannerStats rebuilds the query planner's cardinality statistics -// (#2058). -// -// Without a sqlite_stat1 table the planner works from built-in guesses, and on -// the channel queries it guesses wrong: it drives from the plain -// idx_transmissions_payload_type rather than idx_tx_channel_hash, the partial -// index (WHERE payload_type = 5) this schema already carries for that exact -// filter. -// -// Measured on the 9.4 GB staging database (1,250,489 transmissions, 14,169,329 -// observations), region-filtered GetChannels, counting page-cache misses -// because wall time there is dominated by the OS page cache (56.7s cold, 0.80s -// warm, for the same query and plan): -// -// analysis_limit ANALYZE driving index page misses -// none (no stats) - idx_transmissions_payload_type 143,442 -// 400 171ms idx_transmissions_payload_type 143,449 -// 1000 171ms idx_transmissions_payload_type 143,450 -// 10000 2.0s* idx_tx_channel_hash 107,429 -// 0 (unbounded) 242.9s idx_tx_channel_hash 107,429 -// -// (*) Every ANALYZE duration in that table was timed on a warm page cache, one -// after another. The same statement at limit 10000 took 3m43.9s cold, on a -// freshly started container. See EnsurePlannerStats. -// -// So 10000 buys the whole plan change, and the four-minute unbounded ANALYZE -// buys nothing beyond it. 400, the value SQLite's documentation offers for the -// bounded form, changes nothing at all on this data: it samples too few rows to -// separate the 126,336-row partial index from the 920,700-row plain one. -// -// ANALYZE, not PRAGMA optimize. Measured on the same database: optimize is a -// no-op here, because it only analyzes tables that the calling connection has -// itself queried during the session, and a maintenance call has queried none. -// PRAGMA optimize(0x03) returned no statements and sqlite_stat1 was not -// created. -// -// Note that analysis_limit=0 means *no* limit to SQLite, not "use a default". -// Config.AnalysisLimit maps an unset config to 10000 for that reason, and a -// negative value here disables the refresh. -// -// Returns whether the statistics were refreshed. This function owns its -// logging; callers need add nothing. -func (s *Store) RefreshPlannerStats(analysisLimit int) bool { - if analysisLimit < 0 { - return false - } - first := !s.hasPlannerStats() - start := time.Now() - // Tagged for /api/perf writer-lock visibility (#1340). - if _, err := s.instrumentedExec("analyze", fmt.Sprintf("PRAGMA analysis_limit=%d", analysisLimit)); err != nil { - log.Printf("[analyze] could not set analysis_limit: %v", err) - return false - } - if _, err := s.instrumentedExec("analyze", "ANALYZE"); err != nil { - log.Printf("[analyze] ANALYZE failed: %v", err) - return false - } - elapsed := time.Since(start).Round(time.Millisecond) - s.recordPlannerStatsRefresh(time.Now()) - if first { - log.Printf("[analyze] planner statistics built in %v (analysis_limit=%d, first run against this database)", elapsed, analysisLimit) - } else { - log.Printf("[analyze] planner statistics refreshed in %v (analysis_limit=%d)", elapsed, analysisLimit) - } - return true -} - -// EnsurePlannerStats builds planner statistics when the database has none, and -// reports whether it did (#2058). -// -// This exists to close the window the startup stagger opens. The refresh ticker -// waits 2 minutes before its first run, and a query arriving in that window -// against a database with no statistics at all gets the plan measured in -// RefreshPlannerStats above: 143,442 pages read, which timed at 56.7s cold on -// the 9.4 GB staging file. -// -// It fires once per database, not once per restart. sqlite_stat1 is an ordinary -// table, so once written it stays in the file and a fresh read-only connection -// reads it back (verified across connection close, and through a mode=ro -// handle). Every later start therefore costs one query against sqlite_master and -// leaves the work to the ticker. -// -// The trade is a one-time ANALYZE early in startup, and it is not cheap on a -// cold page cache. Observed on staging at 9.4 GB: 3m43.9s, against the 2.0s the -// same statement takes warm. For those 3m44s it holds the store's single write -// connection (SetMaxOpenConns(1), db.go:142), so ingest stalls and buffers: the -// observations table took zero rows for four minutes and then 1027 in the minute -// the ANALYZE finished, against about 130 a minute either side, with nothing -// dropped. Hence the warning below, so an operator watching a first deploy can -// tell this apart from a hang. -// -// That cost belongs to the first ANALYZE, not to running it here. The daily -// refresh would otherwise pay exactly the same 3m44s two minutes later; this -// moves it earlier, where it overlaps the startup burst the ingest buffer is -// already sized for. It also records the refresh time, so the daily refresh -// then waits 24 hours (#2146). -func (s *Store) EnsurePlannerStats(analysisLimit int) bool { - if s.hasPlannerStats() { - return false - } - log.Printf("[analyze] this database has no planner statistics; building them now. " + - "ANALYZE holds the single write connection until it finishes (3m43.9s measured on 9.4 GB, cold), " + - "so ingest will buffer and catch up. Once per database, not once per restart.") - return s.RefreshPlannerStats(analysisLimit) -} - -// plannerStatsInterval is how often the planner statistics are refreshed, and -// plannerStatsStagger how long after startup the first refresh waits at least. -const ( - plannerStatsInterval = 24 * time.Hour - plannerStatsStagger = 2 * time.Minute -) - -// nextPlannerStatsRefresh returns how long to wait before the next refresh, -// given when the previous one succeeded (zero if never): until it is -// plannerStatsInterval old, but at least plannerStatsStagger and at most -// plannerStatsInterval. -// -// Before #2146 the first refresh ran plannerStatsStagger after every start. -// On a cold page cache that ANALYZE took 6 to 9 minutes on a 10-11 GB -// database, holding the single write connection, so each restart stalled -// ingest that long for statistics that were hours old at most. -func nextPlannerStatsRefresh(last, now time.Time) time.Duration { - if last.IsZero() { - return plannerStatsStagger - } - d := last.Add(plannerStatsInterval).Sub(now) - if d < plannerStatsStagger { - return plannerStatsStagger - } - if d > plannerStatsInterval { - return plannerStatsInterval - } - return d -} - -// plannerStatsStampPath is the file next to the database that records when -// the planner statistics were last refreshed. -func (s *Store) plannerStatsStampPath() string { - return s.path + ".planner-stats-refreshed" -} - -// lastPlannerStatsRefresh returns when RefreshPlannerStats last succeeded on -// this database, or the zero time when that is not known: the later of the -// time this process remembers and the time recorded in the file. -func (s *Store) lastPlannerStatsRefresh() time.Time { - var last time.Time - if ns := s.plannerStatsAt.Load(); ns != 0 { - last = time.Unix(0, ns) - } - b, err := os.ReadFile(s.plannerStatsStampPath()) - if err != nil { - return last - } - t, err := time.Parse(time.RFC3339, strings.TrimSpace(string(b))) - if err != nil || t.Before(last) { - return last - } - return t -} - -// recordPlannerStatsRefresh remembers when the statistics were refreshed and -// writes it next to the database for the next start. If the write fails, this -// process still waits the full interval; only the first refresh after the -// next restart comes early. -func (s *Store) recordPlannerStatsRefresh(at time.Time) { - s.plannerStatsAt.Store(at.UnixNano()) - tmp := s.plannerStatsStampPath() + ".tmp" - if err := os.WriteFile(tmp, []byte(at.UTC().Format(time.RFC3339)), 0o644); err != nil { - log.Printf("[analyze] could not record the refresh time: %v", err) - return - } - if err := os.Rename(tmp, s.plannerStatsStampPath()); err != nil { - log.Printf("[analyze] could not record the refresh time: %v", err) - } -} - -// hasPlannerStats reports whether ANALYZE has ever run against this database. -// A query error reads as "no stats", which is the safe direction for both -// callers: it costs the log line a wrong word, and costs EnsurePlannerStats one -// ANALYZE that was not needed, rather than skipping one that was. -func (s *Store) hasPlannerStats() bool { - var n int - if err := s.db.QueryRow( - `SELECT count(*) FROM sqlite_master WHERE type='table' AND name='sqlite_stat1'`).Scan(&n); err != nil { - return false - } - return n > 0 -} - -// Checkpoint runs a WAL checkpoint (TRUNCATE mode). -// Returns the number of WAL frames checkpointed (0 if WAL was already empty). -// TRUNCATE resets the WAL file to zero bytes when all frames are checkpointed; -// if active readers hold frames, it checkpoints what it can and leaves the rest. -func (s *Store) Checkpoint() int { - var busy, walFrames, checkpointed int - if err := s.db.QueryRow("PRAGMA wal_checkpoint(TRUNCATE)").Scan(&busy, &walFrames, &checkpointed); err != nil { - log.Printf("[db] WAL checkpoint error: %v", err) - return 0 - } - if walFrames > 0 { - log.Printf("[db] WAL checkpoint: %d/%d frames checkpointed (blocked=%v)", checkpointed, walFrames, busy != 0) - } - return checkpointed -} - // BackfillPathJSONAsync launches the path_json backfill in a background goroutine. // It processes observations with NULL/empty path_json that have raw_hex available, // decoding hop paths and updating the column. Safe to run concurrently with ingest @@ -1807,7 +920,7 @@ func (s *Store) BackfillPathJSONAsync() { -- to prevent the infinite re-UPDATE loop fixed in #1119. AND (o.path_json IS NULL OR o.path_json = '') AND t.payload_type != 9 - LIMIT ?`, batchSize) + LIMIT `+s.parameter(1), batchSize) if err != nil { log.Printf("[backfill] path_json query error: %v", err) errored = true @@ -1831,7 +944,7 @@ func (s *Store) BackfillPathJSONAsync() { for _, r := range batch { hops, err := packetpath.DecodePathFromRawHex(r.rawHex) if err != nil || len(hops) == 0 { - if _, execErr := s.db.Exec(`UPDATE observations SET path_json = '[]' WHERE id = ?`, r.id); execErr != nil { + if _, execErr := s.db.Exec(`UPDATE observations SET path_json = '[]' WHERE id = `+s.parameter(1), r.id); execErr != nil { log.Printf("[backfill] write error (id=%d): %v", r.id, execErr) } else { s.Stats.IncBackfill("path_json") @@ -1839,7 +952,7 @@ func (s *Store) BackfillPathJSONAsync() { continue } b, _ := json.Marshal(hops) - if _, execErr := s.db.Exec(`UPDATE observations SET path_json = ? WHERE id = ?`, string(b), r.id); execErr != nil { + if _, execErr := s.db.Exec(`UPDATE observations SET path_json = `+s.parameter(1)+` WHERE id = `+s.parameter(2), string(b), r.id); execErr != nil { log.Printf("[backfill] write error (id=%d): %v", r.id, execErr) } else { updated++ @@ -1920,7 +1033,7 @@ func (s *Store) BackfillDefaultScopeAsync(regionSet *regionKeySet) { // inserted before the transport_codes_v1 migration, by re-parsing raw_hex. // Runs in a background goroutine so it does not block MQTT startup. // -// Batched: SQLite holds a single write connection (SetMaxOpenConns(1)), so a +// Batched: the ingestor retains a single write connection, so a // multi-million-row UPDATE in one transaction would stall live ingest for the // whole run. 5k-row batches keep each transaction short. func (s *Store) BackfillTransportCodesAsync() { @@ -1970,9 +1083,9 @@ func (s *Store) backfillTransportCodes(batchSize int) (int, error) { for { rows, err := s.db.Query(` SELECT id, raw_hex FROM transmissions - WHERE id > ? AND code1 IS NULL AND route_type IN (0, 3) + WHERE id > `+s.parameter(1)+` AND code1 IS NULL AND route_type IN (0, 3) ORDER BY id - LIMIT ?`, lastID, batchSize) + LIMIT `+s.parameter(2), lastID, batchSize) if err != nil { return total, fmt.Errorf("transport_codes select: %w", err) } @@ -2003,11 +1116,11 @@ func (s *Store) backfillTransportCodes(batchSize int) (int, error) { } if len(items) > 0 { - tx, err := s.db.Begin() + tx, err := beginWrite(s.db, s.Backend()) if err != nil { return total, fmt.Errorf("transport_codes begin: %w", err) } - stmt, err := tx.Prepare(`UPDATE transmissions SET code1 = ?, code2 = ? WHERE id = ?`) + stmt, err := tx.Prepare(`UPDATE transmissions SET code1 = ` + s.parameter(1) + `, code2 = ` + s.parameter(2) + ` WHERE id = ` + s.parameter(3)) if err != nil { tx.Rollback() return total, fmt.Errorf("transport_codes prepare: %w", err) @@ -2054,7 +1167,7 @@ func (s *Store) LogStats() { // carries the requirement for future callers. // // Ownership (#1283/#1287/#1289): nodes is written by the ingestor only. -// The server opens SQLite mode=ro; its former touchRelayLastSeen has been +// The server uses a restricted PostgreSQL reader; its former touchRelayLastSeen has been // failing on every call since that refactor and is removed in this change. // cmd/server/readonly_invariant_test.go now guards against reintroduction. // @@ -2066,7 +1179,7 @@ func (s *Store) LogStats() { // Never inserts: the UPDATE matches an existing row or does nothing. // Never rewinds: the last_seen guard makes out-of-order ingest a no-op. // UpsertNode's ON CONFLICT clause is monotonic in the same direction -// (MAX(MIN(last_seen, ingestNow), rxTime) reduces to MAX(last_seen, rxTime) +// (GREATEST(LEAST(last_seen, ingestNow), rxTime) reduces to MAX(last_seen, rxTime) // for any non-future stored value), so an ADVERT cannot undo a touch. func (s *Store) touchRelayNodesLocked(pubkeys []string, rxTime string) { if len(pubkeys) == 0 || s.stmtTouchNodeLastSeen == nil { @@ -2130,17 +1243,17 @@ func (s *Store) compactRelayTouched(now time.Time) { // Returns the number of nodes moved. func (s *Store) MoveStaleNodes(nodeDays int) (int64, error) { cutoff := time.Now().UTC().AddDate(0, 0, -nodeDays).Format(time.RFC3339) - tx, err := s.db.Begin() + tx, err := beginWrite(s.db, s.Backend()) if err != nil { return 0, fmt.Errorf("begin tx: %w", err) } defer tx.Rollback() - _, err = tx.Exec(`INSERT OR REPLACE INTO inactive_nodes SELECT * FROM nodes WHERE last_seen < ?`, cutoff) + _, err = tx.Exec(`INSERT INTO inactive_nodes (public_key,name,role,lat,lon,last_seen,first_seen,advert_count,battery_mv,temperature_c,foreign_advert,default_scope,configured_scope,configured_scope_at,multibyte_sup,multibyte_evidence) SELECT public_key,name,role,lat,lon,last_seen,first_seen,advert_count,battery_mv,temperature_c,foreign_advert,default_scope,configured_scope,configured_scope_at,multibyte_sup,multibyte_evidence FROM nodes WHERE last_seen < `+s.parameter(1)+` ON CONFLICT(public_key) DO UPDATE SET name=excluded.name,role=excluded.role,lat=excluded.lat,lon=excluded.lon,last_seen=excluded.last_seen,first_seen=excluded.first_seen,advert_count=excluded.advert_count,battery_mv=excluded.battery_mv,temperature_c=excluded.temperature_c,foreign_advert=excluded.foreign_advert,default_scope=excluded.default_scope,configured_scope=excluded.configured_scope,configured_scope_at=excluded.configured_scope_at,multibyte_sup=excluded.multibyte_sup,multibyte_evidence=excluded.multibyte_evidence`, cutoff) if err != nil { return 0, fmt.Errorf("insert inactive: %w", err) } - result, err := tx.Exec(`DELETE FROM nodes WHERE last_seen < ?`, cutoff) + result, err := tx.Exec(`DELETE FROM nodes WHERE last_seen < `+s.parameter(1), cutoff) if err != nil { return 0, fmt.Errorf("delete stale: %w", err) } @@ -2165,7 +1278,7 @@ func (s *Store) RemoveStaleObservers(observerDays int) (int64, error) { } cutoff := time.Now().UTC().AddDate(0, 0, -observerDays).Format(time.RFC3339) // Tagged for /api/perf writer-lock visibility (#1340). - result, err := s.instrumentedExec("prune_observers", `UPDATE observers SET inactive = 1 WHERE last_seen < ? AND (inactive IS NULL OR inactive = 0)`, cutoff) + result, err := s.instrumentedExec("prune_observers", `UPDATE observers SET inactive = 1 WHERE last_seen < `+s.parameter(1)+` AND (inactive IS NULL OR inactive = 0)`, cutoff) if err != nil { return 0, fmt.Errorf("mark stale observers inactive: %w", err) } @@ -2200,7 +1313,7 @@ func (s *Store) PurgeStaleObservers(purgeDays int) (int64, error) { result, err := s.instrumentedExec("purge_observers", ` DELETE FROM observers WHERE inactive = 1 - AND last_seen < ? + AND last_seen < `+s.parameter(1)+` AND NOT EXISTS (SELECT 1 FROM observations o WHERE o.observer_idx = observers.rowid) AND NOT EXISTS (SELECT 1 FROM observer_metrics m WHERE m.observer_id = observers.id) AND NOT EXISTS (SELECT 1 FROM dropped_packets d WHERE d.observer_id = observers.id)`, cutoff) @@ -2227,9 +1340,13 @@ type DroppedPacket struct { // InsertDroppedPacket records a rejected packet in the dropped_packets table. func (s *Store) InsertDroppedPacket(dp *DroppedPacket) error { + // The decoder's byte cap can split a UTF-8 rune. Drop incomplete bytes + // without expanding the name past that cap; raw_hex retains the evidence. + nodeName := strings.ToValidUTF8(dp.NodeName, "") + observerName := clampObserverField(dp.ObserverName, maxObserverTextLen) _, err := s.db.Exec( - `INSERT INTO dropped_packets (hash, raw_hex, reason, observer_id, observer_name, node_pubkey, node_name) VALUES (?, ?, ?, ?, ?, ?, ?)`, - dp.Hash, dp.RawHex, dp.Reason, dp.ObserverID, dp.ObserverName, dp.NodePubKey, dp.NodeName, + `INSERT INTO dropped_packets (hash, raw_hex, reason, observer_id, observer_name, node_pubkey, node_name) VALUES (`+s.parameter(1)+`, `+s.parameter(2)+`, `+s.parameter(3)+`, `+s.parameter(4)+`, `+s.parameter(5)+`, `+s.parameter(6)+`, `+s.parameter(7)+`)`, + dp.Hash, dp.RawHex, dp.Reason, dp.ObserverID, observerName, dp.NodePubKey, nodeName, ) if err != nil { s.Stats.WriteErrors.Add(1) @@ -2245,7 +1362,7 @@ func (s *Store) PruneDroppedPackets(retentionDays int) (int64, error) { return 0, nil } cutoff := time.Now().UTC().AddDate(0, 0, -retentionDays).Format(time.RFC3339) - result, err := s.db.Exec(`DELETE FROM dropped_packets WHERE dropped_at < ?`, cutoff) + result, err := s.db.Exec(`DELETE FROM dropped_packets WHERE dropped_at < `+s.parameter(1), cutoff) if err != nil { return 0, fmt.Errorf("prune dropped packets: %w", err) } @@ -2290,7 +1407,7 @@ func nilIfEmpty(s string) interface{} { return s } -// boolToInt converts a bool to SQLite's 0/1 INTEGER representation. +// boolToInt converts a bool to the retained 0/1 integer representation. func boolToInt(b bool) int { if b { return 1 @@ -2324,15 +1441,15 @@ func (s *Store) UpdateNodeDefaultScope(pubkey, scope string) error { } // Short-circuit: skip if already stored. var cur sql.NullString - row := s.db.QueryRow(`SELECT default_scope FROM nodes WHERE public_key = ?`, pubkey) + row := s.db.QueryRow(`SELECT default_scope FROM nodes WHERE public_key = `+s.parameter(1), pubkey) if row.Scan(&cur) == nil && cur.Valid && cur.String == scope { return nil } - if _, err := s.db.Exec(`UPDATE nodes SET default_scope = ? WHERE public_key = ?`, scope, pubkey); err != nil { + if _, err := s.db.Exec(`UPDATE nodes SET default_scope = `+s.parameter(1)+` WHERE public_key = `+s.parameter(2), scope, pubkey); err != nil { return err } // Mirror to inactive_nodes (node may be there if recently moved by retention). - _, err := s.db.Exec(`UPDATE inactive_nodes SET default_scope = ? WHERE public_key = ?`, scope, pubkey) + _, err := s.db.Exec(`UPDATE inactive_nodes SET default_scope = `+s.parameter(1)+` WHERE public_key = `+s.parameter(2), scope, pubkey) return err } @@ -2431,19 +1548,19 @@ func (s *Store) UpdateNodeConfiguredScope(pubkey, scope, reportedAt string) erro // Last-write-wins: skip if the stored confirmation is newer-or-equal. if reportedAt != "" { var curAt sql.NullString - row := s.db.QueryRow(`SELECT configured_scope_at FROM nodes WHERE public_key = ?`, pubkey) + row := s.db.QueryRow(`SELECT configured_scope_at FROM nodes WHERE public_key = `+s.parameter(1), pubkey) if row.Scan(&curAt) == nil && curAt.Valid && curAt.String != "" && curAt.String >= reportedAt { return nil } } if _, err := s.db.Exec( - `UPDATE nodes SET configured_scope = ?, configured_scope_at = ? WHERE public_key = ?`, + `UPDATE nodes SET configured_scope = `+s.parameter(1)+`, configured_scope_at = `+s.parameter(2)+` WHERE public_key = `+s.parameter(3), scope, reportedAt, pubkey); err != nil { return err } // Mirror to inactive_nodes (node may be there if recently moved by retention). _, err := s.db.Exec( - `UPDATE inactive_nodes SET configured_scope = ?, configured_scope_at = ? WHERE public_key = ?`, + `UPDATE inactive_nodes SET configured_scope = `+s.parameter(1)+`, configured_scope_at = `+s.parameter(2)+` WHERE public_key = `+s.parameter(3), scope, reportedAt, pubkey) return err } @@ -2468,14 +1585,14 @@ func (s *Store) RecordNaiveSkew(observerID string, deltaSec int64, now time.Time // increments it. _, err := s.db.Exec(` INSERT INTO observers (id, clock_skew_seconds, clock_skew_count_24h, clock_last_naive_at) - VALUES (?, ?, 1, ?) + VALUES (`+s.parameter(1)+`, `+s.parameter(2)+`, 1, `+s.parameter(3)+`) ON CONFLICT(id) DO UPDATE SET clock_skew_seconds = excluded.clock_skew_seconds, clock_last_naive_at = excluded.clock_last_naive_at, clock_skew_count_24h = CASE - WHEN clock_last_naive_at IS NULL OR clock_last_naive_at < ? + WHEN observers.clock_last_naive_at IS NULL OR observers.clock_last_naive_at < `+s.parameter(4)+` THEN 1 - ELSE COALESCE(clock_skew_count_24h, 0) + 1 + ELSE COALESCE(observers.clock_skew_count_24h, 0) + 1 END `, observerID, deltaSec, nowStr, cutoff) return err @@ -2577,7 +1694,7 @@ func BuildPacketData(msg *MQTTPacketMessage, decoded *DecodedPacket, observerID, // ─── Writer-lock instrumentation (issue #1340) ──────────────────────────── // -// Make SQLite writer-lock starvation visible to operators. Per-component +// Make serialized writer-lock starvation visible to operators. Per-component // wait_ms / hold_ms / contention_total histograms, surfaced via // /api/perf/write-sources under the "writer_perf" key. Component tags: // neighbor_builder, mqtt_handler, prune_packets, prune_observers, @@ -2589,7 +1706,7 @@ func BuildPacketData(msg *MQTTPacketMessage, decoded *DecodedPacket, observerID, // every wrapped call site through the same package-level mutex. // WriterStatsSnapshot is a per-component wait/hold latency snapshot -// surfaced via /api/perf to make SQLite writer-lock starvation visible +// surfaced via /api/perf to make serialized writer-lock starvation visible // to operators (issue #1340). Times are in milliseconds. type WriterStatsSnapshot struct { Count int64 `json:"count"` @@ -2805,7 +1922,7 @@ func recordWriterTiming(component string, wait, hold time.Duration, queryForLog // writerMu serialises every wrapped writer call so the wait the next // caller sees is the wait the perf snapshot can attribute. The -// SQLite driver also enforces serial writes (SetMaxOpenConns(1)), +// connection pool also serializes writes (SetMaxOpenConns(1)), // but the wait inside the driver is invisible to Go — writerMu makes // it Go-visible. var writerMu sync.Mutex @@ -2831,7 +1948,7 @@ func (s *Store) WriterTx(component string, fn func(*sql.Tx) error) error { writerMu.Lock() wait := time.Since(waitStart) holdStart := time.Now() - tx, err := s.db.Begin() + tx, err := beginWrite(s.db, s.Backend()) if err != nil { hold := time.Since(holdStart) writerMu.Unlock() diff --git a/cmd/ingestor/db_test.go b/cmd/ingestor/db_test.go index 9cd4fbfef..5d0ef086b 100644 --- a/cmd/ingestor/db_test.go +++ b/cmd/ingestor/db_test.go @@ -13,6 +13,7 @@ import ( "time" "github.com/meshcore-analyzer/packetpath" + "github.com/meshcore-analyzer/pgutil" ) func tempDBPath(t *testing.T) string { @@ -33,14 +34,14 @@ func tempDBPath(t *testing.T) string { } func TestOpenStore(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } defer s.Close() // Verify tables exist - rows, err := s.db.Query("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name") + rows, err := s.db.Query(testNativeSQL(`SELECT name FROM sqlite_master WHERE type='table' ORDER BY name`, "SELECT table_name FROM information_schema.tables WHERE table_schema=current_schema() ORDER BY table_name")) if err != nil { t.Fatal(err) } @@ -69,7 +70,7 @@ func TestOpenStore(t *testing.T) { // Verify packets_v view exists var viewCount int - err = s.db.QueryRow("SELECT COUNT(*) FROM sqlite_master WHERE type='view' AND name='packets_v'").Scan(&viewCount) + err = s.db.QueryRow(testNativeSQL(`SELECT COUNT(*) FROM sqlite_master WHERE type='view' AND name='packets_v'`, "SELECT COUNT(*) FROM information_schema.views WHERE table_schema=current_schema() AND table_name='packets_v'")).Scan(&viewCount) if err != nil { t.Fatal(err) } @@ -79,7 +80,7 @@ func TestOpenStore(t *testing.T) { } func TestInsertTransmission(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -129,7 +130,7 @@ func TestInsertTransmission(t *testing.T) { } func TestPacketsViewQueryable(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -177,7 +178,7 @@ func TestPacketsViewQueryable(t *testing.T) { } func TestUpsertNode(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -216,7 +217,7 @@ func TestUpsertNode(t *testing.T) { } func TestUpsertObserver(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -237,7 +238,7 @@ func TestUpsertObserver(t *testing.T) { } func TestUpsertObserverWithMeta(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -298,21 +299,21 @@ func TestUpsertObserverWithMeta(t *testing.T) { // Verify typeof returns correct SQLite types var typBattery, typUptime, typNoise string - s.db.QueryRow("SELECT typeof(battery_mv), typeof(uptime_secs), typeof(noise_floor) FROM observers WHERE id = 'obs1'"). + s.db.QueryRow(testNativeSQL(`SELECT typeof(battery_mv), typeof(uptime_secs), typeof(noise_floor) FROM observers WHERE id = 'obs1'`, "SELECT pg_typeof(battery_mv)::text, pg_typeof(uptime_secs)::text, pg_typeof(noise_floor)::text FROM observers WHERE id = 'obs1'")). Scan(&typBattery, &typUptime, &typNoise) - if typBattery != "integer" { + if typBattery != testNativeSQL("integer", "bigint") { t.Errorf("typeof(battery_mv)=%s, want integer", typBattery) } - if typUptime != "integer" { + if typUptime != testNativeSQL("integer", "bigint") { t.Errorf("typeof(uptime_secs)=%s, want integer", typUptime) } - if typNoise != "real" { + if typNoise != testNativeSQL("real", "double precision") { t.Errorf("typeof(noise_floor)=%s, want real", typNoise) } } func TestUpsertObserverMetaPreservesExisting(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -453,14 +454,14 @@ func TestExtractObserverMeta(t *testing.T) { } func TestSchemaNoiseFloorIsReal(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } defer s.Close() // Check column type affinity via PRAGMA - rows, err := s.db.Query("PRAGMA table_info(observers)") + rows, err := s.db.Query(testNativeSQL(`SELECT cid,name,type,"notnull",dflt_value,pk FROM pragma_table_info('observers') ORDER BY cid`, `SELECT ordinal_position::int,column_name,CASE data_type WHEN 'bigint' THEN 'INTEGER' WHEN 'double precision' THEN 'REAL' ELSE upper(data_type) END,CASE is_nullable WHEN 'NO' THEN 1 ELSE 0 END,column_default,0 FROM information_schema.columns WHERE table_schema=current_schema() AND table_name='observers' ORDER BY ordinal_position`)) if err != nil { t.Fatal(err) } @@ -486,7 +487,7 @@ func TestSchemaNoiseFloorIsReal(t *testing.T) { } func TestInsertTransmissionWithObserver(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -522,7 +523,7 @@ func TestInsertTransmissionWithObserver(t *testing.T) { // #463: Verify that inserting a packet updates the observer's last_seen, // so low-traffic observers don't incorrectly appear offline. func TestInsertTransmissionUpdatesObserverLastSeen(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -534,11 +535,11 @@ func TestInsertTransmissionUpdatesObserverLastSeen(t *testing.T) { } // Backdate last_seen to 2 hours ago oldTime := "2026-03-24T22:00:00Z" - s.db.Exec("UPDATE observers SET last_seen = ? WHERE id = ?", oldTime, "obs1") + s.db.Exec("UPDATE observers SET last_seen = $1 WHERE id = $2", oldTime, "obs1") // Verify it was backdated var lastSeenBefore string - s.db.QueryRow("SELECT last_seen FROM observers WHERE id = ?", "obs1").Scan(&lastSeenBefore) + s.db.QueryRow("SELECT last_seen FROM observers WHERE id = $1", "obs1").Scan(&lastSeenBefore) if lastSeenBefore != oldTime { t.Fatalf("expected last_seen=%s, got %s", oldTime, lastSeenBefore) } @@ -562,7 +563,7 @@ func TestInsertTransmissionUpdatesObserverLastSeen(t *testing.T) { // Verify last_seen was updated to INGEST time, not envelope time (#1465). var lastSeenAfter string - s.db.QueryRow("SELECT last_seen FROM observers WHERE id = ?", "obs1").Scan(&lastSeenAfter) + s.db.QueryRow("SELECT last_seen FROM observers WHERE id = $1", "obs1").Scan(&lastSeenAfter) if lastSeenAfter == oldTime { t.Error("observer last_seen was NOT updated after packet insertion — low-traffic observers will appear offline") } @@ -578,7 +579,7 @@ func TestInsertTransmissionUpdatesObserverLastSeen(t *testing.T) { } func TestLastPacketAtUpdatedOnPacketOnly(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -590,7 +591,7 @@ func TestLastPacketAtUpdatedOnPacketOnly(t *testing.T) { } var lastPacketAt sql.NullString - s.db.QueryRow("SELECT last_packet_at FROM observers WHERE id = ?", "obs1").Scan(&lastPacketAt) + s.db.QueryRow("SELECT last_packet_at FROM observers WHERE id = $1", "obs1").Scan(&lastPacketAt) if lastPacketAt.Valid { t.Fatalf("expected last_packet_at to be NULL after UpsertObserver, got %s", lastPacketAt.String) } @@ -612,7 +613,7 @@ func TestLastPacketAtUpdatedOnPacketOnly(t *testing.T) { } after := time.Now().Unix() - s.db.QueryRow("SELECT last_packet_at FROM observers WHERE id = ?", "obs1").Scan(&lastPacketAt) + s.db.QueryRow("SELECT last_packet_at FROM observers WHERE id = $1", "obs1").Scan(&lastPacketAt) if !lastPacketAt.Valid { t.Fatal("expected last_packet_at to be non-NULL after InsertTransmission") } @@ -634,14 +635,14 @@ func TestLastPacketAtUpdatedOnPacketOnly(t *testing.T) { } var lastPacketAtAfterStatus sql.NullString - s.db.QueryRow("SELECT last_packet_at FROM observers WHERE id = ?", "obs1").Scan(&lastPacketAtAfterStatus) + s.db.QueryRow("SELECT last_packet_at FROM observers WHERE id = $1", "obs1").Scan(&lastPacketAtAfterStatus) if !lastPacketAtAfterStatus.Valid || lastPacketAtAfterStatus.String != lastPacketAt.String { t.Errorf("UpsertObserver should not change last_packet_at; expected %s, got %v", lastPacketAt.String, lastPacketAtAfterStatus) } } func TestEndToEndIngest(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -677,7 +678,7 @@ func TestEndToEndIngest(t *testing.T) { // Verify node was created var nodeName string - err = s.db.QueryRow("SELECT name FROM nodes WHERE public_key = ?", decoded.Payload.PubKey).Scan(&nodeName) + err = s.db.QueryRow("SELECT name FROM nodes WHERE public_key = $1", decoded.Payload.PubKey).Scan(&nodeName) if err != nil { t.Fatal(err) } @@ -687,7 +688,7 @@ func TestEndToEndIngest(t *testing.T) { } func TestInsertTransmissionEmptyHash(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -711,7 +712,7 @@ func TestInsertTransmissionEmptyHash(t *testing.T) { } func TestInsertTransmissionEmptyTimestamp(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -729,14 +730,14 @@ func TestInsertTransmissionEmptyTimestamp(t *testing.T) { } var firstSeen string - s.db.QueryRow("SELECT first_seen FROM transmissions WHERE hash = ?", data.Hash).Scan(&firstSeen) + s.db.QueryRow("SELECT first_seen FROM transmissions WHERE hash = $1", data.Hash).Scan(&firstSeen) if firstSeen == "" { t.Error("first_seen should be set even with empty timestamp") } } func TestInsertTransmissionEarlierFirstSeen(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -765,14 +766,14 @@ func TestInsertTransmissionEarlierFirstSeen(t *testing.T) { } var firstSeen string - s.db.QueryRow("SELECT first_seen FROM transmissions WHERE hash = ?", data.Hash).Scan(&firstSeen) + s.db.QueryRow("SELECT first_seen FROM transmissions WHERE hash = $1", data.Hash).Scan(&firstSeen) if firstSeen != "2026-03-25T06:00:00Z" { t.Errorf("first_seen=%s, want 2026-03-25T06:00:00Z (earlier timestamp)", firstSeen) } } func TestInsertTransmissionLaterFirstSeenNotUpdated(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -800,14 +801,14 @@ func TestInsertTransmissionLaterFirstSeenNotUpdated(t *testing.T) { } var firstSeen string - s.db.QueryRow("SELECT first_seen FROM transmissions WHERE hash = ?", data.Hash).Scan(&firstSeen) + s.db.QueryRow("SELECT first_seen FROM transmissions WHERE hash = $1", data.Hash).Scan(&firstSeen) if firstSeen != "2026-03-25T06:00:00Z" { t.Errorf("first_seen=%s should not change to later time", firstSeen) } } func TestInsertTransmissionNilSNRRSSI(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -926,7 +927,7 @@ func TestBuildPacketDataNilSNRRSSI(t *testing.T) { } func TestUpsertNodeEmptyLastSeen(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -949,14 +950,14 @@ func TestUpsertNodeEmptyLastSeen(t *testing.T) { func TestOpenStoreTwice(t *testing.T) { // Opening same DB twice tests the "observations already exists" path in applySchema path := tempDBPath(t) - s1, err := OpenStore(path) + s1, err := openPostgresTestStore(t, path) if err != nil { t.Fatal(err) } s1.Close() // Second open — observations table already exists - s2, err := OpenStore(path) + s2, err := openPostgresTestStore(t, path) if err != nil { t.Fatal(err) } @@ -971,7 +972,7 @@ func TestOpenStoreTwice(t *testing.T) { } func TestInsertTransmissionDedupObservation(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -1005,7 +1006,7 @@ func TestInsertTransmissionDedupObservation(t *testing.T) { } func TestSchemaCompatibility(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -1013,7 +1014,7 @@ func TestSchemaCompatibility(t *testing.T) { // Verify column names match what Node.js expects expectedTxCols := []string{"id", "raw_hex", "hash", "first_seen", "route_type", "payload_type", "payload_version", "decoded_json", "created_at"} - rows, _ := s.db.Query("PRAGMA table_info(transmissions)") + rows, _ := s.db.Query(testNativeSQL(`SELECT cid,name,type,"notnull",dflt_value,pk FROM pragma_table_info('transmissions') ORDER BY cid`, `SELECT ordinal_position::int,column_name,CASE data_type WHEN 'bigint' THEN 'INTEGER' WHEN 'double precision' THEN 'REAL' ELSE upper(data_type) END,CASE is_nullable WHEN 'NO' THEN 1 ELSE 0 END,column_default,0 FROM information_schema.columns WHERE table_schema=current_schema() AND table_name='transmissions' ORDER BY ordinal_position`)) var txCols []string for rows.Next() { var cid int @@ -1041,7 +1042,7 @@ func TestSchemaCompatibility(t *testing.T) { // Verify observations columns expectedObsCols := []string{"id", "transmission_id", "observer_idx", "direction", "snr", "rssi", "score", "path_json", "timestamp"} - rows, _ = s.db.Query("PRAGMA table_info(observations)") + rows, _ = s.db.Query(testNativeSQL(`SELECT cid,name,type,"notnull",dflt_value,pk FROM pragma_table_info('observations') ORDER BY cid`, `SELECT ordinal_position::int,column_name,CASE data_type WHEN 'bigint' THEN 'INTEGER' WHEN 'double precision' THEN 'REAL' ELSE upper(data_type) END,CASE is_nullable WHEN 'NO' THEN 1 ELSE 0 END,column_default,0 FROM information_schema.columns WHERE table_schema=current_schema() AND table_name='observations' ORDER BY ordinal_position`)) var obsCols []string for rows.Next() { var cid int @@ -1069,7 +1070,7 @@ func TestSchemaCompatibility(t *testing.T) { } func TestConcurrentWrites(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -1167,7 +1168,7 @@ func TestConcurrentWrites(t *testing.T) { } func TestDBStats(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -1231,7 +1232,7 @@ func TestDBStats(t *testing.T) { } func TestLoadTestThroughput(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -1372,7 +1373,7 @@ func TestLoadTestThroughput(t *testing.T) { } func TestUpdateNodeTelemetry(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -1420,7 +1421,7 @@ func TestUpdateNodeTelemetry(t *testing.T) { } func TestTelemetryMigrationAddsColumns(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -1437,9 +1438,9 @@ func TestTelemetryMigrationAddsColumns(t *testing.T) { } var count int - s.db.QueryRow("SELECT COUNT(*) FROM _migrations WHERE name = 'node_telemetry_v1'").Scan(&count) + s.db.QueryRow(testNativeSQL(`SELECT COUNT(*) FROM _migrations WHERE name='observers_identity_autoincrement_v1'`, "SELECT COUNT(*) FROM corescope_schema WHERE kind='telemetry' AND ready=true")).Scan(&count) if count != 1 { - t.Errorf("migration node_telemetry_v1 should be recorded, count=%d", count) + t.Errorf("canonical telemetry schema should be ready, count=%d", count) } } @@ -1591,7 +1592,7 @@ func TestObsTimestampIndexMigration(t *testing.T) { // Case 1: new DB — OpenStore should create idx_observations_timestamp as part // of the observations table schema. t.Run("NewDB", func(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -1599,7 +1600,7 @@ func TestObsTimestampIndexMigration(t *testing.T) { var count int err = s.db.QueryRow( - "SELECT COUNT(*) FROM sqlite_master WHERE type='index' AND name='idx_observations_timestamp'", + testNativeSQL(`SELECT COUNT(*) FROM sqlite_master WHERE type='index' AND name='idx_observations_timestamp'`, "SELECT COUNT(*) FROM pg_indexes WHERE schemaname=current_schema() AND indexname='idx_observations_timestamp'"), ).Scan(&count) if err != nil { t.Fatal(err) @@ -1622,83 +1623,35 @@ func TestObsTimestampIndexMigration(t *testing.T) { // Case 2: existing DB that has the observations table but lacks the index // and lacks the _migrations entry — simulates an older installation. - t.Run("MigrationPath", func(t *testing.T) { - path := tempDBPath(t) - - // Build a bare-bones DB that mimics an old installation: - // observations table exists but idx_observations_timestamp does NOT. - db, err := sql.Open("sqlite3", path) - if err != nil { - t.Fatal(err) - } - _, err = db.Exec(` - CREATE TABLE IF NOT EXISTS _migrations (name TEXT PRIMARY KEY); - CREATE TABLE IF NOT EXISTS transmissions ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - raw_hex TEXT NOT NULL, - hash TEXT NOT NULL UNIQUE, - first_seen TEXT NOT NULL, - route_type INTEGER, - payload_type INTEGER, - payload_version INTEGER, - decoded_json TEXT, - created_at TEXT DEFAULT (datetime('now')) - ); - CREATE TABLE IF NOT EXISTS observations ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - transmission_id INTEGER NOT NULL REFERENCES transmissions(id), - observer_idx INTEGER, - direction TEXT, - snr REAL, - rssi REAL, - score INTEGER, - path_json TEXT, - timestamp INTEGER NOT NULL - ); - `) - if err != nil { - db.Close() - t.Fatal(err) - } - // Confirm the index is absent before OpenStore runs. - var preCount int - db.QueryRow( - "SELECT COUNT(*) FROM sqlite_master WHERE type='index' AND name='idx_observations_timestamp'", - ).Scan(&preCount) - db.Close() - if preCount != 0 { - t.Fatalf("pre-condition failed: idx_observations_timestamp should not exist yet, got count=%d", preCount) - } - - // Now open via OpenStore — the migration should add the index. - s, err := OpenStore(path) + t.Run("MissingIndexIsRefused", func(t *testing.T) { + key := tempDBPath(t) + store, err := openPostgresTestStore(t, key) if err != nil { t.Fatal(err) } - defer s.Close() - - var idxCount int - err = s.db.QueryRow( - "SELECT COUNT(*) FROM sqlite_master WHERE type='index' AND name='idx_observations_timestamp'", - ).Scan(&idxCount) - if err != nil { + owner := testAdmin(t, store) + if _, err := owner.Exec(`DROP INDEX idx_observations_timestamp`); err != nil { t.Fatal(err) } - if idxCount != 1 { - t.Error("idx_observations_timestamp should exist after migration on old DB") - } - - var migCount int - err = s.db.QueryRow( - "SELECT COUNT(*) FROM _migrations WHERE name='obs_timestamp_index_v1'", - ).Scan(&migCount) - if err != nil { - t.Fatal(err) + store.Close() + if store.Backend() == "sqlite" { + next, err := openPostgresTestStore(t, key) + if err != nil { + t.Fatal(err) + } + defer next.Close() + var count int + if err = next.db.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE type='index' AND name='idx_observations_timestamp'`).Scan(&count); err != nil || count != 1 { + t.Fatalf("SQLite writer did not restore its index: %d %v", count, err) + } + return } - if migCount != 1 { - t.Errorf("migration obs_timestamp_index_v1 should be recorded, got count=%d", migCount) + if next, err := openPostgresTestStore(t, key); err == nil { + next.Close() + t.Fatal("runtime accepted missing canonical index") } }) + } func TestBuildPacketDataScoreAndDirection(t *testing.T) { @@ -1739,7 +1692,7 @@ func TestBuildPacketDataNilScoreDirection(t *testing.T) { } func TestInsertTransmissionWithScoreAndDirection(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -1805,7 +1758,7 @@ func TestRoundToInterval(t *testing.T) { } func TestInsertMetrics(t *testing.T) { - store, err := OpenStore(tempDBPath(t)) + store, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -1849,7 +1802,7 @@ func TestInsertMetrics(t *testing.T) { } func TestInsertMetricsIdempotent(t *testing.T) { - store, err := OpenStoreWithInterval(tempDBPath(t), 300) + store, err := openPostgresTestStoreInterval(t, tempDBPath(t), 300) if err != nil { t.Fatal(err) } @@ -1879,7 +1832,7 @@ func TestInsertMetricsIdempotent(t *testing.T) { } func TestInsertMetricsNullFields(t *testing.T) { - store, err := OpenStore(tempDBPath(t)) + store, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -1908,7 +1861,7 @@ func TestInsertMetricsNullFields(t *testing.T) { } func TestPruneOldMetrics(t *testing.T) { - store, err := OpenStore(tempDBPath(t)) + store, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -1917,8 +1870,8 @@ func TestPruneOldMetrics(t *testing.T) { // Insert old and new metrics directly oldTs := time.Now().UTC().AddDate(0, 0, -40).Format(time.RFC3339) newTs := time.Now().UTC().Format(time.RFC3339) - store.db.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor) VALUES (?, ?, ?)", "obs1", oldTs, -110.0) - store.db.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor) VALUES (?, ?, ?)", "obs1", newTs, -112.0) + store.db.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor) VALUES ($1, $2, $3)", "obs1", oldTs, -110.0) + store.db.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor) VALUES ($1, $2, $3)", "obs1", newTs, -112.0) n, err := store.PruneOldMetrics(30) if err != nil { @@ -1975,7 +1928,7 @@ func TestExtractObserverMetaNewFields(t *testing.T) { // idx_observations_dedup CREATE UNIQUE INDEX statements elsewhere in this file.) func TestInsertObservationSNRFillIn(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -2058,7 +2011,7 @@ func TestInsertObservationSNRFillIn(t *testing.T) { // TestPerObservationRawHex verifies that two MQTT packets for the same hash // from different observers store distinct raw_hex per observation (#881). func TestPerObservationRawHex(t *testing.T) { - store, err := OpenStore(tempDBPath(t)) + store, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -2210,14 +2163,14 @@ func TestBuildPacketData_NonTracePathJSON(t *testing.T) { func TestScopeNameMigration(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "test.db") - store, err := OpenStore(dbPath) + store, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore: %v", err) } defer store.Close() // Verify column exists - rows, err := store.db.Query("PRAGMA table_info(transmissions)") + rows, err := store.db.Query(testNativeSQL(`SELECT cid,name,type,"notnull",dflt_value,pk FROM pragma_table_info('transmissions') ORDER BY cid`, `SELECT ordinal_position::int,column_name,CASE data_type WHEN 'bigint' THEN 'INTEGER' WHEN 'double precision' THEN 'REAL' ELSE upper(data_type) END,CASE is_nullable WHEN 'NO' THEN 1 ELSE 0 END,column_default,0 FROM information_schema.columns WHERE table_schema=current_schema() AND table_name='transmissions' ORDER BY ordinal_position`)) if err != nil { t.Fatalf("PRAGMA: %v", err) } @@ -2266,7 +2219,7 @@ func TestScopeNameMigration(t *testing.T) { } func TestTransportCodesStored(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -2307,7 +2260,7 @@ func TestTransportCodesStored(t *testing.T) { } var c1, c2 *string - if err := s.db.QueryRow(`SELECT code1, code2 FROM transmissions WHERE hash = ?`, + if err := s.db.QueryRow(`SELECT code1, code2 FROM transmissions WHERE hash = $1`, ComputeContentHash(rawScoped)).Scan(&c1, &c2); err != nil { t.Fatalf("read scoped: %v", err) } @@ -2315,7 +2268,7 @@ func TestTransportCodesStored(t *testing.T) { t.Errorf("scoped codes = %v/%v, want AABB/CCDD", c1, c2) } - if err := s.db.QueryRow(`SELECT code1, code2 FROM transmissions WHERE hash = ?`, + if err := s.db.QueryRow(`SELECT code1, code2 FROM transmissions WHERE hash = $1`, ComputeContentHash(rawUnscoped)).Scan(&c1, &c2); err != nil { t.Fatalf("read unscoped: %v", err) } @@ -2323,7 +2276,7 @@ func TestTransportCodesStored(t *testing.T) { t.Errorf("unscoped codes = %v/%v, want NULL/NULL", c1, c2) } - if err := s.db.QueryRow(`SELECT code1, code2 FROM transmissions WHERE hash = ?`, + if err := s.db.QueryRow(`SELECT code1, code2 FROM transmissions WHERE hash = $1`, ComputeContentHash(rawZeroCodes)).Scan(&c1, &c2); err != nil { t.Fatalf("read zero-codes: %v", err) } @@ -2349,7 +2302,7 @@ func TestBackfillTransportCodes(t *testing.T) { }{{rawScoped, 0}, {rawFlood, 1}} { if _, err := s.db.Exec( `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type) - VALUES (?,?,?,?,?)`, + VALUES ($1,$2,$3,$4,$5)`, tc.raw, ComputeContentHash(tc.raw), "2026-08-01T00:00:00Z", tc.routeType, 5); err != nil { t.Fatalf("seed: %v", err) } @@ -2359,7 +2312,7 @@ func TestBackfillTransportCodes(t *testing.T) { s.backfillWg.Wait() var c1 *string - if err := s.db.QueryRow(`SELECT code1 FROM transmissions WHERE hash = ?`, + if err := s.db.QueryRow(`SELECT code1 FROM transmissions WHERE hash = $1`, ComputeContentHash(rawScoped)).Scan(&c1); err != nil { t.Fatalf("read: %v", err) } @@ -2367,7 +2320,7 @@ func TestBackfillTransportCodes(t *testing.T) { t.Errorf("backfilled code1 = %v, want AABB", c1) } - if err := s.db.QueryRow(`SELECT code1 FROM transmissions WHERE hash = ?`, + if err := s.db.QueryRow(`SELECT code1 FROM transmissions WHERE hash = $1`, ComputeContentHash(rawFlood)).Scan(&c1); err != nil { t.Fatalf("read flood: %v", err) } @@ -2415,14 +2368,14 @@ func TestBackfillTransportCodes_MultiBatchWithUndecodableRow(t *testing.T) { // good2}, batch 2 = {good3, good4, good5}, batch 3 = {good6}. if _, err := s.db.Exec( `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type) - VALUES (?,?,?,?,?)`, + VALUES ($1,$2,$3,$4,$5)`, badRaw, ComputeContentHash(badRaw), "2026-08-01T00:00:00Z", 0, 5); err != nil { t.Fatalf("seed bad row: %v", err) } for _, raw := range goodRaws { if _, err := s.db.Exec( `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type) - VALUES (?,?,?,?,?)`, + VALUES ($1,$2,$3,$4,$5)`, raw, ComputeContentHash(raw), "2026-08-01T00:00:00Z", 0, 5); err != nil { t.Fatalf("seed good row: %v", err) } @@ -2439,7 +2392,7 @@ func TestBackfillTransportCodes_MultiBatchWithUndecodableRow(t *testing.T) { for i, raw := range goodRaws { wantCode1 := fmt.Sprintf("%04X", i+1) var c1 *string - if err := s.db.QueryRow(`SELECT code1 FROM transmissions WHERE hash = ?`, + if err := s.db.QueryRow(`SELECT code1 FROM transmissions WHERE hash = $1`, ComputeContentHash(raw)).Scan(&c1); err != nil { t.Fatalf("read good row %d: %v", i, err) } @@ -2449,7 +2402,7 @@ func TestBackfillTransportCodes_MultiBatchWithUndecodableRow(t *testing.T) { } var badC1 *string - if err := s.db.QueryRow(`SELECT code1 FROM transmissions WHERE hash = ?`, + if err := s.db.QueryRow(`SELECT code1 FROM transmissions WHERE hash = $1`, ComputeContentHash(badRaw)).Scan(&badC1); err != nil { t.Fatalf("read bad row: %v", err) } @@ -2470,7 +2423,7 @@ func TestBackfillTransportCodes_ErrorWithholdsGuardRow(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "test.db") - store1, err := OpenStore(dbPath) + store1, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore: %v", err) } @@ -2478,7 +2431,7 @@ func TestBackfillTransportCodes_ErrorWithholdsGuardRow(t *testing.T) { raw := "14" + "AABB" + "CCDD" + "00" + strings.Repeat("11", 32) if _, err := store1.db.Exec( `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type) - VALUES (?,?,?,?,?)`, + VALUES ($1,$2,$3,$4,$5)`, raw, ComputeContentHash(raw), "2026-08-01T00:00:00Z", 0, 5); err != nil { t.Fatalf("seed: %v", err) } @@ -2490,7 +2443,7 @@ func TestBackfillTransportCodes_ErrorWithholdsGuardRow(t *testing.T) { t.Fatal("backfillTransportCodes on a closed store returned nil error, want non-nil") } - store2, err := OpenStore(dbPath) + store2, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("reopen: %v", err) } @@ -2508,7 +2461,7 @@ func TestBackfillTransportCodes_ErrorWithholdsGuardRow(t *testing.T) { store2.backfillWg.Wait() var c1 *string - if err := store2.db.QueryRow(`SELECT code1 FROM transmissions WHERE hash = ?`, + if err := store2.db.QueryRow(`SELECT code1 FROM transmissions WHERE hash = $1`, ComputeContentHash(raw)).Scan(&c1); err != nil { t.Fatalf("read: %v", err) } @@ -2526,7 +2479,7 @@ func TestBackfillTransportCodes_ErrorWithholdsGuardRow(t *testing.T) { func TestUpdateNodeDefaultScope(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "test.db") - store, err := OpenStore(dbPath) + store, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore: %v", err) } @@ -2592,7 +2545,7 @@ func TestUpdateNodeDefaultScope(t *testing.T) { func TestBackfillPathJsonFromRawHex(t *testing.T) { dbPath := tempDBPath(t) - s, err := OpenStore(dbPath) + s, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatal(err) } @@ -2601,26 +2554,26 @@ func TestBackfillPathJsonFromRawHex(t *testing.T) { // raw_hex: header 0x05 (route FLOOD, payload 0x01), path byte 0x42 (hash_size=2, count=2), // hops: AABB, CCDD, then some payload bytes rawHex := "0542AABBCCDD0000000000000000000000000000" - s.db.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, payload_type) VALUES (?, 'h1', '2025-01-01T00:00:00Z', 1)`, rawHex) + s.db.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, payload_type) VALUES ($1, 'h1', '2025-01-01T00:00:00Z', 1)`, rawHex) // Insert observation with raw_hex but empty path_json - s.db.Exec(`INSERT INTO observations (transmission_id, timestamp, raw_hex, path_json) VALUES (1, 1000, ?, '[]')`, rawHex) + s.db.Exec(`INSERT INTO observations (transmission_id, timestamp, raw_hex, path_json) VALUES (1, 1000, $1, '[]')`, rawHex) // Insert observation with raw_hex and NULL path_json - s.db.Exec(`INSERT INTO observations (transmission_id, timestamp, raw_hex, path_json) VALUES (1, 1001, ?, NULL)`, rawHex) + s.db.Exec(`INSERT INTO observations (transmission_id, timestamp, raw_hex, path_json) VALUES (1, 1001, $1, NULL)`, rawHex) // Insert observation with existing path_json (should NOT be overwritten) - s.db.Exec(`INSERT INTO observations (transmission_id, timestamp, raw_hex, path_json) VALUES (1, 1002, ?, '["XX","YY"]')`, rawHex) + s.db.Exec(`INSERT INTO observations (transmission_id, timestamp, raw_hex, path_json) VALUES (1, 1002, $1, '["XX","YY"]')`, rawHex) // Insert a TRACE transmission (payload_type = 0x09) — should be skipped traceRaw := "2604302D0D2359FEE7B100000000006733D63367" - s.db.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, payload_type) VALUES (?, 'h2', '2025-01-01T00:00:00Z', 9)`, traceRaw) - s.db.Exec(`INSERT INTO observations (transmission_id, timestamp, raw_hex, path_json) VALUES (2, 1003, ?, '[]')`, traceRaw) + s.db.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, payload_type) VALUES ($1, 'h2', '2025-01-01T00:00:00Z', 9)`, traceRaw) + s.db.Exec(`INSERT INTO observations (transmission_id, timestamp, raw_hex, path_json) VALUES (2, 1003, $1, '[]')`, traceRaw) // Remove the migration marker so it runs again on reopen s.db.Exec(`DELETE FROM _migrations WHERE name = 'backfill_path_json_from_raw_hex_v1'`) s.Close() // Reopen — backfill is now async, must trigger explicitly - s2, err := OpenStore(dbPath) + s2, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatal(err) } @@ -2672,107 +2625,6 @@ func TestBackfillPathJsonFromRawHex(t *testing.T) { } } -func TestCleanupLegacyNullHashTimestamp(t *testing.T) { - path := tempDBPath(t) - - // Create a bare-bones DB with legacy bad data - db, err := sql.Open("sqlite3", path+"?_journal_mode=WAL&_busy_timeout=5000") - if err != nil { - t.Fatal(err) - } - db.Exec(`CREATE TABLE IF NOT EXISTS transmissions ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - raw_hex TEXT NOT NULL, - hash TEXT NOT NULL, - first_seen TEXT NOT NULL, - route_type INTEGER, - payload_type INTEGER, - payload_version INTEGER, - decoded_json TEXT, - created_at TEXT DEFAULT (datetime('now')), - channel_hash TEXT DEFAULT NULL - )`) - db.Exec(`CREATE TABLE IF NOT EXISTS observations ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - transmission_id INTEGER NOT NULL REFERENCES transmissions(id), - observer_idx INTEGER, - direction TEXT, - snr REAL, - rssi REAL, - score INTEGER, - path_json TEXT, - timestamp INTEGER NOT NULL - )`) - db.Exec(`CREATE TABLE IF NOT EXISTS _migrations (name TEXT PRIMARY KEY)`) - db.Exec(`CREATE TABLE IF NOT EXISTS nodes (public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, advert_count INTEGER DEFAULT 0, battery_mv INTEGER, temperature_c REAL)`) - db.Exec(`CREATE TABLE IF NOT EXISTS observers (id TEXT PRIMARY KEY, name TEXT, iata TEXT, last_seen TEXT, first_seen TEXT, packet_count INTEGER DEFAULT 0, model TEXT, firmware TEXT, client_version TEXT, radio TEXT, battery_mv INTEGER, uptime_secs INTEGER, noise_floor REAL, inactive INTEGER DEFAULT 0, last_packet_at TEXT DEFAULT NULL)`) - - // Insert good transmission - db.Exec(`INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (1, 'aabb', 'abc123', '2024-01-01T00:00:00Z')`) - db.Exec(`INSERT INTO observations (transmission_id, observer_idx, timestamp) VALUES (1, 1, 1704067200)`) - - // Insert bad: empty hash - db.Exec(`INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (2, 'ccdd', '', '2024-01-01T00:00:00Z')`) - db.Exec(`INSERT INTO observations (transmission_id, observer_idx, timestamp) VALUES (2, 1, 1704067200)`) - - // Insert bad: empty first_seen - db.Exec(`INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (3, 'eeff', 'def456', '')`) - db.Exec(`INSERT INTO observations (transmission_id, observer_idx, timestamp) VALUES (3, 2, 1704067200)`) - - db.Close() - - // Now open via OpenStore which should run the migration - s, err := OpenStore(path) - if err != nil { - t.Fatal(err) - } - defer s.Close() - - // Good transmission should remain - var count int - s.db.QueryRow("SELECT COUNT(*) FROM transmissions WHERE id = 1").Scan(&count) - if count != 1 { - t.Error("good transmission should not be deleted") - } - - // Bad transmissions should be gone - s.db.QueryRow("SELECT COUNT(*) FROM transmissions WHERE id = 2").Scan(&count) - if count != 0 { - t.Errorf("transmission with empty hash should be deleted, got count=%d", count) - } - s.db.QueryRow("SELECT COUNT(*) FROM transmissions WHERE id = 3").Scan(&count) - if count != 0 { - t.Errorf("transmission with empty first_seen should be deleted, got count=%d", count) - } - - // Observations for bad transmissions should be gone - s.db.QueryRow("SELECT COUNT(*) FROM observations WHERE transmission_id IN (2, 3)").Scan(&count) - if count != 0 { - t.Errorf("observations for bad transmissions should be deleted, got count=%d", count) - } - - // Observation for good transmission should remain - s.db.QueryRow("SELECT COUNT(*) FROM observations WHERE transmission_id = 1").Scan(&count) - if count != 1 { - t.Error("observation for good transmission should remain") - } - - // Migration marker should exist - var migCount int - s.db.QueryRow("SELECT COUNT(*) FROM _migrations WHERE name = 'cleanup_legacy_null_hash_ts'").Scan(&migCount) - if migCount != 1 { - t.Error("migration marker cleanup_legacy_null_hash_ts should be recorded") - } - - // Idempotent: opening again should not error - s.Close() - s2, err := OpenStore(path) - if err != nil { - t.Fatal("second open should not fail:", err) - } - s2.Close() -} - func TestBuildPacketDataRegionFromPayload(t *testing.T) { msg := &MQTTPacketMessage{Raw: "0102030405060708", Region: "PDX"} decoded := &DecodedPacket{ @@ -2806,88 +2658,10 @@ func TestBackfillPathJSONAsync(t *testing.T) { dbPath := filepath.Join(dir, "async_test.db") // Bootstrap schema manually so we can insert test data BEFORE OpenStore - db, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL&_busy_timeout=5000") - if err != nil { - t.Fatal(err) - } - // Create tables manually (minimal schema for this test) - _, err = db.Exec(` - CREATE TABLE _migrations (name TEXT PRIMARY KEY); - CREATE TABLE transmissions ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - raw_hex TEXT NOT NULL, - hash TEXT NOT NULL UNIQUE, - first_seen TEXT NOT NULL, - route_type INTEGER, - payload_type INTEGER, - payload_version INTEGER, - decoded_json TEXT, - created_at TEXT DEFAULT (datetime('now')), - channel_hash TEXT - ); - CREATE TABLE observers ( - id TEXT PRIMARY KEY, - name TEXT, - iata TEXT, - last_seen TEXT, - first_seen TEXT, - packet_count INTEGER DEFAULT 0, - model TEXT, - firmware TEXT, - client_version TEXT, - radio TEXT, - battery_mv INTEGER, - uptime_secs INTEGER, - noise_floor REAL, - inactive INTEGER DEFAULT 0, - last_packet_at TEXT - ); - CREATE TABLE nodes ( - public_key TEXT PRIMARY KEY, - name TEXT, role TEXT, lat REAL, lon REAL, - last_seen TEXT, first_seen TEXT, advert_count INTEGER DEFAULT 0, - battery_mv INTEGER, temperature_c REAL - ); - CREATE TABLE inactive_nodes ( - public_key TEXT PRIMARY KEY, - name TEXT, role TEXT, lat REAL, lon REAL, - last_seen TEXT, first_seen TEXT, advert_count INTEGER DEFAULT 0, - battery_mv INTEGER, temperature_c REAL - ); - CREATE TABLE observations ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - transmission_id INTEGER NOT NULL REFERENCES transmissions(id), - observer_idx INTEGER, - direction TEXT, - snr REAL, rssi REAL, score INTEGER, - path_json TEXT, - timestamp INTEGER NOT NULL, - raw_hex TEXT - ); - CREATE UNIQUE INDEX idx_observations_dedup ON observations(transmission_id, observer_idx, COALESCE(path_json, '')); - CREATE INDEX idx_observations_transmission_id ON observations(transmission_id); - CREATE INDEX idx_observations_observer_idx ON observations(observer_idx); - CREATE INDEX idx_observations_timestamp ON observations(timestamp); - CREATE TABLE observer_metrics ( - observer_id TEXT NOT NULL, - timestamp TEXT NOT NULL, - noise_floor REAL, tx_air_secs INTEGER, rx_air_secs INTEGER, - recv_errors INTEGER, battery_mv INTEGER, - packets_sent INTEGER, packets_recv INTEGER, - PRIMARY KEY (observer_id, timestamp) - ); - CREATE TABLE dropped_packets ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - hash TEXT, raw_hex TEXT, reason TEXT NOT NULL, - observer_id TEXT, observer_name TEXT, - node_pubkey TEXT, node_name TEXT, - dropped_at DATETIME DEFAULT CURRENT_TIMESTAMP - ); - `) + db, err := pgutil.Open(testPostgresURL(t, dbPath), false) if err != nil { - t.Fatal("bootstrap schema:", err) + t.Fatal(err) } - // Mark all migrations as done EXCEPT the path_json backfill for _, m := range []string{ "advert_count_unique_v1", "noise_floor_real_v1", "node_telemetry_v1", @@ -2896,19 +2670,19 @@ func TestBackfillPathJSONAsync(t *testing.T) { "dropped_packets_v1", "observations_raw_hex_v1", "observers_last_packet_at_v1", "cleanup_legacy_null_hash_ts", } { - db.Exec(`INSERT INTO _migrations (name) VALUES (?)`, m) + db.Exec(`INSERT INTO _migrations (name) VALUES ($1)`, m) } // Insert a transmission + observations with NULL path_json and valid raw_hex // raw_hex "0102AABBCCDD0000" has 2-hop path decodable by packetpath rawHex := "41020304AABBCCDD05060708" - _, err = db.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, payload_type) VALUES (?, 'hash1', '2025-01-01T00:00:00Z', 4)`, rawHex) + _, err = db.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, payload_type) VALUES ($1, 'hash1', '2025-01-01T00:00:00Z', 4)`, rawHex) if err != nil { t.Fatal("insert tx:", err) } // Insert 100 observations needing backfill for i := 0; i < 100; i++ { - _, err = db.Exec(`INSERT INTO observations (transmission_id, observer_idx, timestamp, raw_hex, path_json) VALUES (1, ?, ?, ?, NULL)`, + _, err = db.Exec(`INSERT INTO observations (transmission_id, observer_idx, timestamp, raw_hex, path_json) VALUES (1, $1, $2, $3, NULL)`, i+1, 1700000000+i, rawHex) if err != nil { // dedup index might fire — use unique observer_idx @@ -2919,7 +2693,7 @@ func TestBackfillPathJSONAsync(t *testing.T) { // Now open store via OpenStore — this must return QUICKLY (non-blocking) start := time.Now() - store, err := OpenStoreWithInterval(dbPath, 300) + store, err := openPostgresTestStoreInterval(t, dbPath, 300) elapsed := time.Since(start) if err != nil { t.Fatal("OpenStore:", err) @@ -2967,7 +2741,7 @@ func TestBackfillPathJSONAsync(t *testing.T) { func TestBackfillPathJSONAsyncMethodExists(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "method_test.db") - store, err := OpenStoreWithInterval(dbPath, 300) + store, err := openPostgresTestStoreInterval(t, dbPath, 300) if err != nil { t.Fatal(err) } @@ -2995,73 +2769,10 @@ func TestBackfillPathJSONAsync_BracketRowsTerminate(t *testing.T) { // Bootstrap a minimal schema directly so we can seed pre-existing '[]' rows // before OpenStore runs. - db, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL&_busy_timeout=5000") - if err != nil { - t.Fatal(err) - } - _, err = db.Exec(` - CREATE TABLE _migrations (name TEXT PRIMARY KEY); - CREATE TABLE transmissions ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - raw_hex TEXT NOT NULL, - hash TEXT NOT NULL UNIQUE, - first_seen TEXT NOT NULL, - route_type INTEGER, - payload_type INTEGER, - payload_version INTEGER, - decoded_json TEXT, - created_at TEXT DEFAULT (datetime('now')), - channel_hash TEXT - ); - CREATE TABLE observers ( - id TEXT PRIMARY KEY, name TEXT, iata TEXT, - last_seen TEXT, first_seen TEXT, packet_count INTEGER DEFAULT 0, - model TEXT, firmware TEXT, client_version TEXT, radio TEXT, - battery_mv INTEGER, uptime_secs INTEGER, noise_floor REAL, - inactive INTEGER DEFAULT 0, last_packet_at TEXT - ); - CREATE TABLE nodes ( - public_key TEXT PRIMARY KEY, name TEXT, role TEXT, - lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, - advert_count INTEGER DEFAULT 0, battery_mv INTEGER, temperature_c REAL - ); - CREATE TABLE inactive_nodes ( - public_key TEXT PRIMARY KEY, name TEXT, role TEXT, - lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, - advert_count INTEGER DEFAULT 0, battery_mv INTEGER, temperature_c REAL - ); - CREATE TABLE observations ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - transmission_id INTEGER NOT NULL REFERENCES transmissions(id), - observer_idx INTEGER, direction TEXT, - snr REAL, rssi REAL, score INTEGER, - path_json TEXT, - timestamp INTEGER NOT NULL, - raw_hex TEXT - ); - CREATE UNIQUE INDEX idx_observations_dedup ON observations(transmission_id, observer_idx, COALESCE(path_json, '')); - CREATE INDEX idx_observations_transmission_id ON observations(transmission_id); - CREATE INDEX idx_observations_observer_idx ON observations(observer_idx); - CREATE INDEX idx_observations_timestamp ON observations(timestamp); - CREATE TABLE observer_metrics ( - observer_id TEXT NOT NULL, timestamp TEXT NOT NULL, - noise_floor REAL, tx_air_secs INTEGER, rx_air_secs INTEGER, - recv_errors INTEGER, battery_mv INTEGER, - packets_sent INTEGER, packets_recv INTEGER, - PRIMARY KEY (observer_id, timestamp) - ); - CREATE TABLE dropped_packets ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - hash TEXT, raw_hex TEXT, reason TEXT NOT NULL, - observer_id TEXT, observer_name TEXT, - node_pubkey TEXT, node_name TEXT, - dropped_at DATETIME DEFAULT CURRENT_TIMESTAMP - ); - `) + db, err := pgutil.Open(testPostgresURL(t, dbPath), false) if err != nil { - t.Fatal("bootstrap schema:", err) + t.Fatal(err) } - // Mark all migrations done EXCEPT backfill_path_json_from_raw_hex_v1. for _, m := range []string{ "advert_count_unique_v1", "noise_floor_real_v1", "node_telemetry_v1", @@ -3070,20 +2781,20 @@ func TestBackfillPathJSONAsync_BracketRowsTerminate(t *testing.T) { "dropped_packets_v1", "observations_raw_hex_v1", "observers_last_packet_at_v1", "cleanup_legacy_null_hash_ts", } { - db.Exec(`INSERT INTO _migrations (name) VALUES (?)`, m) + db.Exec(`INSERT INTO _migrations (name) VALUES ($1)`, m) } // raw_hex producing ZERO hops via DecodePathFromRawHex: // DIRECT route (type=2), payload_type=2, version=0 → header 0x0A; path byte 0x00. // (See internal/packetpath/path_test.go: TestDecodePathFromRawHex_ZeroHops.) rawHex := "0A00DEADBEEF" - _, err = db.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, payload_type) VALUES (?, 'h_brackets', '2025-01-01T00:00:00Z', 2)`, rawHex) + _, err = db.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, payload_type) VALUES ($1, 'h_brackets', '2025-01-01T00:00:00Z', 2)`, rawHex) if err != nil { t.Fatal("insert tx:", err) } const seedCount = 100 for i := 0; i < seedCount; i++ { - _, err = db.Exec(`INSERT INTO observations (transmission_id, observer_idx, timestamp, raw_hex, path_json) VALUES (1, ?, ?, ?, '[]')`, + _, err = db.Exec(`INSERT INTO observations (transmission_id, observer_idx, timestamp, raw_hex, path_json) VALUES (1, $1, $2, $3, '[]')`, i+1, 1700000000+i, rawHex) if err != nil { t.Fatalf("insert obs %d: %v", i, err) @@ -3091,7 +2802,7 @@ func TestBackfillPathJSONAsync_BracketRowsTerminate(t *testing.T) { } db.Close() - store, err := OpenStoreWithInterval(dbPath, 300) + store, err := openPostgresTestStoreInterval(t, dbPath, 300) if err != nil { t.Fatal("OpenStore:", err) } @@ -3133,14 +2844,14 @@ func TestSchemaMultibyteSupColumns(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "test.db") - store, err := OpenStore(dbPath) + store, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore: %v", err) } defer store.Close() for _, table := range []string{"nodes", "inactive_nodes"} { - rows, err := store.db.Query("PRAGMA table_info(" + table + ")") + rows, err := store.db.Query(testNativeSQL(`SELECT cid,name,type,"notnull",dflt_value,pk FROM pragma_table_info($1) ORDER BY cid`, `SELECT ordinal_position::int,column_name,data_type,CASE is_nullable WHEN 'NO' THEN 1 ELSE 0 END,column_default,0 FROM information_schema.columns WHERE table_schema=current_schema() AND table_name=$1`), table) if err != nil { t.Fatalf("PRAGMA table_info(%s): %v", table, err) } @@ -3173,7 +2884,7 @@ func TestSchemaMultibyteSupColumns(t *testing.T) { // in the legacy _migrations marker table — so we just re-assert the // columns exist and the second OpenStore is a no-op. store.Close() - store2, err := OpenStore(dbPath) + store2, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore (second open): %v", err) } @@ -3189,7 +2900,7 @@ func TestSchemaMultibyteSupColumns(t *testing.T) { func TestUpdateNodeDefaultScope_EmptyScopeIsNoop(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "test.db") - store, err := OpenStore(dbPath) + store, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore: %v", err) } @@ -3243,7 +2954,7 @@ func TestInsertClientRxObservation(t *testing.T) { // a swap is detectable. var gotLat, gotLon, gotSNR, gotPosAccM float64 var gotRSSI int - if err := s.db.QueryRow(`SELECT lat, lon, snr, rssi, pos_acc_m FROM client_rx_observations WHERE pkt_hash = ?`, o.PktHash). + if err := s.db.QueryRow(`SELECT lat, lon, snr, rssi, pos_acc_m FROM client_rx_observations WHERE pkt_hash = $1`, o.PktHash). Scan(&gotLat, &gotLon, &gotSNR, &gotRSSI, &gotPosAccM); err != nil { t.Fatalf("read back geometry: %v", err) } @@ -3273,7 +2984,7 @@ func TestInsertClientRxObservation(t *testing.T) { t.Fatalf("second copy: ins=%v err=%v", ins, err) } var n int - s.db.QueryRow(`SELECT COUNT(*) FROM client_rx_observations WHERE pkt_hash = ?`, o.PktHash).Scan(&n) + s.db.QueryRow(`SELECT COUNT(*) FROM client_rx_observations WHERE pkt_hash = $1`, o.PktHash).Scan(&n) if n != 2 { t.Errorf("rows for pkt_hash = %d, want 2", n) } @@ -3331,7 +3042,7 @@ func TestInsertClientRfSample(t *testing.T) { SELECT lat, lon, pos_acc_m, stationary, uptime_secs, battery_mv, queue_len, errors, noise_floor, last_rssi, last_snr, tx_air_secs, rx_air_secs, recv, sent, flood_rx, direct_rx, flood_tx, direct_tx, recv_errors - FROM client_rf_samples WHERE sampled_at = ?`, o.SampledAt).Scan( + FROM client_rf_samples WHERE sampled_at = $1`, o.SampledAt).Scan( &gotLat, &gotLon, &gotPosAccM, &gotStationary, &gotUptimeSecs, &gotBatteryMV, &gotQueueLen, &gotErrors, &gotNoiseFloor, &gotLastRSSI, &gotLastSNR, &gotTxAirSecs, &gotRxAirSecs, &gotRecv, &gotSent, &gotFloodRx, &gotDirectRx, &gotFloodTx, &gotDirectTx, &gotRecvErrors); err != nil { @@ -3406,7 +3117,7 @@ func TestInsertClientRfSample(t *testing.T) { t.Fatalf("insert second sample: ins=%v err=%v", ins, err) } var gotRecvErrors2 *int64 - if err := s.db.QueryRow(`SELECT recv_errors FROM client_rf_samples WHERE sampled_at = ?`, o2.SampledAt). + if err := s.db.QueryRow(`SELECT recv_errors FROM client_rf_samples WHERE sampled_at = $1`, o2.SampledAt). Scan(&gotRecvErrors2); err != nil { t.Fatalf("read second sample: %v", err) } @@ -3423,16 +3134,20 @@ func TestInsertClientRfSample(t *testing.T) { // this is a regression guard. func TestPruneClientRfSamplesUsesIndex(t *testing.T) { s := newTestStore(t) - rows, err := s.db.Query(`EXPLAIN QUERY PLAN DELETE FROM client_rf_samples WHERE sampled_at < ?`, "2026-01-01T00:00:00Z") + if testBackendValue() == "postgres" { + if _, err := s.db.Exec(`SET enable_seqscan=off`); err != nil { + t.Fatal(err) + } + } + rows, err := s.db.Query(testNativeSQL(`EXPLAIN QUERY PLAN DELETE FROM client_rf_samples WHERE sampled_at < $1`, `EXPLAIN (COSTS OFF) DELETE FROM client_rf_samples WHERE sampled_at < $1`), "2026-01-01T00:00:00Z") if err != nil { t.Fatal(err) } defer rows.Close() plan := "" for rows.Next() { - var id, parent, notused int var detail string - if err := rows.Scan(&id, &parent, ¬used, &detail); err != nil { + if err := scanTestPlan(rows, &detail); err != nil { t.Fatal(err) } plan += detail + "\n" @@ -3492,7 +3207,7 @@ func TestPruneOldClientRfSamples(t *testing.T) { t.Fatalf("expected 2 samples remaining (recent + boundary), got %d", remaining) } var boundarySurvived int - s.db.QueryRow(`SELECT COUNT(*) FROM client_rf_samples WHERE sampled_at = ?`, boundary).Scan(&boundarySurvived) + s.db.QueryRow(`SELECT COUNT(*) FROM client_rf_samples WHERE sampled_at = $1`, boundary).Scan(&boundarySurvived) if boundarySurvived != 1 { t.Fatalf("boundary row (500ms after cutoff instant) must survive; an RFC3339 (no-ms) cutoff would wrongly delete it because '.' < 'Z' lexicographically — got %d", boundarySurvived) } diff --git a/cmd/ingestor/db_writer_perf_test.go b/cmd/ingestor/db_writer_perf_test.go index 740663f92..4c1312a18 100644 --- a/cmd/ingestor/db_writer_perf_test.go +++ b/cmd/ingestor/db_writer_perf_test.go @@ -30,7 +30,7 @@ func TestWriterStarvationVisibleInPerf(t *testing.T) { // follower samples can't move p99 above 50s. ResetWriterStatsForTest() - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -46,7 +46,7 @@ func TestWriterStarvationVisibleInPerf(t *testing.T) { go func() { defer close(blockerDone) err := s.WriterTx("neighbor_builder", func(tx *sql.Tx) error { - if _, err := tx.Exec(`UPDATE nodes SET name = name WHERE 0`); err != nil { + if _, err := tx.Exec(`UPDATE nodes SET name = name WHERE false`); err != nil { return err } close(blockStarted) @@ -59,7 +59,11 @@ func TestWriterStarvationVisibleInPerf(t *testing.T) { }() // Wait for the blocker to be inside its transaction. - <-blockStarted + select { + case <-blockStarted: + case <-blockerDone: + t.Fatal("blocker failed before acquiring its transaction") + } // Small safety margin so the blocker is firmly holding the conn. time.Sleep(100 * time.Millisecond) @@ -74,7 +78,7 @@ func TestWriterStarvationVisibleInPerf(t *testing.T) { defer wg.Done() _, err := s.WriterExec( "mqtt_handler", - `INSERT OR IGNORE INTO _migrations (name) VALUES (?)`, + `INSERT INTO _migrations (name) VALUES ($1) ON CONFLICT DO NOTHING`, fmt.Sprintf("writer_starvation_test_%d", i), ) if err != nil { diff --git a/cmd/ingestor/decoder.go b/cmd/ingestor/decoder.go index 41aa3afc4..ae0c2f9a4 100644 --- a/cmd/ingestor/decoder.go +++ b/cmd/ingestor/decoder.go @@ -1068,55 +1068,7 @@ func DecodePacket(hexString string, channelKeys map[string]string, validateSigna // route-independent identifier for the same logical packet. For TRACE packets, // path_len is included in the hash to match firmware behavior. func ComputeContentHash(rawHex string) string { - buf, err := hex.DecodeString(rawHex) - if err != nil || len(buf) < 2 { - if len(rawHex) >= 16 { - return rawHex[:16] - } - return rawHex - } - - headerByte := buf[0] - offset := 1 - if isTransportRoute(int(headerByte & 0x03)) { - offset += 4 - } - if offset >= len(buf) { - if len(rawHex) >= 16 { - return rawHex[:16] - } - return rawHex - } - pathByte := buf[offset] - offset++ - hashSize := int((pathByte>>6)&0x3) + 1 - hashCount := int(pathByte & 0x3F) - pathBytes := hashSize * hashCount - - payloadStart := offset + pathBytes - if payloadStart > len(buf) { - if len(rawHex) >= 16 { - return rawHex[:16] - } - return rawHex - } - - payload := buf[payloadStart:] - - // Hash payload-type byte only (bits 2-5 of header), not the full header. - // Firmware: SHA256(payload_type + [path_len for TRACE] + payload) - // Using the full header caused different hashes for the same logical packet - // when route type or version bits differed. See issue #786. - payloadType := (headerByte >> 2) & 0x0F - toHash := []byte{payloadType} - if int(payloadType) == PayloadTRACE { - // Firmware uses uint16_t path_len (2 bytes, little-endian) - toHash = append(toHash, pathByte, 0x00) - } - toHash = append(toHash, payload...) - - h := sha256.Sum256(toHash) - return hex.EncodeToString(h[:])[:16] + return packetpath.ContentHash(rawHex) } // PayloadJSON serializes the payload to JSON for DB storage. diff --git a/cmd/ingestor/from_pubkey_test.go b/cmd/ingestor/from_pubkey_test.go index 5fb614961..c0a47b862 100644 --- a/cmd/ingestor/from_pubkey_test.go +++ b/cmd/ingestor/from_pubkey_test.go @@ -10,7 +10,7 @@ import ( ) func TestInsertTransmission_FromPubkeyPopulatedForAdvert(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -34,14 +34,14 @@ func TestInsertTransmission_FromPubkeyPopulatedForAdvert(t *testing.T) { } var got sql.NullString - s.db.QueryRow("SELECT from_pubkey FROM transmissions WHERE hash = ?", data.Hash).Scan(&got) + s.db.QueryRow("SELECT from_pubkey FROM transmissions WHERE hash = $1", data.Hash).Scan(&got) if !got.Valid || got.String != pk { t.Fatalf("from_pubkey = %v (valid=%v), want %q", got.String, got.Valid, pk) } } func TestInsertTransmission_FromPubkeyNullForNonAdvert(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -64,7 +64,7 @@ func TestInsertTransmission_FromPubkeyNullForNonAdvert(t *testing.T) { } var got sql.NullString - s.db.QueryRow("SELECT from_pubkey FROM transmissions WHERE hash = ?", data.Hash).Scan(&got) + s.db.QueryRow("SELECT from_pubkey FROM transmissions WHERE hash = $1", data.Hash).Scan(&got) if got.Valid { t.Fatalf("from_pubkey for non-ADVERT must be NULL, got %q", got.String) } diff --git a/cmd/ingestor/go.mod b/cmd/ingestor/go.mod index a758d56c6..7f26c9703 100644 --- a/cmd/ingestor/go.mod +++ b/cmd/ingestor/go.mod @@ -1,6 +1,6 @@ module github.com/corescope/ingestor -go 1.22 +go 1.25.0 require ( github.com/eclipse/paho.mqtt.golang v1.5.0 @@ -30,21 +30,30 @@ replace github.com/meshcore-analyzer/dbschema => ../../internal/dbschema require ( github.com/gorilla/websocket v1.5.3 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/pgx/v5 v5.7.6 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + golang.org/x/crypto v0.37.0 // indirect golang.org/x/net v0.27.0 // indirect - golang.org/x/sync v0.7.0 // indirect + golang.org/x/sync v0.13.0 // indirect + golang.org/x/text v0.24.0 // indirect ) require github.com/meshcore-analyzer/prunequeue v0.0.0 replace github.com/meshcore-analyzer/prunequeue => ../../internal/prunequeue -require ( - github.com/mattn/go-sqlite3 v1.14.52 - github.com/meshcore-analyzer/mbcapqueue v0.0.0 -) +require github.com/meshcore-analyzer/mbcapqueue v0.0.0 replace github.com/meshcore-analyzer/mbcapqueue => ../../internal/mbcapqueue -require github.com/meshcore-analyzer/channel v0.0.0 +require ( + github.com/mattn/go-sqlite3 v1.14.52 + github.com/meshcore-analyzer/channel v0.0.0 + github.com/meshcore-analyzer/pgutil v0.0.0 +) replace github.com/meshcore-analyzer/channel => ../../internal/channel + +replace github.com/meshcore-analyzer/pgutil => ../../internal/pgutil diff --git a/cmd/ingestor/go.sum b/cmd/ingestor/go.sum index 554a48272..b17071293 100644 --- a/cmd/ingestor/go.sum +++ b/cmd/ingestor/go.sum @@ -1,10 +1,36 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/eclipse/paho.mqtt.golang v1.5.0 h1:EH+bUVJNgttidWFkLLVKaQPGmkTUfQQqjOsyvMGvD6o= github.com/eclipse/paho.mqtt.golang v1.5.0/go.mod h1:du/2qNQVqJf/Sqs4MEL77kR8QTqANF7XU7Fk0aOTAgk= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.7.6 h1:rWQc5FwZSPX58r1OQmkuaNicxdmExaEz5A2DO2hUuTk= +github.com/jackc/pgx/v5 v5.7.6/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/mattn/go-sqlite3 v1.14.52 h1:wVbm2Qnf4OXkqhBTSPuCRZDRnxfbVrrmiCEroVdog8U= github.com/mattn/go-sqlite3 v1.14.52/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= +golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= +golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= +golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/cmd/ingestor/ingest_buffer.go b/cmd/ingestor/ingest_buffer.go index 641801a3d..a762e6159 100644 --- a/cmd/ingestor/ingest_buffer.go +++ b/cmd/ingestor/ingest_buffer.go @@ -10,7 +10,7 @@ import ( // IngestBuffer decouples MQTT message receipt from DB writes (#1608). // // On boot the ingestor must subscribe to MQTT immediately, but the single -// SQLite writer (#1283) can be held for minutes by a startup migration +// serialized writer (#1283) can be held for minutes by a startup migration // (e.g. a large CREATE INDEX) or prune. Without buffering, every QoS-0 packet // received in that window is lost. IngestBuffer holds received work in a // bounded FIFO and a single consumer goroutine drains it once Ready() is diff --git a/cmd/ingestor/issue1283_test.go b/cmd/ingestor/issue1283_test.go index 4cca01641..96a04afac 100644 --- a/cmd/ingestor/issue1283_test.go +++ b/cmd/ingestor/issue1283_test.go @@ -1,12 +1,9 @@ package main import ( - "database/sql" "path/filepath" "testing" "time" - - _ "github.com/mattn/go-sqlite3" ) // TestIngestorPruneOldPackets enforces #1283: the writer for @@ -16,7 +13,7 @@ import ( func TestIngestorPruneOldPackets(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "prune.db") - store, err := OpenStore(path) + store, err := openPostgresTestStore(t, path) if err != nil { t.Fatalf("OpenStore: %v", err) } @@ -27,7 +24,7 @@ func TestIngestorPruneOldPackets(t *testing.T) { for i, ts := range []string{old, old, new} { _, err := store.db.Exec( `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) - VALUES (?, ?, ?, 0, 1, 1, '{}')`, + VALUES ($1, $2, $3, 0, 1, 1, '{}')`, "AA", "h"+string(rune('a'+i)), ts, ) if err != nil { @@ -52,47 +49,9 @@ func TestIngestorPruneOldPackets(t *testing.T) { } } -// TestIngestorVacuumOnStartupMigratesNONEtoINCREMENTAL exercises the -// scenario that originally broke in #1283: a fresh DB with -// auto_vacuum=NONE, vacuumOnStartup=true, no contention from a server -// process. The ingestor must complete the VACUUM and flip auto_vacuum to -// INCREMENTAL. Before the fix, the migration ran inside cmd/server and -// hit SQLITE_BUSY because the ingestor (sharing the container) was -// already writing. -func TestIngestorVacuumOnStartupMigratesNONEtoINCREMENTAL(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "vac.db") - - // Create a NONE-auto_vacuum DB (simulates an older deployment). - seed, err := sql.Open("sqlite3", path+"?_journal_mode=WAL") - if err != nil { - t.Fatal(err) - } - seed.SetMaxOpenConns(1) - if _, err := seed.Exec(`CREATE TABLE dummy(id INTEGER PRIMARY KEY)`); err != nil { - t.Fatal(err) - } - var before int - seed.QueryRow("PRAGMA auto_vacuum").Scan(&before) - if before != 0 { - t.Fatalf("precondition: auto_vacuum=%d, want 0", before) - } - seed.Close() - - store, err := OpenStore(path) - if err != nil { - t.Fatalf("OpenStore: %v", err) - } - defer store.Close() - - cfg := &Config{DB: &DBConfig{VacuumOnStartup: true}} - store.CheckAutoVacuum(cfg) - - var after int - if err := store.db.QueryRow("PRAGMA auto_vacuum").Scan(&after); err != nil { - t.Fatal(err) - } - if after != 2 { - t.Fatalf("expected auto_vacuum=2 after ingestor VACUUM, got %d", after) +func TestPostgresRuntimeCannotChangeTableVacuumPolicy(t *testing.T) { + s := newTestStore(t) + if _, err := s.db.Exec(`ALTER TABLE transmissions SET (autovacuum_enabled=false)`); err == nil { + t.Fatal("runtime writer changed owner-only vacuum policy") } } diff --git a/cmd/ingestor/issue1690_last_seen_test.go b/cmd/ingestor/issue1690_last_seen_test.go index 5b31917ec..6578f1f14 100644 --- a/cmd/ingestor/issue1690_last_seen_test.go +++ b/cmd/ingestor/issue1690_last_seen_test.go @@ -13,7 +13,7 @@ import ( ) func TestIssue1690_LastSeenUpdatedOnObservation(t *testing.T) { - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } @@ -42,7 +42,7 @@ func TestIssue1690_LastSeenUpdatedOnObservation(t *testing.T) { // Sanity: confirm the seed last_seen is the 7d-ago time. var seededLastSeen int64 - if err := s.db.QueryRow(`SELECT COALESCE(last_seen, 0) FROM transmissions WHERE hash = ?`, hash).Scan(&seededLastSeen); err != nil { + if err := s.db.QueryRow(`SELECT COALESCE(last_seen, 0) FROM transmissions WHERE hash = $1`, hash).Scan(&seededLastSeen); err != nil { t.Fatalf("seed select last_seen: %v (column missing? post-fix must add it)", err) } weekAgoUnix, _ := time.Parse(time.RFC3339, weekAgo) @@ -71,7 +71,7 @@ func TestIssue1690_LastSeenUpdatedOnObservation(t *testing.T) { } var ls int64 - if err := s.db.QueryRow(`SELECT last_seen FROM transmissions WHERE hash = ?`, hash).Scan(&ls); err != nil { + if err := s.db.QueryRow(`SELECT last_seen FROM transmissions WHERE hash = $1`, hash).Scan(&ls); err != nil { t.Fatalf("post-insert select last_seen: %v", err) } // The post-fix writer must bump last_seen to at least the new observation's diff --git a/cmd/ingestor/issue1865_test.go b/cmd/ingestor/issue1865_test.go index 4bf3e241e..fff3ab600 100644 --- a/cmd/ingestor/issue1865_test.go +++ b/cmd/ingestor/issue1865_test.go @@ -14,10 +14,10 @@ import ( // seedNode inserts a node into both nodes and inactive_nodes (lowercase key). func seedNode(t *testing.T, store *Store, pubkey string) { t.Helper() - if _, err := store.db.Exec(`INSERT INTO nodes (public_key, name) VALUES (?, ?)`, pubkey, "n_"+pubkey[:4]); err != nil { + if _, err := store.db.Exec(`INSERT INTO nodes (public_key, name) VALUES ($1, $2)`, pubkey, "n_"+pubkey[:4]); err != nil { t.Fatalf("seed node %s: %v", pubkey, err) } - if _, err := store.db.Exec(`INSERT INTO inactive_nodes (public_key, name) VALUES (?, ?)`, pubkey, "n_"+pubkey[:4]); err != nil { + if _, err := store.db.Exec(`INSERT INTO inactive_nodes (public_key, name) VALUES ($1, $2)`, pubkey, "n_"+pubkey[:4]); err != nil { t.Fatalf("seed inactive node %s: %v", pubkey, err) } } @@ -26,7 +26,7 @@ func configuredScope(t *testing.T, store *Store, pubkey string) (sql.NullString, t.Helper() var sc, at sql.NullString if err := store.db.QueryRow( - `SELECT configured_scope, configured_scope_at FROM nodes WHERE public_key = ?`, pubkey, + `SELECT configured_scope, configured_scope_at FROM nodes WHERE public_key = $1`, pubkey, ).Scan(&sc, &at); err != nil { t.Fatalf("read configured_scope for %s: %v", pubkey, err) } @@ -35,7 +35,7 @@ func configuredScope(t *testing.T, store *Store, pubkey string) (sql.NullString, func openNeighborsStore(t *testing.T) *Store { t.Helper() - store, err := OpenStore(filepath.Join(t.TempDir(), "test.db")) + store, err := openPostgresTestStore(t, filepath.Join(t.TempDir(), "test.db")) if err != nil { t.Fatalf("OpenStore: %v", err) } @@ -151,7 +151,7 @@ func TestUpdateNodeConfiguredScopeLastWriteWins(t *testing.T) { } // inactive_nodes mirrored. var inactive sql.NullString - if err := store.db.QueryRow(`SELECT configured_scope FROM inactive_nodes WHERE public_key = ?`, pk).Scan(&inactive); err != nil { + if err := store.db.QueryRow(`SELECT configured_scope FROM inactive_nodes WHERE public_key = $1`, pk).Scan(&inactive); err != nil { t.Fatal(err) } if inactive.String != "#de" { diff --git a/cmd/ingestor/key_backfill.go b/cmd/ingestor/key_backfill.go index 3aa9867f5..476e78109 100644 --- a/cmd/ingestor/key_backfill.go +++ b/cmd/ingestor/key_backfill.go @@ -159,7 +159,7 @@ func (b *keyBackfiller) pass(ctx context.Context) { // pruneFeed drops feed rows older than txRewriteFeedRetention. func (b *keyBackfiller) pruneFeed(ctx context.Context) { - if _, err := b.store.db.ExecContext(ctx, `DELETE FROM tx_rewrite_feed WHERE created_at < ?`, + if _, err := b.store.db.ExecContext(ctx, `DELETE FROM tx_rewrite_feed WHERE created_at < $1`, time.Now().Add(-txRewriteFeedRetention).Unix()); err != nil && ctx.Err() == nil { log.Printf("[key-backfill] pruning tx_rewrite_feed: %v", err) } @@ -282,7 +282,7 @@ func (b *keyBackfiller) runKind(ctx context.Context, kind string) error { if len(p.pending) == 0 { // Nothing new. Drop a state row a since-superseded run left behind. - _, err := db.ExecContext(ctx, `DELETE FROM key_backfill_state WHERE kind=?`, kind) + _, err := db.ExecContext(ctx, `DELETE FROM key_backfill_state WHERE kind=$1`, kind) return err } @@ -363,7 +363,7 @@ func (b *keyBackfiller) keySetMoved(kind string, p *keyBackfillPlan, runFP strin } func loadDoneKeys(ctx context.Context, db *sql.DB, kind string) (map[string]bool, error) { - rows, err := db.QueryContext(ctx, `SELECT key_fp FROM key_backfill_done WHERE kind=?`, kind) + rows, err := db.QueryContext(ctx, `SELECT key_fp FROM key_backfill_done WHERE kind=$1`, kind) if err != nil { return nil, fmt.Errorf("load done keys: %w", err) } @@ -388,7 +388,7 @@ func forgetRemovedKeys(ctx context.Context, db *sql.DB, kind string, all []strin if inForce[fp] { continue } - if _, err := db.ExecContext(ctx, `DELETE FROM key_backfill_done WHERE kind=? AND key_fp=?`, kind, fp); err != nil { + if _, err := db.ExecContext(ctx, `DELETE FROM key_backfill_done WHERE kind=$1 AND key_fp=$2`, kind, fp); err != nil { return fmt.Errorf("forget removed key: %w", err) } } @@ -402,7 +402,7 @@ func forgetRemovedKeys(ctx context.Context, db *sql.DB, kind string, all []strin func (b *keyBackfiller) resumeOrStart(ctx context.Context, kind, runFP string) (cursor, upper, rewritten int64, err error) { db := b.store.db var storedFP string - err = db.QueryRowContext(ctx, `SELECT run_fp, cursor, upper, rewritten FROM key_backfill_state WHERE kind=?`, kind). + err = db.QueryRowContext(ctx, `SELECT run_fp, cursor, upper, rewritten FROM key_backfill_state WHERE kind=$1`, kind). Scan(&storedFP, &cursor, &upper, &rewritten) if err == nil && storedFP == runFP { return cursor, upper, rewritten, nil @@ -413,7 +413,7 @@ func (b *keyBackfiller) resumeOrStart(ctx context.Context, kind, runFP string) ( if err := db.QueryRowContext(ctx, `SELECT COALESCE(MAX(id),0) FROM transmissions`).Scan(&upper); err != nil { return 0, 0, 0, fmt.Errorf("backfill upper bound: %w", err) } - if _, err := db.ExecContext(ctx, `INSERT OR REPLACE INTO key_backfill_state(kind, run_fp, cursor, upper, rewritten) VALUES(?,?,0,?,0)`, + if _, err := db.ExecContext(ctx, `INSERT INTO key_backfill_state(kind, run_fp, cursor, upper, rewritten) VALUES($1,$2,0,$3,0) ON CONFLICT(kind) DO UPDATE SET run_fp=excluded.run_fp,cursor=0,upper=excluded.upper,rewritten=0,started_at=excluded.started_at`, kind, runFP, upper); err != nil { return 0, 0, 0, fmt.Errorf("start backfill run: %w", err) } @@ -429,11 +429,11 @@ func finishRun(ctx context.Context, db *sql.DB, kind, runFP string, all []string } defer tx.Rollback() for _, fp := range all { - if _, err := tx.ExecContext(ctx, `INSERT OR IGNORE INTO key_backfill_done(kind, key_fp) VALUES(?,?)`, kind, fp); err != nil { + if _, err := tx.ExecContext(ctx, `INSERT INTO key_backfill_done(kind, key_fp) VALUES($1,$2) ON CONFLICT(kind,key_fp) DO NOTHING`, kind, fp); err != nil { return fmt.Errorf("record done key: %w", err) } } - if _, err := tx.ExecContext(ctx, `DELETE FROM key_backfill_state WHERE kind=? AND run_fp=?`, kind, runFP); err != nil { + if _, err := tx.ExecContext(ctx, `DELETE FROM key_backfill_state WHERE kind=$1 AND run_fp=$2`, kind, runFP); err != nil { return err } return tx.Commit() @@ -475,18 +475,20 @@ func (b *keyBackfiller) batchOnce(ctx context.Context, kind string, p keyBackfil return b.commitEmpty(ctx, kind, runFP, upper) } args := []interface{}{cursor, hi} - for _, h := range p.channelHash { + hashParams := make([]string, len(p.channelHash)) + for i, h := range p.channelHash { args = append(args, h) + hashParams[i] = b.store.parameter(i + 3) } rows, err = db.QueryContext(ctx, ` SELECT id, COALESCE(raw_hex,''), channel_hash FROM transmissions - WHERE id > ? AND id <= ? AND payload_type = 5 - AND channel_hash IN (`+strings.TrimSuffix(strings.Repeat("?,", len(p.channelHash)), ",")+`) + WHERE id > $1 AND id <= $2 AND payload_type = 5 + AND channel_hash IN (`+strings.Join(hashParams, ",")+`) ORDER BY id`, args...) case keyBackfillKindRegion: rows, err = db.QueryContext(ctx, ` SELECT id, COALESCE(raw_hex,''), '' FROM transmissions - WHERE id > ? AND id <= ? AND scope_name = '' + WHERE id > $1 AND id <= $2 AND scope_name = '' ORDER BY id`, cursor, hi) default: return cursor, 0, fmt.Errorf("unknown backfill kind %q", kind) @@ -570,7 +572,7 @@ func (b *keyBackfiller) decide(kind string, p keyBackfillPlan, id int64, raw, ol func (b *keyBackfiller) commit(ctx context.Context, kind, runFP string, next int64, rewrites []keyRewrite) (int, error) { writerMu.Lock() defer writerMu.Unlock() - tx, err := b.store.db.BeginTx(ctx, nil) + tx, err := beginWriteContext(ctx, b.store.db, b.store.Backend()) if err != nil { return 0, err } @@ -581,10 +583,10 @@ func (b *keyBackfiller) commit(ctx context.Context, kind, runFP string, next int var res sql.Result switch kind { case keyBackfillKindChannel: - res, err = tx.ExecContext(ctx, `UPDATE transmissions SET channel_hash=?, decoded_json=? WHERE id=? AND channel_hash=?`, + res, err = tx.ExecContext(ctx, `UPDATE transmissions SET channel_hash=$1, decoded_json=$2 WHERE id=$3 AND channel_hash=$4`, r.channelHash, r.decodedJSON, r.id, r.oldChannel) case keyBackfillKindRegion: - res, err = tx.ExecContext(ctx, `UPDATE transmissions SET scope_name=? WHERE id=? AND scope_name=''`, r.scopeName, r.id) + res, err = tx.ExecContext(ctx, `UPDATE transmissions SET scope_name=$1 WHERE id=$2 AND scope_name=''`, r.scopeName, r.id) } if err != nil { return 0, fmt.Errorf("rewrite tx %d: %w", r.id, err) @@ -593,7 +595,7 @@ func (b *keyBackfiller) commit(ctx context.Context, kind, runFP string, next int continue // resolved meanwhile by someone else; theirs stands } n++ - if _, err := tx.ExecContext(ctx, `INSERT INTO tx_rewrite_feed(tx_id) VALUES(?)`, r.id); err != nil { + if _, err := tx.ExecContext(ctx, `INSERT INTO tx_rewrite_feed(tx_id) VALUES($1)`, r.id); err != nil { return 0, fmt.Errorf("feed tx %d: %w", r.id, err) } // A node whose default_scope is still unknown gets the region its own @@ -601,14 +603,14 @@ func (b *keyBackfiller) commit(ctx context.Context, kind, runFP string, next int // backfilled advert can be, so it is never overwritten. if r.advertFrom != "" { for _, table := range []string{"nodes", "inactive_nodes"} { - if _, err := tx.ExecContext(ctx, `UPDATE `+table+` SET default_scope=? - WHERE public_key=? AND (default_scope IS NULL OR default_scope='')`, r.scopeName, r.advertFrom); err != nil { + if _, err := tx.ExecContext(ctx, `UPDATE `+table+` SET default_scope=$1 + WHERE public_key=$2 AND (default_scope IS NULL OR default_scope='')`, r.scopeName, r.advertFrom); err != nil { return 0, fmt.Errorf("default_scope for %s: %w", r.advertFrom, err) } } } } - res, err := tx.ExecContext(ctx, `UPDATE key_backfill_state SET cursor=?, rewritten=rewritten+? WHERE kind=? AND run_fp=?`, + res, err := tx.ExecContext(ctx, `UPDATE key_backfill_state SET cursor=$1, rewritten=rewritten+$2 WHERE kind=$3 AND run_fp=$4`, next, n, kind, runFP) if err != nil { return 0, fmt.Errorf("advance cursor: %w", err) diff --git a/cmd/ingestor/key_backfill_test.go b/cmd/ingestor/key_backfill_test.go index d6699321a..b797540bd 100644 --- a/cmd/ingestor/key_backfill_test.go +++ b/cmd/ingestor/key_backfill_test.go @@ -99,13 +99,13 @@ func insertTx(t *testing.T, s *Store, raw string, payloadType int, channel strin if channel != "" { ch = channel } - res, err := s.db.Exec(`INSERT INTO transmissions(raw_hex, hash, first_seen, payload_type, decoded_json, channel_hash, scope_name) - VALUES(?,?,?,?,?,?,?)`, raw, fmt.Sprintf("kb-%d", txSeq.Add(1)), "2026-10-01T00:00:00Z", - payloadType, decoded, ch, scope) + var id int64 + err := s.db.QueryRow(`INSERT INTO transmissions(raw_hex, hash, first_seen, payload_type, decoded_json, channel_hash, scope_name) + VALUES($1,$2,$3,$4,$5,$6,$7) RETURNING id`, raw, fmt.Sprintf("kb-%d", txSeq.Add(1)), "2026-10-01T00:00:00Z", + payloadType, decoded, ch, scope).Scan(&id) if err != nil { t.Fatal(err) } - id, _ := res.LastInsertId() return id } @@ -149,7 +149,7 @@ func feedRows(t *testing.T, s *Store) []int64 { func colOf(t *testing.T, s *Store, col string, id int64) string { t.Helper() var v *string - if err := s.db.QueryRow(`SELECT `+col+` FROM transmissions WHERE id=?`, id).Scan(&v); err != nil { + if err := s.db.QueryRow(`SELECT `+col+` FROM transmissions WHERE id=$1`, id).Scan(&v); err != nil { t.Fatal(err) } if v == nil { @@ -307,8 +307,8 @@ func TestKeyBackfillAdvertFillsOnlyEmptyDefaultScope(t *testing.T) { t.Fatal(err) } var e, k string - s.db.QueryRow(`SELECT COALESCE(default_scope,'') FROM nodes WHERE public_key=?`, pkEmpty).Scan(&e) - s.db.QueryRow(`SELECT COALESCE(default_scope,'') FROM nodes WHERE public_key=?`, pkKnown).Scan(&k) + s.db.QueryRow(`SELECT COALESCE(default_scope,'') FROM nodes WHERE public_key=$1`, pkEmpty).Scan(&e) + s.db.QueryRow(`SELECT COALESCE(default_scope,'') FROM nodes WHERE public_key=$1`, pkKnown).Scan(&k) if e != "#saar" { t.Fatalf("empty default_scope = %q, want #saar", e) } @@ -398,7 +398,7 @@ func TestKeyBackfillKeySetGrowthRestartsFromBeginning(t *testing.T) { low := insertTx(t, s, transportFrame(regionKey("#new"), 5, []byte("early")), 5, "", "{}", strp("")) high := insertTx(t, s, transportFrame(regionKey("#new"), 5, []byte("late")), 5, "", "{}", strp("")) // A half-finished run for an older key set, already past `low`. - if _, err := s.db.Exec(`INSERT INTO key_backfill_state(kind, run_fp, cursor, upper) VALUES('region','stale-set',?,?)`, low, high); err != nil { + if _, err := s.db.Exec(`INSERT INTO key_backfill_state(kind, run_fp, cursor, upper) VALUES('region','stale-set',$1,$2)`, low, high); err != nil { t.Fatal(err) } b := newTestBackfiller(s, nil, regionSetOf("#old", "#new")) @@ -462,10 +462,10 @@ func TestKeyBackfillCommitLeavesConcurrentlyResolvedRow(t *testing.T) { s := newTestStore(t) id := insertTx(t, s, transportFrame(regionKey("#saar"), 5, []byte("race")), 5, "", "{}", strp("")) b := newTestBackfiller(s, nil, regionSetOf("#saar")) - if _, err := s.db.Exec(`INSERT INTO key_backfill_state(kind, run_fp, cursor, upper) VALUES('region','fp',0,?)`, id); err != nil { + if _, err := s.db.Exec(`INSERT INTO key_backfill_state(kind, run_fp, cursor, upper) VALUES('region','fp',0,$1)`, id); err != nil { t.Fatal(err) } - if _, err := s.db.Exec(`UPDATE transmissions SET scope_name='#live' WHERE id=?`, id); err != nil { + if _, err := s.db.Exec(`UPDATE transmissions SET scope_name='#live' WHERE id=$1`, id); err != nil { t.Fatal(err) } n, err := b.commit(context.Background(), keyBackfillKindRegion, "fp", id, []keyRewrite{{id: id, scopeName: "#saar"}}) @@ -488,7 +488,7 @@ func TestKeyBackfillPrunesOldFeedRows(t *testing.T) { s := newTestStore(t) id := insertTx(t, s, "1500aa", 5, "", "{}", nil) old := time.Now().Add(-48 * time.Hour).Unix() - if _, err := s.db.Exec(`INSERT INTO tx_rewrite_feed(tx_id, created_at) VALUES(?,?),(?,?)`, id, old, id, time.Now().Unix()); err != nil { + if _, err := s.db.Exec(`INSERT INTO tx_rewrite_feed(tx_id, created_at) VALUES($1,$2),($3,$4)`, id, old, id, time.Now().Unix()); err != nil { t.Fatal(err) } newTestBackfiller(s, nil, nil).pass(context.Background()) @@ -594,7 +594,7 @@ func TestKeyBackfillPrunesFeedWithoutPass(t *testing.T) { // Let the startup pass finish, then add a stale row no pass will see. time.Sleep(50 * time.Millisecond) id := insertTx(t, s, "1500aa", 5, "", "{}", nil) - if _, err := s.db.Exec(`INSERT INTO tx_rewrite_feed(tx_id, created_at) VALUES(?,?)`, + if _, err := s.db.Exec(`INSERT INTO tx_rewrite_feed(tx_id, created_at) VALUES($1,$2)`, id, time.Now().Add(-48*time.Hour).Unix()); err != nil { t.Fatal(err) } @@ -611,20 +611,29 @@ func TestKeyBackfillPrunesFeedWithoutPass(t *testing.T) { // transmission retention deletes would scan the whole feed. func TestTxRewriteFeedTxIDIsIndexed(t *testing.T) { s := newTestStore(t) - rows, err := s.db.Query(`EXPLAIN QUERY PLAN SELECT id FROM tx_rewrite_feed WHERE tx_id = ?`, 1) + if testBackendValue() == "postgres" { + // The empty fixture otherwise makes a sequential scan cheaper; require + // an applicable tx_id index, matching the retention-plan regressions. + if _, err := s.db.Exec(`SET enable_seqscan=off`); err != nil { + t.Fatal(err) + } + } + rows, err := s.db.Query(testNativeSQL(`EXPLAIN QUERY PLAN `, `EXPLAIN (COSTS OFF) `)+`SELECT id FROM tx_rewrite_feed WHERE tx_id = $1`, 1) if err != nil { t.Fatal(err) } defer rows.Close() var plan []string for rows.Next() { - var id, parent, notused int var detail string - if err := rows.Scan(&id, &parent, ¬used, &detail); err != nil { + if err := scanTestPlan(rows, &detail); err != nil { t.Fatal(err) } plan = append(plan, detail) } + if err := rows.Err(); err != nil { + t.Fatal(err) + } if !strings.Contains(strings.Join(plan, " | "), "idx_tx_rewrite_feed_tx") { t.Fatalf("lookup by tx_id does not use idx_tx_rewrite_feed_tx: %v", plan) } diff --git a/cmd/ingestor/linked_companions.go b/cmd/ingestor/linked_companions.go index d8f3cf2d8..cc51ce72a 100644 --- a/cmd/ingestor/linked_companions.go +++ b/cmd/ingestor/linked_companions.go @@ -1,15 +1,14 @@ package main import ( - "database/sql" - "fmt" + "context" "log" - "os" - "path/filepath" "strings" "sync" "sync/atomic" "time" + + "github.com/meshcore-analyzer/dbconfig" ) // Linked-only ingest (docs/specs/2026-10-08-companion-linking-design.md, @@ -36,12 +35,11 @@ const linkedCompanionsQuery = `SELECT pubkey FROM companion_links` // pubkeys. Snapshots are never mutated after they are stored, so a hit is // lock-free. Reads (refresh, or a miss after the gap) are serialized by mu. type linkedCompanionSet struct { - path string - now func() time.Time - cur atomic.Pointer[map[string]struct{}] + reader *channelKeySet + now func() time.Time + cur atomic.Pointer[map[string]struct{}] mu sync.Mutex // guards everything below - db *sql.DB // opened lazily, read-only lastRead time.Time // last read attempt, for the miss gap reads int // read attempts (tests) failing bool // the last read failed; logged once per failure streak @@ -53,7 +51,7 @@ type linkedCompanionSet struct { // companion_links, every pubkey passes. A users.db that is missing, unreadable // or older than companion linking must not cost the data of linked companions. func newLinkedCompanionSet(usersDBPath string) *linkedCompanionSet { - s := &linkedCompanionSet{path: usersDBPath, now: time.Now} + s := &linkedCompanionSet{reader: newChannelKeySet(nil, usersDBPath, 0), now: time.Now} empty := map[string]struct{}{} s.cur.Store(&empty) return s @@ -106,9 +104,9 @@ func (s *linkedCompanionSet) readLocked() { s.lastRead = s.now() s.reads++ keys, err := s.query() - if err != nil && strings.Contains(err.Error(), "no such table") { + if err != nil && (strings.Contains(err.Error(), "no such table") || strings.Contains(err.Error(), "SQLSTATE 42P01")) { if !s.noTable { - log.Printf("[companions] %s has no companion_links table (a server older than companion linking); passing client data until it appears", s.path) + log.Printf("[companions] selected %s account store has no companion_links table/projection (a server older than companion linking); passing client data until it appears", s.reader.backend) s.noTable = true } return @@ -121,7 +119,7 @@ func (s *linkedCompanionSet) readLocked() { return } if s.failing { - log.Printf("[companions] linked companions readable again from %s", s.path) + log.Printf("[companions] linked companions readable again from %s", s.reader.backend) s.failing = false } next := make(map[string]struct{}, len(keys)) @@ -133,21 +131,18 @@ func (s *linkedCompanionSet) readLocked() { } func (s *linkedCompanionSet) query() ([]string, error) { - if s.db == nil { - if _, err := os.Stat(s.path); err != nil { - if os.IsNotExist(err) { - return nil, fmt.Errorf("%w at %s", errUsersDBMissing, s.path) - } - return nil, err - } - db, err := sql.Open("sqlite3", usersDBReadOnlyDSN(s.path)) - if err != nil { - return nil, err - } - db.SetMaxOpenConns(1) - s.db = db + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + tx, err := s.reader.readAccountSnapshot(ctx) + if err != nil { + return nil, err } - rows, err := s.db.Query(linkedCompanionsQuery) + defer tx.Rollback() + query := linkedCompanionsQuery + if s.reader.backend == dbconfig.Postgres { + query = `SELECT pubkey FROM linked_companions` + } + rows, err := tx.QueryContext(ctx, query) if err != nil { return nil, err } @@ -160,17 +155,21 @@ func (s *linkedCompanionSet) query() ([]string, error) { } out = append(out, pk) } - return out, rows.Err() + if err := rows.Err(); err != nil { + return nil, err + } + if err := rows.Close(); err != nil { + return nil, err + } + return out, tx.Commit() } // Close closes the read-only handle, if any. func (s *linkedCompanionSet) Close() { s.mu.Lock() defer s.mu.Unlock() - if s.db != nil { - s.db.Close() - s.db = nil - } + s.reader.Close() + s.reader.db = nil } // allowClientPubkey applies the linked-only filter to a client topic's @@ -188,7 +187,7 @@ func (s *Store) allowClientPubkey(pubkey string) bool { // once, when clientRxCoverage.requireLinkedCompanion is in effect, else nil. // The setting without userManagement.enabled is ignored with a startup // warning, as on the server. -func newLinkedCompanionFilter(cfg *Config) *linkedCompanionSet { +func newLinkedCompanionFilter(cfg *Config, storage dbconfig.Storage) *linkedCompanionSet { if !cfg.RequireLinkedCompanionSet() { return nil } @@ -196,12 +195,9 @@ func newLinkedCompanionFilter(cfg *Config) *linkedCompanionSet { log.Printf("[companions] WARNING: clientRxCoverage.requireLinkedCompanion is ignored: it needs userManagement.enabled") return nil } - shown := cfg.UsersDBPath() - if abs, err := filepath.Abs(shown); err == nil { - shown = abs - } - log.Printf("[companions] only linked companions may publish client data; reading companion_links from %s", shown) - s := newLinkedCompanionSet(cfg.UsersDBPath()) + log.Printf("[companions] only linked companions may publish client data; reading the selected %s account projection", storage.Backend) + s := newLinkedCompanionSet("") + s.reader = newChannelKeySetStorage(nil, storage, 0) s.refresh() return s } diff --git a/cmd/ingestor/linked_companions_startup_test.go b/cmd/ingestor/linked_companions_startup_test.go index 0fca77392..a2addf1dd 100644 --- a/cmd/ingestor/linked_companions_startup_test.go +++ b/cmd/ingestor/linked_companions_startup_test.go @@ -7,6 +7,8 @@ import ( "path/filepath" "strings" "testing" + + "github.com/meshcore-analyzer/dbconfig" ) func TestLinkedCompanionFilterStartup(t *testing.T) { @@ -15,10 +17,11 @@ func TestLinkedCompanionFilterStartup(t *testing.T) { defer log.SetOutput(os.Stderr) usersPath := filepath.Join(t.TempDir(), "users.db") setLinks(t, usersPath, testCompanionPK) + storage := dbconfig.Storage{Backend: approvedTestBackend(t), UsersDBPath: usersPath, ApprovedChannelsDatabaseURL: approvedFixture(t, usersPath).reader} um := func(on bool) *UserManagementConfig { return &UserManagementConfig{Enabled: on, DBPath: usersPath} } off := &Config{ClientRxCoverage: &ClientRxCoverageConfig{Enabled: true}, UserManagement: um(true)} - if f := newLinkedCompanionFilter(off); f != nil { + if f := newLinkedCompanionFilter(off, storage); f != nil { t.Fatal("filter installed with the setting off") } if buf.Len() != 0 { @@ -26,7 +29,7 @@ func TestLinkedCompanionFilterStartup(t *testing.T) { } ignored := &Config{ClientRxCoverage: &ClientRxCoverageConfig{RequireLinkedCompanion: true}, UserManagement: um(false)} - if f := newLinkedCompanionFilter(ignored); f != nil { + if f := newLinkedCompanionFilter(ignored, storage); f != nil { t.Fatal("filter installed with user management off") } if !strings.Contains(buf.String(), "requireLinkedCompanion is ignored") { @@ -34,7 +37,7 @@ func TestLinkedCompanionFilterStartup(t *testing.T) { } on := &Config{ClientRxCoverage: &ClientRxCoverageConfig{RequireLinkedCompanion: true}, UserManagement: um(true)} - f := newLinkedCompanionFilter(on) + f := newLinkedCompanionFilter(on, storage) if f == nil { t.Fatal("no filter with the setting and user management on") } diff --git a/cmd/ingestor/linked_companions_test.go b/cmd/ingestor/linked_companions_test.go index 7dd27e012..710764683 100644 --- a/cmd/ingestor/linked_companions_test.go +++ b/cmd/ingestor/linked_companions_test.go @@ -10,6 +10,8 @@ import ( "sync" "testing" "time" + + "github.com/meshcore-analyzer/dbconfig" ) // The columns of companion_links in cmd/server's users.db (internal/users @@ -51,6 +53,16 @@ func setLinks(t testing.TB, path string, pks ...string) { stmts = append(stmts, fmt.Sprintf(`INSERT INTO companion_links (pubkey, user_id, linked_at) VALUES ('%s', 1, %d)`, pk, i)) } execUsersDB(t, path, stmts...) + if approvedTestBackend(t) == dbconfig.SQLite { + execUsersDB(t, path, `CREATE TABLE IF NOT EXISTS schema_version(version INTEGER NOT NULL); DELETE FROM schema_version; INSERT INTO schema_version VALUES(6)`) + } else { + f := approvedFixture(t, path) + execUsersDB(t, path, + `CREATE TABLE IF NOT EXISTS corescope_schema(version INTEGER NOT NULL,kind TEXT PRIMARY KEY,ready BOOLEAN NOT NULL)`, + `INSERT INTO corescope_schema VALUES(1,'accounts',true) ON CONFLICT(kind) DO NOTHING`, + `CREATE OR REPLACE VIEW linked_companions AS SELECT pubkey FROM companion_links`, + `GRANT SELECT ON linked_companions,corescope_schema TO `+f.role) + } } // newTestLinkedSet returns a set over a fresh users.db holding pks, on a @@ -60,7 +72,7 @@ func newTestLinkedSet(t testing.TB, pks ...string) (*linkedCompanionSet, string, path := filepath.Join(t.TempDir(), "users.db") setLinks(t, path, pks...) clk := &linkClock{t: time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC)} - s := newLinkedCompanionSet(path) + s := newLinkedCompanionSet(approvedFixture(t, path).reader) s.now = clk.now t.Cleanup(s.Close) return s, path, clk @@ -151,7 +163,7 @@ func TestLinkedCompanionSetMissingTablePassesAndWarnsOnce(t *testing.T) { defer log.SetOutput(os.Stderr) path := filepath.Join(t.TempDir(), "users.db") writeUsersDB(t, path) // an older users.db: proposals, no companion_links - s := newLinkedCompanionSet(path) + s := newLinkedCompanionSet(approvedFixture(t, path).reader) defer s.Close() s.refresh() s.refresh() @@ -172,11 +184,14 @@ func TestLinkedCompanionSetMissingTablePassesAndWarnsOnce(t *testing.T) { } func TestLinkedCompanionSetMissingFileLogsOnceAndRecovers(t *testing.T) { + if approvedTestBackend(t) != dbconfig.SQLite { + t.Skip("SQLite file disappearance only") + } var buf bytes.Buffer log.SetOutput(&buf) defer log.SetOutput(os.Stderr) path := filepath.Join(t.TempDir(), "users.db") - s := newLinkedCompanionSet(path) + s := newLinkedCompanionSet(approvedFixture(t, path).reader) defer s.Close() s.refresh() s.refresh() @@ -208,8 +223,12 @@ func TestLinkedCompanionSetFailsClosedOnlyAfterFirstRead(t *testing.T) { t.Fatal("an unlinked pubkey passed after a successful read") } s.Close() - if err := os.Remove(path); err != nil { - t.Fatal(err) + if approvedTestBackend(t) == dbconfig.SQLite { + if err := os.Remove(path); err != nil { + t.Fatal(err) + } + } else { + execUsersDB(t, path, `DROP VIEW linked_companions`) } clock.add(linkedCompanionsMissGap) s.refresh() // fails: users.db is gone @@ -223,15 +242,12 @@ func TestLinkedCompanionSetFailsClosedOnlyAfterFirstRead(t *testing.T) { } func TestLinkedCompanionSetIsReadOnly(t *testing.T) { - s, path, _ := newTestLinkedSet(t, testCompanionPK) + s, _, _ := newTestLinkedSet(t, testCompanionPK) s.refresh() - _, err := s.db.Exec(`INSERT INTO companion_links (pubkey, user_id, linked_at) VALUES ('x', 1, 1)`) - if err == nil || !strings.Contains(strings.ToLower(err.Error()), "readonly") { + _, err := s.reader.db.Exec(`INSERT INTO companion_links (pubkey, user_id, linked_at) VALUES ('x', 1, 1)`) + if err == nil || (!strings.Contains(strings.ToLower(err.Error()), "readonly") && !strings.Contains(strings.ToLower(err.Error()), "read-only")) { t.Fatalf("write through the ingestor's users.db handle: err = %v; want a read-only refusal", err) } - if !strings.Contains(usersDBReadOnlyDSN(path), "mode=ro") { - t.Fatal("DSN without mode=ro") - } } // benchLinkedSet is a loaded set with n linked pubkeys, the first returned. diff --git a/cmd/ingestor/main.go b/cmd/ingestor/main.go index cd5a5a3a4..3d8cbf06c 100644 --- a/cmd/ingestor/main.go +++ b/cmd/ingestor/main.go @@ -8,6 +8,7 @@ import ( "crypto/tls" "encoding/hex" "encoding/json" + "errors" "flag" "fmt" "log" @@ -25,6 +26,7 @@ import ( "time" mqtt "github.com/eclipse/paho.mqtt.golang" + "github.com/meshcore-analyzer/dbconfig" ) func main() { @@ -43,6 +45,10 @@ func main() { } configPath := flag.String("config", "config.json", "path to config file") + databaseURL := flag.String("database-url", "", "PostgreSQL telemetry writer URL") + stateDir := flag.String("state-dir", "", "local queue and statistics directory") + legacyDB := flag.String("db", "", "SQLite telemetry path (bootstrap override)") + backend := flag.String("backend", "", "Storage backend: sqlite or postgres (bootstrap choice)") flag.Parse() log.SetFlags(log.LstdFlags | log.Lmsgprefix) @@ -74,12 +80,32 @@ func main() { sources := cfg.ResolvedSources() - store, err := OpenStoreWithInterval(cfg.DBPath, cfg.MetricsSampleInterval()) + base, err := os.Getwd() + if err != nil { + log.Fatal("resolve working directory") + } + rawStorage, err := cfg.storageInputs(base, storageFlags{Backend: dbconfig.Backend(*backend), DBPath: *legacyDB, DatabaseURL: *databaseURL, StateDir: *stateDir}, os.Getenv) + if err != nil { + log.Fatalf("storage: %v", err) + } + storage, selectionLease, err := resolveRuntimeStorage(rawStorage) + if errors.Is(err, dbconfig.ErrSelectionMissing) { + if err = adoptLegacyStorage(rawStorage); err == nil { + storage, selectionLease, err = resolveRuntimeStorage(rawStorage) + } + } + if err != nil { + log.Fatalf("storage: %v", err) + } + defer selectionLease.Close() + cfg.StateDir = storage.StateDir + store, err := openStoreStorage(storage, cfg.DB, cfg.MetricsSampleInterval()) + if err != nil { log.Fatalf("db: %v", err) } defer store.Close() - log.Printf("SQLite opened: %s", cfg.DBPath) + log.Printf("%s telemetry writer connected", storage.Backend) // Async backfill: path_json from raw_hex (#888) — must not block MQTT startup store.BackfillPathJSONAsync() @@ -103,15 +129,15 @@ func main() { log.Printf("No channel keys loaded — GRP_TXT packets will not be decrypted") } - keySet := newChannelKeySet(channelKeys, cfg.UsersDBPath(), cfg.ApprovedChannelsMax()) + keySet := newChannelKeySetStorage(channelKeys, storage, cfg.ApprovedChannelsMax()) if cfg.ApprovedChannelsEnabled() { - logApprovedChannelsSource(cfg.UsersDBPath()) + log.Printf("[channels] approved channel storage: %s", storage.Backend) keySet.refresh() } // Linked-only ingest (clientRxCoverage.requireLinkedCompanion): nil when // off or ignored. Installed before any MQTT source connects. - if lc := newLinkedCompanionFilter(cfg); lc != nil { + if lc := newLinkedCompanionFilter(cfg, storage); lc != nil { store.linkedCompanions = lc go lc.refreshLoop() } @@ -144,7 +170,7 @@ func main() { // Subscribe-early + buffer (#1608): the MQTT subscription is brought up // before startup maintenance so no packets are missed while the single - // SQLite writer is blocked (e.g. a large CREATE INDEX migration). Received + // serialized writer is busy with a bounded data backfill. Received // messages are buffered here and drained once Ready() is called below. ingestBuffer := NewIngestBuffer(cfg.IngestBufferSizeOrDefault()) ingestBuffer.Start() @@ -381,8 +407,7 @@ func main() { store.RunIncrementalVacuum(vacuumPages) // Gate open: the synchronous startup writes above cannot return until the - // single SQLite writer is free, which means any blocking async migration - // (e.g. the CREATE INDEX) has finished. WaitForAsyncMigrations() makes that + // single serialized writer is free. WaitForAsyncMigrations() makes that // explicit. Now drain everything the subscription buffered during startup. store.WaitForAsyncMigrations() ingestBuffer.Ready() @@ -424,10 +449,12 @@ func main() { store.RemoveStaleObservers(observerDays) store.PurgeStaleObservers(observerPurgeDays) store.RunIncrementalVacuum(vacuumPages) + for range observerRetentionTicker.C { store.RemoveStaleObservers(observerDays) store.PurgeStaleObservers(observerPurgeDays) store.RunIncrementalVacuum(vacuumPages) + } }() @@ -442,7 +469,7 @@ func main() { }() // Daily trim of the reach hop index, independent of packet retention. - if cfg.HopIndexDaysOrZero() > 0 { + if store.hopIndexDays > 0 { go func() { for range time.Tick(24 * time.Hour) { if n, err := store.TrimObservationHops(); err != nil { @@ -540,63 +567,68 @@ func main() { log.Printf("[regions] auto-derived region keys enabled: refreshing every %v, cap %d", interval, cfg.AutoRegionKeysMaxDerived()) } - // Hourly WAL checkpoint to prevent unbounded WAL growth. - // TRUNCATE resets the WAL file to zero bytes when all frames are flushed; - // if the server's read connection holds frames, remaining pages stay in the - // WAL until the next tick. Staggered 30s after startup to avoid competing - // with the initial burst of ingest writes. - walCheckpointTicker := time.NewTicker(1 * time.Hour) - go func() { - time.Sleep(30 * time.Second) - store.Checkpoint() - for range walCheckpointTicker.C { + // PostgreSQL maintenance remains owned by its database service. + if storage.Backend == dbconfig.SQLite { + // Hourly WAL checkpoint to prevent unbounded WAL growth. + // TRUNCATE resets the WAL file to zero bytes when all frames are flushed; + // if the server's read connection holds frames, remaining pages stay in the + // WAL until the next tick. Staggered 30s after startup to avoid competing + // with the initial burst of ingest writes. + walCheckpointTicker := time.NewTicker(1 * time.Hour) + go func() { + time.Sleep(30 * time.Second) store.Checkpoint() - } - }() - log.Printf("[db] WAL checkpoint scheduled every 1h") + for range walCheckpointTicker.C { + store.Checkpoint() + } + }() + log.Printf("[db] WAL checkpoint scheduled every 1h") - // Daily planner statistics refresh (#2058), in two parts. - // - // The routine refresh runs 24 hours after the previous one, recorded next to - // the database (#2146), and is staggered at least 2 minutes past startup for the same reason - // as the checkpoint above: it takes the write lock, and by then the initial - // ingest burst has passed, so it also sees the rows that burst added. - // - // The build in front of it deliberately does compete with that burst, because - // a database with no statistics at all has nothing better to offer the queries - // arriving in those 2 minutes. It only runs once per database; see - // Store.EnsurePlannerStats, which also carries what that costs. - // - // Bounded by analysis_limit either way, so neither grows with the file the way - // an unbounded ANALYZE does: 2.0s against 242.9s on a 9.4 GB database, both - // timed warm. Cold, on a first start, it is 3m43.9s. - { - analysisLimit := cfg.AnalysisLimit() - if analysisLimit < 0 { - log.Printf("[analyze] planner statistics refresh disabled (db.analysisLimit=%d)", analysisLimit) - } else { - go func() { - // Before the stagger, and only on a database that has never been - // analyzed: the stagger is a 2 minute window in which the first - // query would otherwise run on no statistics at all. A restart - // finds sqlite_stat1 already in the file and skips this. - store.EnsurePlannerStats(analysisLimit) - // Each refresh waits until the previous one is 24h old, also - // across restarts (#2146): a restart no longer pays a cold - // ANALYZE for statistics that are hours old at most. - for { - wait := nextPlannerStatsRefresh(store.lastPlannerStatsRefresh(), time.Now()) - log.Printf("[analyze] next planner statistics refresh in %v", wait.Round(time.Minute)) - time.Sleep(wait) - if !store.RefreshPlannerStats(analysisLimit) { - // Not recorded, so the next wait would be the 2 minute - // stagger; keep the daily rhythm instead of retrying. - time.Sleep(plannerStatsInterval) + // Daily planner statistics refresh (#2058), in two parts. + // + // The routine refresh runs 24 hours after the previous one, recorded next to + // the database (#2146), and is staggered at least 2 minutes past startup for the same reason + // as the checkpoint above: it takes the write lock, and by then the initial + // ingest burst has passed, so it also sees the rows that burst added. + // + // The build in front of it deliberately does compete with that burst, because + // a database with no statistics at all has nothing better to offer the queries + // arriving in those 2 minutes. It only runs once per database; see + // Store.EnsurePlannerStats, which also carries what that costs. + // + // Bounded by analysis_limit either way, so neither grows with the file the way + // an unbounded ANALYZE does: 2.0s against 242.9s on a 9.4 GB database, both + // timed warm. Cold, on a first start, it is 3m43.9s. + { + analysisLimit := cfg.AnalysisLimit() + if analysisLimit < 0 { + log.Printf("[analyze] planner statistics refresh disabled (db.analysisLimit=%d)", analysisLimit) + } else { + go func() { + // Before the stagger, and only on a database that has never been + // analyzed: the stagger is a 2 minute window in which the first + // query would otherwise run on no statistics at all. A restart + // finds sqlite_stat1 already in the file and skips this. + store.EnsurePlannerStats(analysisLimit) + // Each refresh waits until the previous one is 24h old, also + // across restarts (#2146): a restart no longer pays a cold + // ANALYZE for statistics that are hours old at most. + for { + wait := nextPlannerStatsRefresh(store.lastPlannerStatsRefresh(), time.Now()) + log.Printf("[analyze] next planner statistics refresh in %v", wait.Round(time.Minute)) + time.Sleep(wait) + if !store.RefreshPlannerStats(analysisLimit) { + // Not recorded, so the next wait would be the 2 minute + // stagger; keep the daily rhythm instead of retrying. + time.Sleep(plannerStatsInterval) + } } - } - }() - log.Printf("[analyze] planner statistics refresh scheduled every 24h (analysis_limit=%d)", analysisLimit) + }() + log.Printf("[analyze] planner statistics refresh scheduled every 24h (analysis_limit=%d)", analysisLimit) + } + } + } // Daily neighbor_edges retention (#1287 — moved from cmd/server). @@ -712,7 +744,6 @@ func main() { } statsTicker.Stop() pruneQueueTicker.Stop() - walCheckpointTicker.Stop() stopWatchdog() // A deploy is a SIGTERM, which is exactly the case that used to lose // the tally: save before the process goes away rather than leaving up diff --git a/cmd/ingestor/main_test.go b/cmd/ingestor/main_test.go index df57c7255..dfed98948 100644 --- a/cmd/ingestor/main_test.go +++ b/cmd/ingestor/main_test.go @@ -125,7 +125,7 @@ func newTestStore(t *testing.T) *Store { t.Helper() dir := t.TempDir() dbPath := dir + "/test.db" - s, err := OpenStore(dbPath) + s, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatal(err) } @@ -1135,7 +1135,7 @@ func TestHandleMessageAdvert_EmptyScopeSkipsDefaultScopeUpdate(t *testing.T) { // Pre-seed the node with a non-empty default_scope so we can detect an // erroneous overwrite with "". - if _, err := store.db.Exec(`INSERT INTO nodes (public_key, name, default_scope) VALUES (?, 'Node1', '#belgium')`, pubkey); err != nil { + if _, err := store.db.Exec(`INSERT INTO nodes (public_key, name, default_scope) VALUES ($1, 'Node1', '#belgium')`, pubkey); err != nil { t.Fatalf("seed node: %v", err) } @@ -1147,7 +1147,7 @@ func TestHandleMessageAdvert_EmptyScopeSkipsDefaultScopeUpdate(t *testing.T) { handleMessage(store, "test", source, msg, nil, nil, &Config{}) var got sql.NullString - if err := store.db.QueryRow(`SELECT default_scope FROM nodes WHERE public_key = ?`, pubkey).Scan(&got); err != nil { + if err := store.db.QueryRow(`SELECT default_scope FROM nodes WHERE public_key = $1`, pubkey).Scan(&got); err != nil { t.Fatalf("read default_scope: %v", err) } if !got.Valid || got.String != "#belgium" { @@ -1188,7 +1188,7 @@ func TestHandleMessageAdvert_MatchedScopeUpdatesDefaultScope(t *testing.T) { code1 := fmt.Sprintf("%02X%02X", byte(code&0xFF), byte(code>>8)) rawHex := "10" + code1 + "000000" + advertBytes - if _, err := store.db.Exec(`INSERT INTO nodes (public_key, name, default_scope) VALUES (?, 'Node1', '#old')`, pubkey); err != nil { + if _, err := store.db.Exec(`INSERT INTO nodes (public_key, name, default_scope) VALUES ($1, 'Node1', '#old')`, pubkey); err != nil { t.Fatalf("seed node: %v", err) } @@ -1199,7 +1199,7 @@ func TestHandleMessageAdvert_MatchedScopeUpdatesDefaultScope(t *testing.T) { handleMessage(store, "test", source, msg, nil, regionKeySetFromKeys(map[string][]byte{"#de": regionKey}), &Config{}) var got sql.NullString - if err := store.db.QueryRow(`SELECT default_scope FROM nodes WHERE public_key = ?`, pubkey).Scan(&got); err != nil { + if err := store.db.QueryRow(`SELECT default_scope FROM nodes WHERE public_key = $1`, pubkey).Scan(&got); err != nil { t.Fatalf("read default_scope: %v", err) } if !got.Valid || got.String != "#de" { diff --git a/cmd/ingestor/maintenance.go b/cmd/ingestor/maintenance.go index 6ee0fe77d..43cbb73a6 100644 --- a/cmd/ingestor/maintenance.go +++ b/cmd/ingestor/maintenance.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "log" + "runtime" "time" "github.com/meshcore-analyzer/dbschema" @@ -34,18 +35,15 @@ const pruneBatchTransmissions = 250 // set: nothing modifies `transmissions` between them inside the transaction. // // The ORDER BY must be satisfiable from idx_transmissions_first_seen. That -// index carries the rowid as its tiebreaker, so "first_seen, id" is walked -// straight off it and the LIMIT stays deterministic even when timestamps tie. -// Ordering by id alone looks equivalent but makes SQLite abandon the index for -// a rowid SCAN. That is harmless while rows are being deleted — the oldest -// rows have the lowest rowids and match at once — but the batch that finds -// nothing, which is the steady state whenever nothing has aged out, walks the -// whole table under writerMu. TestPruneAgedTransmissionIDsUsesFirstSeenIndex -// pins the plan. -const pruneAgedTransmissionIDs = `SELECT id FROM transmissions WHERE first_seen < ? ORDER BY first_seen, id LIMIT ?` +// PostgreSQL index explicitly includes id as the tiebreaker (SQLite carried +// its rowid implicitly). This lets the LIMIT stop the ordered scan even when +// timestamps tie. Ordering by id alone could scan the entire table when no +// rows have aged out. TestPruneAgedTransmissionIDsHasOrderedIndex pins ordered +// index eligibility for both the batch query and its DELETE callers. +const pruneAgedTransmissionIDs = `SELECT id FROM transmissions WHERE first_seen < $1 ORDER BY first_seen, id LIMIT $2` -// The two statements of one prune batch. Child observations go first (no -// CASCADE in SQLite). +// The two statements of one prune batch. Child observations go first, retaining +// the bounded deletion behavior of the SQLite implementation. const ( pruneObservationsBatch = `DELETE FROM observations WHERE transmission_id IN (` + pruneAgedTransmissionIDs + `)` pruneTransmissionsBatch = `DELETE FROM transmissions WHERE id IN (` + pruneAgedTransmissionIDs + `)` @@ -67,6 +65,9 @@ const ( // returned alongside it — those rows are gone, so reporting 0 would be // wrong. func (s *Store) PruneOldPackets(days int) (int64, error) { + return s.pruneOldPackets(days, runtime.Gosched) +} +func (s *Store) pruneOldPackets(days int, betweenBatches func()) (int64, error) { if days <= 0 { return 0, nil } @@ -97,6 +98,8 @@ func (s *Store) PruneOldPackets(days int) (int64, error) { if batch < pruneBatchTransmissions { break } + // Yield only after the transaction commits and releases the writer. + betweenBatches() } if total > 0 { log.Printf("[prune] deleted %d transmissions older than %d days", total, days) @@ -117,13 +120,13 @@ func (s *Store) PruneOldClientReceptions(days int) (int64, error) { var n int64 err := s.WriterTx("prune_client_receptions", func(tx *sql.Tx) error { - res, err := tx.Exec(`DELETE FROM client_receptions WHERE rx_at < ?`, cutoff) + res, err := tx.Exec(`DELETE FROM client_receptions WHERE rx_at < `+s.parameter(1), cutoff) if err != nil { return fmt.Errorf("prune client_receptions: %w", err) } n, _ = res.RowsAffected() // Drop companion name rows not refreshed within the window. - if _, err := tx.Exec(`DELETE FROM client_observers WHERE last_seen < ?`, cutoff); err != nil { + if _, err := tx.Exec(`DELETE FROM client_observers WHERE last_seen < `+s.parameter(1), cutoff); err != nil { return fmt.Errorf("prune client_observers: %w", err) } return nil @@ -149,7 +152,7 @@ func (s *Store) PruneOldClientRxObservations(days int) (int64, error) { return 0, nil } cutoff := time.Now().UTC().AddDate(0, 0, -days).Format(rxTimeMillisLayout) - res, err := s.db.Exec(`DELETE FROM client_rx_observations WHERE rx_at < ?`, cutoff) + res, err := s.db.Exec(`DELETE FROM client_rx_observations WHERE rx_at < `+s.parameter(1), cutoff) if err != nil { return 0, fmt.Errorf("prune client_rx_observations: %w", err) } @@ -172,7 +175,7 @@ func (s *Store) PruneOldClientRfSamples(days int) (int64, error) { return 0, nil } cutoff := time.Now().UTC().AddDate(0, 0, -days).Format(rxTimeMillisLayout) - res, err := s.db.Exec(`DELETE FROM client_rf_samples WHERE sampled_at < ?`, cutoff) + res, err := s.db.Exec(`DELETE FROM client_rf_samples WHERE sampled_at < `+s.parameter(1), cutoff) if err != nil { return 0, fmt.Errorf("prune client_rf_samples: %w", err) } @@ -210,7 +213,7 @@ func (s *Store) PruneOldClientDeclaredRegions(days int) (int64, error) { // specific side of it. func (s *Store) pruneOldClientDeclaredRegionsAt(cutoffInstant time.Time) (int64, error) { cutoff := cutoffInstant.Format(rxTimeMillisLayout) - res, err := s.db.Exec(`DELETE FROM node_declared_regions WHERE observed_at < ?`, cutoff) + res, err := s.db.Exec(`DELETE FROM node_declared_regions WHERE observed_at < `+s.parameter(1), cutoff) if err != nil { return 0, fmt.Errorf("prune node_declared_regions: %w", err) } @@ -240,7 +243,7 @@ func (s *Store) PruneNeighborEdges(maxAgeDays int) (int64, error) { return 0, nil } cutoff := time.Now().UTC().Add(-time.Duration(maxAgeDays) * 24 * time.Hour).Format(time.RFC3339) - res, err := s.db.Exec("DELETE FROM neighbor_edges WHERE last_seen < ?", cutoff) + res, err := s.db.Exec("DELETE FROM neighbor_edges WHERE last_seen < "+s.parameter(1), cutoff) if err != nil { return 0, fmt.Errorf("prune neighbor_edges: %w", err) } @@ -302,7 +305,7 @@ func (s *Store) BackfillFromPubkey(chunkSize int, yieldDuration time.Duration, p } log.Printf("[backfill] from_pubkey starting: %d ADVERT rows", total) - stmt, err := s.db.Prepare("UPDATE transmissions SET from_pubkey = ? WHERE id = ?") + stmt, err := s.db.Prepare("UPDATE transmissions SET from_pubkey = " + s.parameter(1) + " WHERE id = " + s.parameter(2)) if err != nil { log.Printf("[backfill] from_pubkey prepare: %v", err) return @@ -312,7 +315,7 @@ func (s *Store) BackfillFromPubkey(chunkSize int, yieldDuration time.Duration, p var processed int64 for { rows, err := s.db.Query( - "SELECT id, decoded_json FROM transmissions WHERE from_pubkey IS NULL AND payload_type = 4 LIMIT ?", + "SELECT id, decoded_json FROM transmissions WHERE from_pubkey IS NULL AND payload_type = 4 LIMIT "+s.parameter(1), chunkSize) if err != nil { log.Printf("[backfill] from_pubkey select: %v", err) @@ -336,7 +339,7 @@ func (s *Store) BackfillFromPubkey(chunkSize int, yieldDuration time.Duration, p break } - tx, err := s.db.Begin() + tx, err := beginWrite(s.db, s.Backend()) if err != nil { log.Printf("[backfill] from_pubkey begin tx: %v", err) return diff --git a/cmd/ingestor/multibyte_persist.go b/cmd/ingestor/multibyte_persist.go index d9938da55..b24058b4f 100644 --- a/cmd/ingestor/multibyte_persist.go +++ b/cmd/ingestor/multibyte_persist.go @@ -76,7 +76,7 @@ func (s *Store) RunMultibyteCapPersist() (MultibyteCapPersistStats, error) { return stats, nil } - tx, err := s.db.Begin() + tx, err := beginWrite(s.db, s.Backend()) if err != nil { return stats, err } @@ -85,19 +85,19 @@ func (s *Store) RunMultibyteCapPersist() (MultibyteCapPersistStats, error) { // inactive_nodes. The pre-#1386 implementation issued one UPDATE // against each table per entry — 50% guaranteed-empty. We now // look up the table once, then issue the matching UPDATE. - stmtN, err := tx.Prepare(`UPDATE nodes SET multibyte_sup=?, multibyte_evidence=? WHERE public_key=?`) + stmtN, err := tx.Prepare(`UPDATE nodes SET multibyte_sup=` + s.parameter(1) + `, multibyte_evidence=` + s.parameter(2) + ` WHERE public_key=` + s.parameter(3)) if err != nil { return stats, err } defer stmtN.Close() - stmtI, err := tx.Prepare(`UPDATE inactive_nodes SET multibyte_sup=?, multibyte_evidence=? WHERE public_key=?`) + stmtI, err := tx.Prepare(`UPDATE inactive_nodes SET multibyte_sup=` + s.parameter(1) + `, multibyte_evidence=` + s.parameter(2) + ` WHERE public_key=` + s.parameter(3)) if err != nil { return stats, err } defer stmtI.Close() // Membership probe: one indexed PK lookup. Cheap; avoids the // guaranteed-miss second UPDATE. - stmtProbe, err := tx.Prepare(`SELECT 1 FROM nodes WHERE public_key=? LIMIT 1`) + stmtProbe, err := tx.Prepare(`SELECT 1 FROM nodes WHERE public_key=` + s.parameter(1) + ` LIMIT 1`) if err != nil { return stats, err } @@ -187,24 +187,9 @@ func containsCI(s, sub string) bool { // RunMultibyteCapPersist on legacy DBs that pre-date the // internal/dbschema migration (#1386). func (s *Store) hasMultibyteSupColumns() bool { - rows, err := s.db.Query(`PRAGMA table_info(nodes)`) - if err != nil { - return false - } - defer rows.Close() - for rows.Next() { - var cid int - var name, ctype string - var notnull, pk int - var dflt interface{} - if err := rows.Scan(&cid, &name, &ctype, ¬null, &dflt, &pk); err != nil { - return false - } - if name == "multibyte_sup" { - return true - } - } - return false + var found bool + _ = s.db.QueryRow(s.nativeSQL(`SELECT EXISTS(SELECT 1 FROM pragma_table_info('nodes') WHERE name='multibyte_sup')`, `SELECT EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema=current_schema() AND table_name='nodes' AND column_name='multibyte_sup')`)).Scan(&found) + return found } // multibyteStatusToInt mirrors the mapping the server used before relocation. diff --git a/cmd/ingestor/multibyte_persist_helpers_test.go b/cmd/ingestor/multibyte_persist_helpers_test.go index 5e57fd8a5..bee01ce3a 100644 --- a/cmd/ingestor/multibyte_persist_helpers_test.go +++ b/cmd/ingestor/multibyte_persist_helpers_test.go @@ -57,22 +57,9 @@ func logContains(buf *syncBuffer, substr string) bool { // columnExists reports whether the named column exists on the table. func columnExists(t *testing.T, db *sql.DB, table, col string) bool { t.Helper() - rows, err := db.Query("PRAGMA table_info(" + table + ")") - if err != nil { - t.Fatalf("PRAGMA table_info(%s): %v", table, err) + var found bool + if err := db.QueryRow(testNativeSQL(`SELECT EXISTS(SELECT 1 FROM pragma_table_info(?1) WHERE name=?2)`, `SELECT EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema=current_schema() AND table_name=$1 AND column_name=$2)`), table, col).Scan(&found); err != nil { + t.Fatal(err) } - defer rows.Close() - for rows.Next() { - var cid int - var name, ctype string - var notnull, pk int - var dfltValue sql.NullString - if err := rows.Scan(&cid, &name, &ctype, ¬null, &dfltValue, &pk); err != nil { - t.Fatalf("scan PRAGMA: %v", err) - } - if name == col { - return true - } - } - return false + return found } diff --git a/cmd/ingestor/multibyte_persist_test.go b/cmd/ingestor/multibyte_persist_test.go index 065a2381c..b270e6c4f 100644 --- a/cmd/ingestor/multibyte_persist_test.go +++ b/cmd/ingestor/multibyte_persist_test.go @@ -23,7 +23,7 @@ import ( func TestRunMultibyteCapPersist_AppliesSnapshot(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "test.db") - store, err := OpenStore(dbPath) + store, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore: %v", err) } @@ -105,7 +105,7 @@ func TestRunMultibyteCapPersist_AppliesSnapshot(t *testing.T) { func TestRunMultibyteCapPersist_NoSnapshot_NoOp(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "test.db") - store, err := OpenStore(dbPath) + store, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore: %v", err) } @@ -135,7 +135,7 @@ func TestRunMultibyteCapPersist_RoundTrip(t *testing.T) { dbPath := filepath.Join(dir, "test.db") // --- Phase 1: open store, seed, persist snapshot --- - store, err := OpenStore(dbPath) + store, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore: %v", err) } @@ -172,7 +172,7 @@ func TestRunMultibyteCapPersist_RoundTrip(t *testing.T) { } // --- Phase 2: fresh Store, verify persisted state survived --- - store2, err := OpenStore(dbPath) + store2, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore (reopen): %v", err) } @@ -211,7 +211,7 @@ func TestRunMultibyteCapPersist_RoundTrip(t *testing.T) { func TestRunMultibyteCapPersist_MalformedSnapshot(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "test.db") - store, err := OpenStore(dbPath) + store, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore: %v", err) } @@ -257,7 +257,7 @@ func TestRunMultibyteCapPersist_MalformedSnapshot(t *testing.T) { func TestRunMultibyteCapPersist_MissingSchemaColumns(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "test.db") - store, err := OpenStore(dbPath) + store, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore: %v", err) } @@ -270,7 +270,7 @@ func TestRunMultibyteCapPersist_MissingSchemaColumns(t *testing.T) { `ALTER TABLE inactive_nodes DROP COLUMN multibyte_sup`, `ALTER TABLE inactive_nodes DROP COLUMN multibyte_evidence`, } { - if _, err := store.db.Exec(stmt); err != nil { + if _, err := testAdmin(t, store).Exec(stmt); err != nil { t.Fatalf("simulate legacy DB (%q): %v", stmt, err) } } @@ -315,7 +315,7 @@ func TestRunMultibyteCapPersist_MissingSchemaColumns(t *testing.T) { func TestRunMultibyteCapPersist_PreservesConfirmedOnUnknown(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "test.db") - store, err := OpenStore(dbPath) + store, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore: %v", err) } diff --git a/cmd/ingestor/neighbor_builder.go b/cmd/ingestor/neighbor_builder.go index 7a49ecc9a..01ba7369e 100644 --- a/cmd/ingestor/neighbor_builder.go +++ b/cmd/ingestor/neighbor_builder.go @@ -20,7 +20,7 @@ const NeighborEdgesBuilderInterval = 60 * time.Second // neighborBuilderMaxBatch caps how many observation rows a single // delta tick may process (#1339). With max_open_conns=1, an unbounded -// scan on a multi-million-row table holds the SQLite write lock for +// scan on a multi-million-row table holds the writer lock for // minutes and starves MQTT ingest. The cap keeps each tick bounded; // if a backlog accumulates, successive ticks drain it 50k rows at a // time without ever blocking ingest for long. @@ -197,9 +197,9 @@ func (s *Store) buildAndPersistNeighborEdges(trust *packetpath.TrustConfig) (int FROM observations o JOIN transmissions t ON t.id = o.transmission_id LEFT JOIN observers obs ON obs.rowid = o.observer_idx - WHERE o.timestamp > ? + WHERE o.timestamp > `+s.parameter(1)+` ORDER BY o.timestamp - LIMIT ?`, watermarkEpoch, neighborBuilderMaxBatch) + LIMIT `+s.parameter(2), watermarkEpoch, neighborBuilderMaxBatch) if err != nil { return 0, fmt.Errorf("scan observations: %w", err) } @@ -254,36 +254,58 @@ func (s *Store) buildAndPersistNeighborEdges(trust *packetpath.TrustConfig) (int return 0, nil } - // Wrap the whole edge-persist tx under writer-perf instrumentation - // (#1340). Slow neighbor-builder ticks (the #1339 root cause) now - // show up on /api/perf under component=neighbor_builder. - var inserted int - err = s.WriterTx("neighbor_builder", func(tx *sql.Tx) error { - stmt, err := tx.Prepare(`INSERT INTO neighbor_edges (node_a, node_b, count, last_seen) - VALUES (?, ?, 1, ?) - ON CONFLICT(node_a, node_b) DO UPDATE SET - count = count + 1, - last_seen = MAX(last_seen, excluded.last_seen)`) - if err != nil { - return fmt.Errorf("prepare: %w", err) + // Aggregate repeated pairs before taking the writer lock. Counts and the + // latest timestamp are additive; the working set is bounded by this scan's + // candidate slice. SQL batches cap parameters and server round trips. + type countedEdge struct { + a, b, ts string + count int + } + counts := make(map[[2]string]*countedEdge) + for _, e := range edges { + key := [2]string{e.a, e.b} + if current := counts[key]; current != nil { + current.count++ + if e.ts > current.ts { + current.ts = e.ts + } + } else { + counts[key] = &countedEdge{e.a, e.b, e.ts, 1} } - defer stmt.Close() - var firstErr error - for _, e := range edges { - if _, err := stmt.Exec(e.a, e.b, e.ts); err != nil && firstErr == nil { - firstErr = err + } + err = s.WriterTx("neighbor_builder", func(tx *sql.Tx) error { + const batchSize = 1000 + values := make([]string, 0, batchSize) + args := make([]any, 0, batchSize*4) + flush := func() error { + if len(values) == 0 { + return nil } + _, err := tx.Exec(`INSERT INTO neighbor_edges(node_a,node_b,count,last_seen) VALUES `+strings.Join(values, ",")+s.nativeSQL(` + ON CONFLICT(node_a,node_b) DO UPDATE SET count=neighbor_edges.count+excluded.count, + last_seen=CASE WHEN neighbor_edges.last_seen IS NULL OR excluded.last_seen IS NULL THEN NULL ELSE MAX(neighbor_edges.last_seen,excluded.last_seen) END`, ` + ON CONFLICT(node_a,node_b) DO UPDATE SET count=neighbor_edges.count+excluded.count, + last_seen=CASE WHEN neighbor_edges.last_seen IS NULL OR excluded.last_seen IS NULL THEN NULL ELSE GREATEST(neighbor_edges.last_seen,excluded.last_seen) END`), args...) + values = values[:0] + args = args[:0] + return err } - if firstErr != nil { - return fmt.Errorf("upsert: %w", firstErr) + for _, e := range counts { + n := len(args) + values = append(values, fmt.Sprintf("(%s,%s,%s,%s)", s.parameter(n+1), s.parameter(n+2), s.parameter(n+3), s.parameter(n+4))) + args = append(args, e.a, e.b, e.count, e.ts) + if len(values) == batchSize { + if err := flush(); err != nil { + return err + } + } } - inserted = len(edges) - return nil + return flush() }) if err != nil { return 0, err } - return inserted, nil + return len(edges), nil } // canonEdge orders the pair so node_a <= node_b (matches the existing diff --git a/cmd/ingestor/neighbor_builder_delta_test.go b/cmd/ingestor/neighbor_builder_delta_test.go index 4e530953e..a60792a5f 100644 --- a/cmd/ingestor/neighbor_builder_delta_test.go +++ b/cmd/ingestor/neighbor_builder_delta_test.go @@ -29,27 +29,27 @@ func TestNeighborEdgesBuilderDeltaScan(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "delta.db") - store, err := OpenStore(dbPath) + store, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore: %v", err) } defer store.Close() if _, err := store.db.Exec( - `INSERT INTO nodes (public_key, name) VALUES (?, ?), (?, ?)`, + `INSERT INTO nodes (public_key, name) VALUES ($1, $2), ($3, $4)`, "aaaaaaaaaa", "from-node", "bbbbbbbbbb", "first-hop", ); err != nil { t.Fatal(err) } if _, err := store.db.Exec( - `INSERT INTO observers (id, name) VALUES (?, ?)`, + `INSERT INTO observers (id, name) VALUES ($1, $2)`, "obs-1", "observer-1", ); err != nil { t.Fatal(err) } var obsRowid int64 - if err := store.db.QueryRow(`SELECT rowid FROM observers WHERE id = ?`, "obs-1").Scan(&obsRowid); err != nil { + if err := store.db.QueryRow(`SELECT rowid FROM observers WHERE id = $1`, "obs-1").Scan(&obsRowid); err != nil { t.Fatal(err) } @@ -58,34 +58,53 @@ func TestNeighborEdgesBuilderDeltaScan(t *testing.T) { const baselineStartTs int64 = 1735689600 // 2025-01-01 UTC baselineMaxTs := baselineStartTs + int64(baseline) - 1 - tx, err := store.db.Begin() - if err != nil { - t.Fatal(err) - } - txStmt, err := tx.Prepare(`INSERT INTO transmissions - (raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json, from_pubkey) - VALUES ('', ?, ?, 0, ?, 0, '{}', 'aaaaaaaaaa')`) - if err != nil { - t.Fatal(err) - } - obsStmt, err := tx.Prepare(`INSERT INTO observations - (transmission_id, observer_idx, path_json, timestamp) VALUES (?, ?, '["bb"]', ?)`) - if err != nil { - t.Fatal(err) - } - for i := 0; i < baseline; i++ { - res, err := txStmt.Exec(fmt.Sprintf("h%d", i), baselineStartTs+int64(i), payloadADVERT) - if err != nil { - t.Fatal(err) + // Seed the same 100K logical rows in one transaction without per-row RPCs. + seed := func(prefix string, start int64, count int) { + t.Helper() + if store.Backend() == "sqlite" { + tx, err := store.db.Begin() + if err != nil { + t.Fatal(err) + } + defer tx.Rollback() + parents, err := tx.Prepare(`INSERT INTO transmissions(raw_hex,hash,first_seen,route_type,payload_type,payload_version,decoded_json,from_pubkey) VALUES('',?1,?2,0,?3,0,'{}','aaaaaaaaaa')`) + if err != nil { + t.Fatal(err) + } + defer parents.Close() + children, err := tx.Prepare(`INSERT INTO observations(transmission_id,observer_idx,path_json,timestamp) VALUES(?1,?2,'["bb"]',?3)`) + if err != nil { + t.Fatal(err) + } + defer children.Close() + for i := 0; i < count; i++ { + stamp := start + int64(i) + res, err := parents.Exec(fmt.Sprint(prefix, i), time.Unix(stamp, 0).UTC().Format(time.RFC3339), payloadADVERT) + if err != nil { + t.Fatal(err) + } + id, _ := res.LastInsertId() + if _, err = children.Exec(id, obsRowid, stamp); err != nil { + t.Fatal(err) + } + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + return } - txID, _ := res.LastInsertId() - if _, err := obsStmt.Exec(txID, obsRowid, baselineStartTs+int64(i)); err != nil { + + _, err := store.db.Exec(`WITH inserted AS ( + INSERT INTO transmissions(raw_hex,hash,first_seen,route_type,payload_type,payload_version,decoded_json,from_pubkey) + SELECT '',$1::text||n::text,to_char(to_timestamp($2::bigint+n) AT TIME ZONE 'UTC','YYYY-MM-DD"T"HH24:MI:SS"Z"'),0,$3,0,'{}','aaaaaaaaaa' + FROM generate_series(0,$4::integer-1) n RETURNING id,hash + ) INSERT INTO observations(transmission_id,observer_idx,path_json,timestamp) + SELECT id,$5,'["bb"]',$2::bigint+substring(hash,2)::bigint FROM inserted`, prefix, start, payloadADVERT, count, obsRowid) + if err != nil { t.Fatal(err) } } - if err := tx.Commit(); err != nil { - t.Fatal(err) - } + seed("h", baselineStartTs, baseline) // Initial warm-up: drain to completion (StartNeighborEdgesBuilder // does the same — call directly so the test doesn't depend on the @@ -136,34 +155,7 @@ func TestNeighborEdgesBuilderDeltaScan(t *testing.T) { // than baselineMaxTs. const delta = 100 deltaStartTs := baselineMaxTs + 1 - tx2, err := store.db.Begin() - if err != nil { - t.Fatal(err) - } - txStmt2, err := tx2.Prepare(`INSERT INTO transmissions - (raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json, from_pubkey) - VALUES ('', ?, ?, 0, ?, 0, '{}', 'aaaaaaaaaa')`) - if err != nil { - t.Fatal(err) - } - obsStmt2, err := tx2.Prepare(`INSERT INTO observations - (transmission_id, observer_idx, path_json, timestamp) VALUES (?, ?, '["bb"]', ?)`) - if err != nil { - t.Fatal(err) - } - for i := 0; i < delta; i++ { - res, err := txStmt2.Exec(fmt.Sprintf("d%d", i), deltaStartTs+int64(i), payloadADVERT) - if err != nil { - t.Fatal(err) - } - txID, _ := res.LastInsertId() - if _, err := obsStmt2.Exec(txID, obsRowid, deltaStartTs+int64(i)); err != nil { - t.Fatal(err) - } - } - if err := tx2.Commit(); err != nil { - t.Fatal(err) - } + seed("d", deltaStartTs, delta) deltaStart := time.Now() n3, err := store.buildAndPersistNeighborEdges(trustAllPrefixes()) @@ -184,6 +176,8 @@ func TestNeighborEdgesBuilderDeltaScan(t *testing.T) { t.Fatalf("delta build of %d rows took %v; expected <500ms. (#1339)", delta, deltaDur) } + t.Logf("100K baseline: empty delta=%s; 100-row delta=%s", noopDur, deltaDur) + // Sanity: MAX(last_seen) advanced. var maxLastSeen2 string if err := store.db.QueryRow(`SELECT MAX(last_seen) FROM neighbor_edges`).Scan(&maxLastSeen2); err != nil { diff --git a/cmd/ingestor/neighbor_builder_test.go b/cmd/ingestor/neighbor_builder_test.go index 5cd777df9..c4dead36b 100644 --- a/cmd/ingestor/neighbor_builder_test.go +++ b/cmd/ingestor/neighbor_builder_test.go @@ -21,7 +21,7 @@ func TestNeighborEdgesBuilderUpsertsFromObservations(t *testing.T) { // Open via the ingestor's normal opener so applySchema and // dbschema.Apply both run (the builder requires neighbor_edges + // observers.iata etc.). - store, err := OpenStore(dbPath) + store, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore: %v", err) } @@ -29,7 +29,7 @@ func TestNeighborEdgesBuilderUpsertsFromObservations(t *testing.T) { // Seed two nodes whose pubkey prefixes will be used as hops. if _, err := store.db.Exec( - `INSERT INTO nodes (public_key, name) VALUES (?, ?), (?, ?)`, + `INSERT INTO nodes (public_key, name) VALUES ($1, $2), ($3, $4)`, "aaaaaaaaaa", "from-node", "bbbbbbbbbb", "first-hop", ); err != nil { @@ -38,31 +38,31 @@ func TestNeighborEdgesBuilderUpsertsFromObservations(t *testing.T) { // Seed one observer. if _, err := store.db.Exec( - `INSERT INTO observers (id, name) VALUES (?, ?)`, + `INSERT INTO observers (id, name) VALUES ($1, $2)`, "obs-1", "observer-1", ); err != nil { t.Fatal(err) } var obsRowid int64 - if err := store.db.QueryRow(`SELECT rowid FROM observers WHERE id = ?`, "obs-1").Scan(&obsRowid); err != nil { + if err := store.db.QueryRow(`SELECT rowid FROM observers WHERE id = $1`, "obs-1").Scan(&obsRowid); err != nil { t.Fatal(err) } // Insert one ADVERT transmission with from_pubkey = aaaaa… - res, err := store.db.Exec( + var txID int64 + err = store.db.QueryRow( `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json, from_pubkey) - VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) RETURNING id`, "", "h1", "2026-01-01T00:00:00Z", 0, payloadADVERT, 0, "{}", "aaaaaaaaaa", - ) + ).Scan(&txID) if err != nil { t.Fatal(err) } - txID, _ := res.LastInsertId() // Insert one observation whose path[0] = "bb" (2-hex prefix unique // to bbbbb… in the nodes table). Expected edge: a↔b. if _, err := store.db.Exec( - `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp) VALUES (?, ?, ?, ?)`, + `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp) VALUES ($1, $2, $3, $4)`, txID, obsRowid, `["bb"]`, int64(1735689600), ); err != nil { t.Fatal(err) @@ -77,7 +77,7 @@ func TestNeighborEdgesBuilderUpsertsFromObservations(t *testing.T) { } var got int - if err := store.db.QueryRow(`SELECT COUNT(*) FROM neighbor_edges WHERE node_a = ? AND node_b = ?`, "aaaaaaaaaa", "bbbbbbbbbb").Scan(&got); err != nil { + if err := store.db.QueryRow(`SELECT COUNT(*) FROM neighbor_edges WHERE node_a = $1 AND node_b = $2`, "aaaaaaaaaa", "bbbbbbbbbb").Scan(&got); err != nil { t.Fatal(err) } if got != 1 { @@ -97,14 +97,14 @@ func TestNeighborEdgesBuilderUpsertsFromAnonReqEphemeralPubKey(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "build.db") - store, err := OpenStore(dbPath) + store, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore: %v", err) } defer store.Close() if _, err := store.db.Exec( - `INSERT INTO nodes (public_key, name) VALUES (?, ?), (?, ?)`, + `INSERT INTO nodes (public_key, name) VALUES ($1, $2), ($3, $4)`, "aaaaaaaaaa", "sender", "bbbbbbbbbb", "first-hop", ); err != nil { @@ -112,31 +112,31 @@ func TestNeighborEdgesBuilderUpsertsFromAnonReqEphemeralPubKey(t *testing.T) { } if _, err := store.db.Exec( - `INSERT INTO observers (id, name) VALUES (?, ?)`, + `INSERT INTO observers (id, name) VALUES ($1, $2)`, "obs-1", "observer-1", ); err != nil { t.Fatal(err) } var obsRowid int64 - if err := store.db.QueryRow(`SELECT rowid FROM observers WHERE id = ?`, "obs-1").Scan(&obsRowid); err != nil { + if err := store.db.QueryRow(`SELECT rowid FROM observers WHERE id = $1`, "obs-1").Scan(&obsRowid); err != nil { t.Fatal(err) } // ANON_REQ transmission: from_pubkey left NULL (as real ingest does — // only ADVERT populates it at write time), sender identity carried in // decoded_json.ephemeralPubKey instead. - res, err := store.db.Exec( + var txID int64 + err = store.db.QueryRow( `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) - VALUES (?, ?, ?, ?, ?, ?, ?)`, + VALUES ($1, $2, $3, $4, $5, $6, $7) RETURNING id`, "", "h2", "2026-01-01T00:00:00Z", 0, payloadAnonReq, 0, `{"ephemeralPubKey":"aaaaaaaaaa"}`, - ) + ).Scan(&txID) if err != nil { t.Fatal(err) } - txID, _ := res.LastInsertId() if _, err := store.db.Exec( - `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp) VALUES (?, ?, ?, ?)`, + `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp) VALUES ($1, $2, $3, $4)`, txID, obsRowid, `["bb"]`, int64(1735689600), ); err != nil { t.Fatal(err) @@ -151,7 +151,7 @@ func TestNeighborEdgesBuilderUpsertsFromAnonReqEphemeralPubKey(t *testing.T) { } var got int - if err := store.db.QueryRow(`SELECT COUNT(*) FROM neighbor_edges WHERE node_a = ? AND node_b = ?`, "aaaaaaaaaa", "bbbbbbbbbb").Scan(&got); err != nil { + if err := store.db.QueryRow(`SELECT COUNT(*) FROM neighbor_edges WHERE node_a = $1 AND node_b = $2`, "aaaaaaaaaa", "bbbbbbbbbb").Scan(&got); err != nil { t.Fatal(err) } if got != 1 { @@ -169,14 +169,14 @@ func TestNeighborEdgesBuilderExcludesOtherNonAdvertTypes(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "build.db") - store, err := OpenStore(dbPath) + store, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore: %v", err) } defer store.Close() if _, err := store.db.Exec( - `INSERT INTO nodes (public_key, name) VALUES (?, ?), (?, ?)`, + `INSERT INTO nodes (public_key, name) VALUES ($1, $2), ($3, $4)`, "aaaaaaaaaa", "sender", "bbbbbbbbbb", "first-hop", ); err != nil { @@ -184,29 +184,29 @@ func TestNeighborEdgesBuilderExcludesOtherNonAdvertTypes(t *testing.T) { } if _, err := store.db.Exec( - `INSERT INTO observers (id, name) VALUES (?, ?)`, + `INSERT INTO observers (id, name) VALUES ($1, $2)`, "obs-1", "observer-1", ); err != nil { t.Fatal(err) } var obsRowid int64 - if err := store.db.QueryRow(`SELECT rowid FROM observers WHERE id = ?`, "obs-1").Scan(&obsRowid); err != nil { + if err := store.db.QueryRow(`SELECT rowid FROM observers WHERE id = $1`, "obs-1").Scan(&obsRowid); err != nil { t.Fatal(err) } const payloadREQ = 2 - res, err := store.db.Exec( + var txID int64 + err = store.db.QueryRow( `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json, from_pubkey) - VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) RETURNING id`, "", "h3", "2026-01-01T00:00:00Z", 0, payloadREQ, 0, "{}", "aaaaaaaaaa", - ) + ).Scan(&txID) if err != nil { t.Fatal(err) } - txID, _ := res.LastInsertId() if _, err := store.db.Exec( - `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp) VALUES (?, ?, ?, ?)`, + `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp) VALUES ($1, $2, $3, $4)`, txID, obsRowid, `["bb"]`, int64(1735689600), ); err != nil { t.Fatal(err) @@ -217,7 +217,7 @@ func TestNeighborEdgesBuilderExcludesOtherNonAdvertTypes(t *testing.T) { } var got int - if err := store.db.QueryRow(`SELECT COUNT(*) FROM neighbor_edges WHERE node_a = ? AND node_b = ?`, "aaaaaaaaaa", "bbbbbbbbbb").Scan(&got); err != nil { + if err := store.db.QueryRow(`SELECT COUNT(*) FROM neighbor_edges WHERE node_a = $1 AND node_b = $2`, "aaaaaaaaaa", "bbbbbbbbbb").Scan(&got); err != nil { t.Fatal(err) } if got != 0 { @@ -240,30 +240,30 @@ func trustAllPrefixes() *packetpath.TrustConfig { func seedTrustFixture(t *testing.T, store *Store, hop string) { t.Helper() if _, err := store.db.Exec( - `INSERT INTO nodes (public_key, name) VALUES (?, ?), (?, ?)`, + `INSERT INTO nodes (public_key, name) VALUES ($1, $2), ($3, $4)`, "aaaaaaaaaa", "from-node", "bbbbbbbbbb", "first-hop", ); err != nil { t.Fatal(err) } - if _, err := store.db.Exec(`INSERT INTO observers (id, name) VALUES (?, ?)`, "obs-1", "observer-1"); err != nil { + if _, err := store.db.Exec(`INSERT INTO observers (id, name) VALUES ($1, $2)`, "obs-1", "observer-1"); err != nil { t.Fatal(err) } var obsRowid int64 - if err := store.db.QueryRow(`SELECT rowid FROM observers WHERE id = ?`, "obs-1").Scan(&obsRowid); err != nil { + if err := store.db.QueryRow(`SELECT rowid FROM observers WHERE id = $1`, "obs-1").Scan(&obsRowid); err != nil { t.Fatal(err) } - res, err := store.db.Exec( + var txID int64 + err := store.db.QueryRow( `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json, from_pubkey) - VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) RETURNING id`, "", "h1", "2026-01-01T00:00:00Z", 0, payloadADVERT, 0, "{}", "aaaaaaaaaa", - ) + ).Scan(&txID) if err != nil { t.Fatal(err) } - txID, _ := res.LastInsertId() if _, err := store.db.Exec( - `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp) VALUES (?, ?, ?, ?)`, + `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp) VALUES ($1, $2, $3, $4)`, txID, obsRowid, `["`+hop+`"]`, int64(1735689600), ); err != nil { t.Fatal(err) @@ -279,7 +279,7 @@ func seedTrustFixture(t *testing.T, store *Store, hop string) { // one byte) a later-joining repeater sharing that byte turns today's // "unique" resolution into a wrong edge. func TestNeighborEdgesBuilderPathTrustExcludesOneByte(t *testing.T) { - store, err := OpenStore(filepath.Join(t.TempDir(), "trust1.db")) + store, err := openPostgresTestStore(t, filepath.Join(t.TempDir(), "trust1.db")) if err != nil { t.Fatalf("OpenStore: %v", err) } @@ -297,7 +297,7 @@ func TestNeighborEdgesBuilderPathTrustExcludesOneByte(t *testing.T) { var got int if err := store.db.QueryRow( - `SELECT COUNT(*) FROM neighbor_edges WHERE node_a = ? AND node_b = ?`, + `SELECT COUNT(*) FROM neighbor_edges WHERE node_a = $1 AND node_b = $2`, "aaaaaaaaaa", "bbbbbbbbbb").Scan(&got); err != nil { t.Fatal(err) } @@ -311,7 +311,7 @@ func TestNeighborEdgesBuilderPathTrustExcludesOneByte(t *testing.T) { // so the threshold narrows the evidence base rather than disabling the // builder. func TestNeighborEdgesBuilderPathTrustAllowsTwoByte(t *testing.T) { - store, err := OpenStore(filepath.Join(t.TempDir(), "trust2.db")) + store, err := openPostgresTestStore(t, filepath.Join(t.TempDir(), "trust2.db")) if err != nil { t.Fatalf("OpenStore: %v", err) } @@ -324,7 +324,7 @@ func TestNeighborEdgesBuilderPathTrustAllowsTwoByte(t *testing.T) { var got int if err := store.db.QueryRow( - `SELECT COUNT(*) FROM neighbor_edges WHERE node_a = ? AND node_b = ?`, + `SELECT COUNT(*) FROM neighbor_edges WHERE node_a = $1 AND node_b = $2`, "aaaaaaaaaa", "bbbbbbbbbb").Scan(&got); err != nil { t.Fatal(err) } diff --git a/cmd/ingestor/observation_hops_backfill.go b/cmd/ingestor/observation_hops_backfill.go index acdf1c1e4..e37bb5b2c 100644 --- a/cmd/ingestor/observation_hops_backfill.go +++ b/cmd/ingestor/observation_hops_backfill.go @@ -7,6 +7,7 @@ import ( "log" "time" + "github.com/meshcore-analyzer/dbconfig" "github.com/meshcore-analyzer/dbschema" ) @@ -45,8 +46,16 @@ var ( // nothing, and switching it off leaves no table and no triggers behind. // days > 0 installs the index (or refreshes its triggers); // ScheduleObservationHopsBackfill then fills whatever part of the window is not -// yet covered. +// yet covered. PostgreSQL has no equivalent accelerator and keeps using its +// native reach scan; no SQLite schema or maintenance SQL may run there. func (s *Store) ConfigureObservationHops(days int) error { + if s.Backend() != dbconfig.SQLite { + s.hopIndexDays = 0 + if days > 0 { + log.Printf("[hop-index] retention.hopIndexDays is SQLite-only; PostgreSQL uses the native reach scan without a hop index") + } + return nil + } s.hopIndexDays = days if days <= 0 { start := time.Now() @@ -92,7 +101,7 @@ func (s *Store) hopFloor() int64 { // the gate, it shares the writer with ingest one short chunk at a time. Close() // cancels it; it resumes from the stored coverage mark on the next boot. func (s *Store) ScheduleObservationHopsBackfill() { - if s.hopIndexDays <= 0 { + if s.Backend() != dbconfig.SQLite || s.hopIndexDays <= 0 { return } since, err := s.hopCoverage() @@ -137,6 +146,9 @@ func (s *Store) ScheduleObservationHopsBackfill() { // would be undone by the next chunk lowering the mark again, and readers would // trust a range whose rows the trim had deleted. func (s *Store) backfillObservationHops(ctx context.Context) error { + if s.Backend() != dbconfig.SQLite { + return nil + } s.hopMaintMu.Lock() defer s.hopMaintMu.Unlock() since, err := s.hopCoverage() @@ -220,7 +232,7 @@ func (s *Store) backfillObservationHops(ctx context.Context) error { // whole range the mark moves across removes it too, instead of leaving its hop // rows under the raised mark where nothing would ever delete them. func (s *Store) TrimObservationHops() (int64, error) { - if s.hopIndexDays <= 0 { + if s.Backend() != dbconfig.SQLite || s.hopIndexDays <= 0 { return 0, nil } s.hopMaintMu.Lock() diff --git a/cmd/ingestor/observation_hops_backfill_test.go b/cmd/ingestor/observation_hops_backfill_test.go index 60828805f..8c0d41a50 100644 --- a/cmd/ingestor/observation_hops_backfill_test.go +++ b/cmd/ingestor/observation_hops_backfill_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "os" + "path/filepath" "sort" "strings" "testing" @@ -15,6 +16,17 @@ import ( const day = int64(86400) +func newSQLiteHopTestStore(t *testing.T) *Store { + t.Helper() + s, err := OpenStore(filepath.Join(t.TempDir(), "hop-index.sqlite")) + if err != nil { + t.Fatal(err) + } + s.WaitForAsyncMigrations() + t.Cleanup(func() { s.Close() }) + return s +} + func fastHopBackfill(t *testing.T, chunk int64) { t.Helper() c, p := hopBackfillChunkSeconds, hopBackfillPause @@ -135,7 +147,7 @@ func insertObs(t *testing.T, s *Store, obs ...ob) { // predate it: no hop rows, and a coverage mark claiming only the present. func legacyStore(t *testing.T, days int, obs ...ob) *Store { t.Helper() - s := newTestStore(t) + s := newSQLiteHopTestStore(t) if _, err := s.db.Exec(`INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (1,'00','h1','2026-01-01T00:00:00Z')`); err != nil { t.Fatal(err) } @@ -163,7 +175,7 @@ func restart(t *testing.T, s *Store, days int) { // The default install and an upgrade that never sets the key: no table, no // triggers, no backfill, and ingest still works. func TestHopIndex_OffByDefault_InstallsNothing(t *testing.T) { - s := newTestStore(t) + s := newSQLiteHopTestStore(t) if got := hopSchemaObjects(t, s); len(got) != 0 { t.Fatalf("OpenStore installed hop index objects: %v", got) } @@ -225,7 +237,11 @@ func TestLoadConfig_RejectsNegativeHopIndexDays(t *testing.T) { // the window. func TestHopIndex_OffRemovesEverything_OnRebuilds(t *testing.T) { fastHopBackfill(t, 3600) - now := time.Now().Unix() + fixed := time.Now().Truncate(time.Second) + clock := hopNow + t.Cleanup(func() { hopNow = clock }) + hopNow = func() time.Time { return fixed } + now := fixed.Unix() s := legacyStore(t, 14, ob{`["AA","BB"]`, now - 2*day}) restart(t, s, 14) checkHopIndex(t, s) @@ -498,7 +514,7 @@ func TestHopIndex_ShippedDefaultsAreOff(t *testing.T) { t.Errorf("%s: hopIndexDays = %d, want 0 (opt-in)", path, d) } } - s := newTestStore(t) + s := newSQLiteHopTestStore(t) restart(t, s, -1) if got := hopSchemaObjects(t, s); len(got) != 0 { t.Fatalf("hopIndexDays -1 installed %v", got) diff --git a/cmd/ingestor/observation_hops_postgres_test.go b/cmd/ingestor/observation_hops_postgres_test.go new file mode 100644 index 000000000..06514c30c --- /dev/null +++ b/cmd/ingestor/observation_hops_postgres_test.go @@ -0,0 +1,70 @@ +package main + +import ( + "bytes" + "context" + "log" + "os" + "strings" + "testing" + "time" +) + +// The optional accelerator uses SQLite JSON triggers. PostgreSQL must keep +// ingesting with either setting and leave reach queries on their native scan. +func TestPostgresHopIndexConfigurationUsesScanFallback(t *testing.T) { + if os.Getenv("CORESCOPE_TEST_BACKEND") != "postgres" { + t.Skip("requires the explicit native PostgreSQL matrix") + } + s := newTestStore(t) + var diagnostic bytes.Buffer + old := log.Writer() + log.SetOutput(&diagnostic) + t.Cleanup(func() { log.SetOutput(old) }) + for _, days := range []int{0, 7, 0} { + if err := s.ConfigureObservationHops(days); err != nil { + t.Fatalf("PostgreSQL startup with hopIndexDays=%d: %v", days, err) + } + if s.hopIndexDays != 0 { + t.Fatal("PostgreSQL claimed an enabled SQLite accelerator") + } + s.ScheduleObservationHopsBackfill() + if s.hopBackfillCancel != nil { + t.Fatal("SQLite hop backfill was scheduled on PostgreSQL") + } + if n, err := s.TrimObservationHops(); err != nil || n != 0 { + t.Fatal("PostgreSQL ran SQLite hop maintenance", n, err) + } + } + if !strings.Contains(diagnostic.String(), "retention.hopIndexDays is SQLite-only; PostgreSQL uses the native reach scan") { + t.Fatal("unsupported accelerator setting was silently ignored") + } + // Maintenance entrypoints enforce the backend independently of startup. + s.hopIndexDays = 7 + s.ScheduleObservationHopsBackfill() + if s.hopBackfillCancel != nil { + t.Fatal("PostgreSQL scheduled SQLite backfill") + } + if n, err := s.TrimObservationHops(); err != nil || n != 0 { + t.Fatal("PostgreSQL entered SQLite trim", n, err) + } + if err := s.backfillObservationHops(context.Background()); err != nil { + t.Fatal("PostgreSQL entered SQLite backfill", err) + } + s.hopIndexDays = 0 + var count int + if err := s.db.QueryRow(`SELECT count(*) FROM pg_class WHERE relnamespace=current_schema()::regnamespace AND relname IN ('observation_hops','observation_hops_coverage')`).Scan(&count); err != nil || count != 0 { + t.Fatal("PostgreSQL acquired a false hop coverage claim", count, err) + } + if _, err := s.db.Exec(`INSERT INTO transmissions(id,raw_hex,hash,first_seen) VALUES(1,'00','hop-fixture','raw timestamp')`); err != nil { + t.Fatal(err) + } + for i, path := range []string{`["aB","CD"]`, `{malformed`, ``} { + if _, err := s.db.Exec(`INSERT INTO observations(transmission_id,observer_idx,path_json,timestamp) VALUES(1,$1,$2,$3)`, i, path, time.Now().Unix()); err != nil { + t.Fatal("PostgreSQL ingest fallback rejected raw path", err) + } + } + if err := s.db.QueryRow(`SELECT count(*) FROM observations`).Scan(&count); err != nil || count != 3 { + t.Fatal("PostgreSQL ingest lost observations", count, err) + } +} diff --git a/cmd/ingestor/observer_lastseen_1465_test.go b/cmd/ingestor/observer_lastseen_1465_test.go index dedd7ab76..9c3809b4f 100644 --- a/cmd/ingestor/observer_lastseen_1465_test.go +++ b/cmd/ingestor/observer_lastseen_1465_test.go @@ -31,7 +31,7 @@ func TestStatusMessage_ObserverLastSeen_AlwaysIngestTime_PastEnvelope_1465(t *te after := time.Now().Unix() var lastSeen string - if err := store.db.QueryRow(`SELECT last_seen FROM observers WHERE id = ?`, "obs-past").Scan(&lastSeen); err != nil { + if err := store.db.QueryRow(`SELECT last_seen FROM observers WHERE id = $1`, "obs-past").Scan(&lastSeen); err != nil { t.Fatalf("scan last_seen: %v", err) } ls, err := time.Parse(time.RFC3339, lastSeen) @@ -61,7 +61,7 @@ func TestStatusMessage_ObserverLastSeen_AlwaysIngestTime_FutureEnvelope_1465(t * after := time.Now().Unix() var lastSeen string - if err := store.db.QueryRow(`SELECT last_seen FROM observers WHERE id = ?`, "obs-future").Scan(&lastSeen); err != nil { + if err := store.db.QueryRow(`SELECT last_seen FROM observers WHERE id = $1`, "obs-future").Scan(&lastSeen); err != nil { t.Fatalf("scan last_seen: %v", err) } ls, err := time.Parse(time.RFC3339, lastSeen) @@ -94,7 +94,7 @@ func TestPacketMessage_ObserverLastSeen_AlwaysIngestTime_PastEnvelope_1465(t *te after := time.Now().Unix() var lastSeen string - if err := store.db.QueryRow(`SELECT last_seen FROM observers WHERE id = ?`, "obs-pkt").Scan(&lastSeen); err != nil { + if err := store.db.QueryRow(`SELECT last_seen FROM observers WHERE id = $1`, "obs-pkt").Scan(&lastSeen); err != nil { t.Fatalf("scan last_seen: %v", err) } ls, err := time.Parse(time.RFC3339, lastSeen) diff --git a/cmd/ingestor/observer_purge_test.go b/cmd/ingestor/observer_purge_test.go index 8a3f6a6a6..5646ba7a2 100644 --- a/cmd/ingestor/observer_purge_test.go +++ b/cmd/ingestor/observer_purge_test.go @@ -13,11 +13,11 @@ func staleObserver(t *testing.T, s *Store, id string) int64 { t.Fatal(err) } old := time.Now().UTC().AddDate(0, 0, -90).Format(time.RFC3339) - if _, err := s.db.Exec(`UPDATE observers SET last_seen = ?, inactive = 1 WHERE id = ?`, old, id); err != nil { + if _, err := s.db.Exec(`UPDATE observers SET last_seen = $1, inactive = 1 WHERE id = $2`, old, id); err != nil { t.Fatal(err) } var rowid int64 - if err := s.db.QueryRow(`SELECT rowid FROM observers WHERE id = ?`, id).Scan(&rowid); err != nil { + if err := s.db.QueryRow(`SELECT rowid FROM observers WHERE id = $1`, id).Scan(&rowid); err != nil { t.Fatal(err) } return rowid @@ -26,7 +26,7 @@ func staleObserver(t *testing.T, s *Store, id string) int64 { func observerExists(t *testing.T, s *Store, id string) bool { t.Helper() var n int - if err := s.db.QueryRow(`SELECT COUNT(*) FROM observers WHERE id = ?`, id).Scan(&n); err != nil { + if err := s.db.QueryRow(`SELECT COUNT(*) FROM observers WHERE id = $1`, id).Scan(&n); err != nil { t.Fatal(err) } return n > 0 @@ -56,13 +56,13 @@ func TestPurgeStaleObserversKeepsObserverWithObservations(t *testing.T) { rowid := staleObserver(t, store, "obs-referenced") if _, err := store.db.Exec( - `INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (1, 'AA', 'h1', ?)`, + `INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (1, 'AA', 'h1', $1)`, time.Now().UTC().Format(time.RFC3339), ); err != nil { t.Fatal(err) } if _, err := store.db.Exec( - `INSERT INTO observations (transmission_id, observer_idx, timestamp) VALUES (1, ?, 0)`, rowid, + `INSERT INTO observations (transmission_id, observer_idx, timestamp) VALUES (1, $1, 0)`, rowid, ); err != nil { t.Fatal(err) } @@ -95,7 +95,7 @@ func TestPurgeStaleObserversKeepsObserverWithMetrics(t *testing.T) { staleObserver(t, store, "obs-metrics") if _, err := store.db.Exec( - `INSERT INTO observer_metrics (observer_id, timestamp) VALUES ('obs-metrics', ?)`, + `INSERT INTO observer_metrics (observer_id, timestamp) VALUES ('obs-metrics', $1)`, time.Now().UTC().Format(time.RFC3339), ); err != nil { t.Fatal(err) @@ -143,7 +143,7 @@ func TestPurgeStaleObserversKeepsActiveObserver(t *testing.T) { t.Fatal(err) } old := time.Now().UTC().AddDate(0, 0, -90).Format(time.RFC3339) - if _, err := store.db.Exec(`UPDATE observers SET last_seen = ? WHERE id = ?`, old, "obs-active"); err != nil { + if _, err := store.db.Exec(`UPDATE observers SET last_seen = $1 WHERE id = $2`, old, "obs-active"); err != nil { t.Fatal(err) } @@ -166,7 +166,7 @@ func TestPurgeStaleObserversKeepsObserverInsideWindow(t *testing.T) { } recent := time.Now().UTC().AddDate(0, 0, -20).Format(time.RFC3339) if _, err := store.db.Exec( - `UPDATE observers SET last_seen = ?, inactive = 1 WHERE id = ?`, recent, "obs-recent", + `UPDATE observers SET last_seen = $1, inactive = 1 WHERE id = $2`, recent, "obs-recent", ); err != nil { t.Fatal(err) } diff --git a/cmd/ingestor/observer_status_meta_test.go b/cmd/ingestor/observer_status_meta_test.go index a08f3db33..47a5ceeb8 100644 --- a/cmd/ingestor/observer_status_meta_test.go +++ b/cmd/ingestor/observer_status_meta_test.go @@ -53,7 +53,7 @@ func TestStatusMessageMetadataPersisted_Issue1044(t *testing.T) { } // Now drive the meta through UpsertObserver and verify the row. - s, err := OpenStore(tempDBPath(t)) + s, err := openPostgresTestStore(t, tempDBPath(t)) if err != nil { t.Fatal(err) } diff --git a/cmd/ingestor/planner_stats_schedule_test.go b/cmd/ingestor/planner_stats_schedule_test.go index 3b945b8f6..2af7209b9 100644 --- a/cmd/ingestor/planner_stats_schedule_test.go +++ b/cmd/ingestor/planner_stats_schedule_test.go @@ -4,6 +4,8 @@ import ( "os" "testing" "time" + + "github.com/meshcore-analyzer/dbconfig" ) // #2146: the daily planner statistics refresh also ran 2 minutes after every @@ -34,6 +36,9 @@ func TestNextPlannerStatsRefresh(t *testing.T) { } func TestRefreshPlannerStatsRecordsWhen(t *testing.T) { + if testBackend(t) != dbconfig.SQLite { + t.Skip("SQLite planner maintenance only") + } s := newTestStore(t) defer s.Close() if !s.lastPlannerStatsRefresh().IsZero() { @@ -60,6 +65,9 @@ func TestRefreshPlannerStatsRecordsWhen(t *testing.T) { } func TestRefreshPlannerStatsDisabledRecordsNothing(t *testing.T) { + if testBackend(t) != dbconfig.SQLite { + t.Skip("SQLite planner maintenance only") + } s := newTestStore(t) defer s.Close() s.RefreshPlannerStats(-1) @@ -71,6 +79,9 @@ func TestRefreshPlannerStatsDisabledRecordsNothing(t *testing.T) { // When the time cannot be written next to the database, the running process // still waits the full interval instead of refreshing again 2 minutes later. func TestFailedRecordStillWaitsTheInterval(t *testing.T) { + if testBackend(t) != dbconfig.SQLite { + t.Skip("SQLite planner maintenance only") + } s := newTestStore(t) defer s.Close() // A directory where the temporary file should go makes the write fail. @@ -91,6 +102,9 @@ func TestFailedRecordStillWaitsTheInterval(t *testing.T) { // Building statistics on a new database records the time too, so the routine // refresh does not run a second ANALYZE 2 minutes later. func TestEnsurePlannerStatsRecordsWhen(t *testing.T) { + if testBackend(t) != dbconfig.SQLite { + t.Skip("SQLite planner maintenance only") + } s := newTestStore(t) defer s.Close() if !s.EnsurePlannerStats(100) { diff --git a/cmd/ingestor/postgres_config_test.go b/cmd/ingestor/postgres_config_test.go new file mode 100644 index 000000000..abaec5c01 --- /dev/null +++ b/cmd/ingestor/postgres_config_test.go @@ -0,0 +1,64 @@ +package main + +import ( + "github.com/meshcore-analyzer/dbconfig" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestPostgresConfigurationPrecedence(t *testing.T) { + for _, tc := range []struct{ name, generic, writer, want string }{ + {"config", "", "", "postgres://configured@localhost/telemetry"}, + {"generic", "postgres://generic@localhost/telemetry", "", "postgres://generic@localhost/telemetry"}, + {"writer", "postgres://generic@localhost/telemetry", "postgres://writer@localhost/telemetry", "postgres://writer@localhost/telemetry"}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Setenv("CORESCOPE_DATABASE_URL", tc.generic) + t.Setenv("CORESCOPE_WRITER_DATABASE_URL", tc.writer) + t.Setenv("CORESCOPE_STATE_DIR", "private-state") + t.Setenv("CORESCOPE_APPROVED_CHANNELS_DATABASE_URL", "postgres://channels@localhost/accounts") + path := filepath.Join(t.TempDir(), "config.json") + if err := os.WriteFile(path, []byte(`{"databaseURL":"postgres://configured@localhost/telemetry","dbPath":"legacy.db"}`), 0600); err != nil { + t.Fatal(err) + } + cfg, err := LoadConfig(path) + if err != nil { + t.Fatal(err) + } + if cfg.DatabaseURL != tc.want || cfg.StateDir != "private-state" || cfg.ApprovedChannelsURL() != "postgres://channels@localhost/accounts" { + t.Fatal("PostgreSQL overrides did not resolve correctly") + } + }) + } +} + +func TestSelectedPostgresCannotFallBackToSQLiteFiles(t *testing.T) { + path := filepath.Join(t.TempDir(), "missing", "database.db") + if s, err := OpenStoreStorage(dbconfig.Storage{Backend: dbconfig.Postgres, DBPath: path}, nil); err == nil { + s.Close() + t.Fatal("missing PostgreSQL URL fell back to SQLite") + } + if _, err := os.Stat(filepath.Dir(path)); !os.IsNotExist(err) { + t.Fatal("legacy path created a directory") + } +} + +func TestOpenStoreConnectionErrorDoesNotLeakCredentials(t *testing.T) { + _, err := OpenStore("postgres://reader:do-not-log-this@localhost:1/test?sslmode=disable") + if err == nil { + t.Fatal("closed port unexpectedly connected") + } + if strings.Contains(err.Error(), "do-not-log-this") || strings.Contains(err.Error(), "postgres://") { + t.Fatal("connection error leaked credentials") + } +} + +func TestStatsDefaultUsesStateDirectory(t *testing.T) { + t.Setenv("CORESCOPE_INGESTOR_STATS", "") + t.Setenv("CORESCOPE_STATE_DIR", "private-state") + if got := statsFilePath(); got != filepath.Join("private-state", "ingestor-stats.json") { + t.Fatalf("stats path %q", got) + } +} diff --git a/cmd/ingestor/postgres_test.go b/cmd/ingestor/postgres_test.go new file mode 100644 index 000000000..dcb997fff --- /dev/null +++ b/cmd/ingestor/postgres_test.go @@ -0,0 +1,189 @@ +package main + +import ( + "context" + "database/sql" + "net/url" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/pgutil" + "github.com/meshcore-analyzer/pgutil/pgtest" +) + +var testDatabases sync.Map + +func openPostgresTestStore(t testing.TB, key string) (*Store, error) { + return openPostgresTestStoreInterval(t, key, 300) +} +func openPostgresTestStoreInterval(t testing.TB, key string, interval int) (*Store, error) { + if testBackend(t) == dbconfig.SQLite { + return OpenStoreWithInterval(key, interval) + } + stateDir := filepath.Dir(key) + if strings.HasPrefix(key, "postgres") { + stateDir = t.TempDir() + } + return OpenStoreWithState(testPostgresURL(t, key), stateDir, interval) +} + +// A historical fixture path is only a test key. Runtime OpenStore accepts +// PostgreSQL URLs exclusively, with a real restricted writer credential. +func testPostgresURL(t testing.TB, key string) string { + if testBackend(t) != dbconfig.Postgres { + t.Skip("PostgreSQL matrix not selected") + } + t.Helper() + if strings.HasPrefix(key, "postgres://") || strings.HasPrefix(key, "postgresql://") { + return key + } + cacheKey := t.Name() + "|" + key + if v, ok := testDatabases.Load(cacheKey); ok { + return v.(string) + } + owner := pgtest.NewSchema(t) + db, err := pgutil.Open(owner, false) + if err != nil { + t.Fatal(err) + } + if err = dbschema.Apply(db, nil); err != nil { + db.Close() + t.Fatal(err) + } + u, _ := url.Parse(owner) + schema := u.Query().Get("search_path") + role := schema + "_writer" + for _, stmt := range []string{`CREATE ROLE ` + role + ` LOGIN PASSWORD '` + role + `'`, + `GRANT USAGE ON SCHEMA ` + schema + ` TO ` + role, + `GRANT SELECT,INSERT,UPDATE,DELETE ON ALL TABLES IN SCHEMA ` + schema + ` TO ` + role, + `GRANT USAGE,SELECT ON ALL SEQUENCES IN SCHEMA ` + schema + ` TO ` + role, + `REVOKE INSERT,UPDATE,DELETE ON ` + schema + `.corescope_schema FROM ` + role} { + if _, err = db.Exec(stmt); err != nil { + db.Close() + t.Fatal(err) + } + } + u.User = url.UserPassword(role, role) + dsn := u.String() + testDatabases.Store(cacheKey, dsn) + t.Cleanup(func() { + testDatabases.Delete(cacheKey) + defer db.Close() + if _, err := db.Exec(`DROP OWNED BY ` + role); err != nil { + t.Error(err) + } + if _, err := db.Exec(`DROP ROLE ` + role); err != nil { + t.Error(err) + } + }) + return dsn +} + +func testAdmin(t testing.TB, s *Store) *sql.DB { + if s.Backend() == dbconfig.SQLite { + return s.db + } + t.Helper() + var schema string + if err := s.db.QueryRow(`SELECT current_schema()`).Scan(&schema); err != nil { + t.Fatal(err) + } + u, _ := url.Parse(os.Getenv("CORESCOPE_TEST_POSTGRES_URL")) + q := u.Query() + q.Set("search_path", schema) + u.RawQuery = q.Encode() + db, err := pgutil.Open(u.String(), false) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { db.Close() }) + return db +} + +func TestPostgresPacketCommitOrdering(t *testing.T) { + s, err := OpenStore(testPostgresURL(t, "cursor")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + owner := testAdmin(t, s) + tx, err := beginWrite(owner) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback() + var earlier int64 + if err = tx.QueryRow(`INSERT INTO transmissions(raw_hex,hash,first_seen) VALUES('AA','delayed','2026-01-01T00:00:00Z') RETURNING id`).Scan(&earlier); err != nil { + t.Fatal(err) + } + done := make(chan error, 1) + go func() { + _, e := s.InsertTransmission(&PacketData{Hash: "following", RawHex: "BB", Timestamp: "2026-01-01T00:00:01Z"}) + done <- e + }() + select { + case err := <-done: + t.Fatalf("later ingest passed unfinished transaction: %v", err) + case <-time.After(75 * time.Millisecond): + } + if err = tx.Commit(); err != nil { + t.Fatal(err) + } + select { + case err := <-done: + if err != nil { + t.Fatal(err) + } + case <-time.After(3 * time.Second): + t.Fatal("writer did not resume") + } + var later int64 + if err = s.db.QueryRow(`SELECT id FROM transmissions WHERE hash='following'`).Scan(&later); err != nil { + t.Fatal(err) + } + if later <= earlier { + t.Fatalf("cursor order %d then %d", earlier, later) + } +} + +func TestPostgresRuntimeCannotMigrate(t *testing.T) { + s, err := OpenStore(testPostgresURL(t, "roles")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + for _, q := range []string{`CREATE TABLE forbidden(id int)`, `UPDATE corescope_schema SET ready=false`, `DROP TABLE transmissions`} { + if _, err = s.db.Exec(q); err == nil { + t.Errorf("runtime role allowed %s", q) + } + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if err = s.db.PingContext(ctx); err != nil { + t.Fatal(err) + } +} + +func TestPostgresNaiveClockUpsert(t *testing.T) { + s := newTestStore(t) + now := time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC) + for i, at := range []time.Time{now, now.Add(time.Hour), now.Add(48 * time.Hour)} { + if err := s.RecordNaiveSkew("clock-observer", -90, at); err != nil { + t.Fatal(err) + } + var count, skew int + if err := s.db.QueryRow(`SELECT clock_skew_count_24h,clock_skew_seconds FROM observers WHERE id='clock-observer'`).Scan(&count, &skew); err != nil { + t.Fatal(err) + } + want := []int{1, 2, 1}[i] + if count != want || skew != -90 { + t.Fatalf("event %d: count=%d skew=%d", i, count, skew) + } + } +} diff --git a/cmd/ingestor/pragma_parity_test.go b/cmd/ingestor/pragma_parity_test.go index e90e4d97a..bfa5c48bc 100644 --- a/cmd/ingestor/pragma_parity_test.go +++ b/cmd/ingestor/pragma_parity_test.go @@ -1,55 +1,24 @@ package main -import ( - "path/filepath" - "testing" -) +import "testing" -// TestOpenStorePragmas pins the pragmas the writer connection runs with. -// -// These were silently at risk during the modernc.org/sqlite → -// github.com/mattn/go-sqlite3 migration in two different ways: -// -// - The DSN dialect changed. modernc understood only `_pragma=name(value)`; -// mattn understands only `_`-prefixed parameters and ignores the other -// form outright. A driver-only rename would have dropped every pragma on -// the floor with no error anywhere. -// -// - mattn runs `PRAGMA synchronous = NORMAL` unconditionally, defaulting the -// mode itself to NORMAL rather than leaving SQLite's own FULL in place. So -// the writer would have quietly moved from FULL (2) to NORMAL (1), -// weakening durability under power loss. `_synchronous=FULL` in the DSN is -// what holds it — this test is what notices if it is ever dropped. -// -// Read through the store's own connection on purpose: a separate connection, or -// the log line in OpenStoreWithInterval, would prove nothing about what the -// writer is actually using. -func TestOpenStorePragmas(t *testing.T) { - store, err := OpenStore(filepath.Join(t.TempDir(), "pragma.db")) - if err != nil { - t.Fatalf("OpenStore: %v", err) +// Durable settings are measured through the runtime connection, not logs. +func TestOpenStoreDurability(t *testing.T) { + s := newTestStore(t) + settings := map[string]string{"SHOW fsync": "on", "SHOW full_page_writes": "on", "SHOW synchronous_commit": "on"} + if s.Backend() == "sqlite" { + settings = map[string]string{"PRAGMA journal_mode": "wal", "PRAGMA synchronous": "2", "PRAGMA foreign_keys": "1"} } - defer store.db.Close() - - for _, want := range []struct { - pragma string - value string - why string - }{ - {"journal_mode", "wal", "concurrent reader (cmd/server) requires WAL"}, - {"synchronous", "2", "FULL; mattn defaults to 1 (NORMAL) unless the DSN says otherwise"}, - {"auto_vacuum", "2", "INCREMENTAL; maintenance.go drives incremental_vacuum"}, - {"foreign_keys", "1", "schema relies on FK enforcement"}, - {"busy_timeout", "5000", "writer serialises behind the reader"}, - {"cache_size", "-2000", "2 MiB of C-allocated page cache, pinned: it sits outside GOMEMLIMIT"}, - } { + for setting, want := range settings { var got string - if err := store.db.QueryRow("PRAGMA " + want.pragma).Scan(&got); err != nil { - t.Errorf("PRAGMA %s: %v", want.pragma, err) - continue + if err := s.db.QueryRow(setting).Scan(&got); err != nil { + t.Fatal(err) } - if got != want.value { - t.Errorf("PRAGMA %s = %q, want %q (%s)", want.pragma, got, want.value, want.why) + if got != want { + t.Errorf("%s=%s; require on", setting, got) } } + if _, err := s.db.Exec(`INSERT INTO observations(transmission_id,timestamp) VALUES(999999,1)`); err == nil { + t.Fatal("foreign key not enforced") + } } diff --git a/cmd/ingestor/prune_chunked_test.go b/cmd/ingestor/prune_chunked_test.go index a3be3261a..f3e186e2d 100644 --- a/cmd/ingestor/prune_chunked_test.go +++ b/cmd/ingestor/prune_chunked_test.go @@ -6,8 +6,6 @@ import ( "strings" "testing" "time" - - _ "github.com/mattn/go-sqlite3" ) // seedAgedTransmissions inserts n transmissions stamped `age` days in the @@ -18,22 +16,22 @@ func seedAgedTransmissions(t *testing.T, store *Store, n, obsPerTx, ageDays int) t.Helper() ts := time.Now().UTC().AddDate(0, 0, -ageDays).Format(time.RFC3339) for i := 0; i < n; i++ { - res, err := store.db.Exec( + var txID int64 + err := store.db.QueryRow( `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) - VALUES (?, ?, ?, 0, 1, 1, '{}')`, + VALUES ($1, $2, $3, 0, 1, 1, '{}') RETURNING id`, "AA", fmt.Sprintf("h%d-%d", ageDays, i), ts, - ) + ).Scan(&txID) if err != nil { t.Fatalf("seed tx %d: %v", i, err) } - txID, err := res.LastInsertId() if err != nil { t.Fatalf("seed tx %d LastInsertId: %v", i, err) } for j := 0; j < obsPerTx; j++ { if _, err := store.db.Exec( `INSERT INTO observations (transmission_id, observer_idx, direction, snr, rssi, score, path_json, timestamp) - VALUES (?, ?, 'rx', 1.0, -100, 0, '[]', ?)`, + VALUES ($1, $2, 'rx', 1.0, -100, 0, '[]', $3)`, txID, j, time.Now().Unix(), ); err != nil { t.Fatalf("seed obs %d/%d: %v", i, j, err) @@ -53,7 +51,7 @@ func countRows(t *testing.T, store *Store, table string) int { func openPruneStore(t *testing.T, name string) *Store { t.Helper() - store, err := OpenStore(filepath.Join(t.TempDir(), name)) + store, err := openPostgresTestStore(t, filepath.Join(t.TempDir(), name)) if err != nil { t.Fatalf("OpenStore: %v", err) } @@ -212,7 +210,7 @@ func TestPruneOldPacketsDisabledTakesNoWriterLock(t *testing.T) { // steady-state cost when nothing has aged out yet: a single empty batch, then // the loop exits. That batch is only cheap because the subquery is walked off // idx_transmissions_first_seen — see -// TestPruneAgedTransmissionIDsUsesFirstSeenIndex. +// TestPruneAgedTransmissionIDsHasOrderedIndex. func TestPruneOldPacketsNothingToDeleteRunsOneEmptyBatch(t *testing.T) { store := openPruneStore(t, "prune-noop.db") seedAgedTransmissions(t, store, 9, 2, 0) @@ -234,7 +232,7 @@ func TestPruneOldPacketsNothingToDeleteRunsOneEmptyBatch(t *testing.T) { } } -// TestPruneAgedTransmissionIDsUsesFirstSeenIndex pins the query plan of the +// TestPruneAgedTransmissionIDsHasOrderedIndex pins the query plan of the // batch subquery and of both statements that embed it. // // Ordering the batch by id instead of first_seen makes SQLite drop @@ -242,7 +240,7 @@ func TestPruneOldPacketsNothingToDeleteRunsOneEmptyBatch(t *testing.T) { // the terminating, nothing-left batch from ~10µs to ~73ms — once per statement, // under writerMu, in the state an instance is in whenever nothing has aged out. // Transaction counts cannot see that regression, so the plan is the assertion. -func TestPruneAgedTransmissionIDsUsesFirstSeenIndex(t *testing.T) { +func TestPruneAgedTransmissionIDsHasOrderedIndex(t *testing.T) { store := openPruneStore(t, "prune-plan.db") seedAgedTransmissions(t, store, 20, 2, 10) @@ -252,15 +250,19 @@ func TestPruneAgedTransmissionIDsUsesFirstSeenIndex(t *testing.T) { "observations delete": pruneObservationsBatch, "transmissions delete": pruneTransmissionsBatch, } { - rows, err := store.db.Query("EXPLAIN QUERY PLAN "+q, cutoff, pruneBatchTransmissions) + if testBackendValue() == "postgres" { + if _, err := store.db.Exec(`SET enable_seqscan=off; SET enable_bitmapscan=off`); err != nil { + t.Fatal(err) + } + } + rows, err := store.db.Query(testNativeSQL(`EXPLAIN QUERY PLAN `, "EXPLAIN (COSTS OFF) ")+q, cutoff, pruneBatchTransmissions) if err != nil { - t.Fatalf("%s: EXPLAIN QUERY PLAN: %v", name, err) + t.Fatalf(testNativeSQL(`%s: EXPLAIN QUERY PLAN: %v`, "%s: EXPLAIN (COSTS OFF): %v"), name, err) } var steps []string for rows.Next() { - var id, parent, notused int var detail string - if err := rows.Scan(&id, &parent, ¬used, &detail); err != nil { + if err := scanTestPlan(rows, &detail); err != nil { rows.Close() t.Fatalf("%s: scan plan row: %v", name, err) } @@ -276,11 +278,11 @@ func TestPruneAgedTransmissionIDsUsesFirstSeenIndex(t *testing.T) { if !strings.Contains(plan, "idx_transmissions_first_seen") { t.Errorf("%s: plan does not use idx_transmissions_first_seen: %s", name, plan) } - if strings.Contains(plan, "SCAN transmissions") { + if strings.Contains(plan, "Seq Scan on transmissions") { t.Errorf("%s: plan scans transmissions, so the empty terminating batch walks the whole table under writerMu: %s", name, plan) } - if strings.Contains(plan, "TEMP B-TREE") { - t.Errorf("%s: plan sorts in a temp b-tree instead of walking the index in order: %s", name, plan) + if strings.Contains(plan, "Sort Key:") { + t.Errorf("%s: ordered-index eligibility query unexpectedly requires a sort: %s", name, plan) } } } diff --git a/cmd/ingestor/prune_geofilter.go b/cmd/ingestor/prune_geofilter.go index 38f75857f..04316415d 100644 --- a/cmd/ingestor/prune_geofilter.go +++ b/cmd/ingestor/prune_geofilter.go @@ -1,7 +1,7 @@ // Package main: ingestor-side processor for prune-request marker files // written by the read-only server (see internal/prunequeue). // -// The server cannot DELETE because it opens SQLite mode=ro (#1283/#1289). +// The server cannot DELETE because it uses a restricted PostgreSQL reader (#1283/#1289). // Instead, the server writes request-.json under /prune-requests/ // and the ingestor consumes it here. package main @@ -22,7 +22,7 @@ func (s *Store) DeleteNodesByPubkeys(pubkeys []string) (int64, error) { if len(pubkeys) == 0 { return 0, nil } - // Chunk to keep statements under SQLite's variable limit (default 999). + // Bound each delete batch and the number of bind parameters. const chunk = 500 var total int64 for start := 0; start < len(pubkeys); start += chunk { @@ -31,8 +31,11 @@ func (s *Store) DeleteNodesByPubkeys(pubkeys []string) (int64, error) { end = len(pubkeys) } batch := pubkeys[start:end] - placeholders := strings.Repeat("?,", len(batch)) - placeholders = placeholders[:len(placeholders)-1] + slots := make([]string, len(batch)) + for i := range slots { + slots[i] = fmt.Sprintf("%s", s.parameter(i+1)) + } + placeholders := strings.Join(slots, ",") args := make([]interface{}, len(batch)) for i, pk := range batch { args[i] = pk @@ -59,7 +62,7 @@ func (s *Store) DeleteNodesByPubkeys(pubkeys []string) (int64, error) { } // RunPendingPruneRequests scans the prune-requests/ directory next to the -// SQLite database and processes any request-.json markers written by +// state directory and processes any request-.json markers written by // the server. Each request is honored verbatim — the server is responsible // for the TOCTOU snapshot (only pubkeys that were still outside the // geofilter at confirm time). After running DELETE, the ingestor writes diff --git a/cmd/ingestor/prune_geofilter_test.go b/cmd/ingestor/prune_geofilter_test.go index f12654c89..def2e353c 100644 --- a/cmd/ingestor/prune_geofilter_test.go +++ b/cmd/ingestor/prune_geofilter_test.go @@ -12,7 +12,7 @@ func TestRunPendingPruneRequests(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "test.db") - store, err := OpenStore(dbPath) + store, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore: %v", err) } @@ -67,7 +67,7 @@ func TestRunPendingPruneRequests(t *testing.T) { func TestRunPendingPruneRequests_EmptyQueueIsNoop(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "test.db") - store, err := OpenStore(dbPath) + store, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore: %v", err) } diff --git a/cmd/ingestor/refresh_planner_stats_test.go b/cmd/ingestor/refresh_planner_stats_test.go index e7f0aee21..a04801709 100644 --- a/cmd/ingestor/refresh_planner_stats_test.go +++ b/cmd/ingestor/refresh_planner_stats_test.go @@ -2,38 +2,100 @@ package main import ( "bytes" + "github.com/meshcore-analyzer/dbconfig" "log" "os" "path/filepath" "strings" "testing" - - "github.com/meshcore-analyzer/dbconfig" ) -// #2058: the planner has no cardinality statistics because ANALYZE has never -// run, so it picks a plain index over the partial index built for the query. -// These pin the three things that make the refresh work at all: the statement is -// one that actually writes statistics, the pragma reaches the connection, and a -// negative limit leaves the database untouched. +func TestAnalysisLimitConfigDefault_Issue2058(t *testing.T) { + cases := []struct { + name string + cfg *Config + want int + }{ + // Zero means "no limit" to SQLite, so an unset config must not be + // passed through as 0: that would turn a 2 second refresh into the + // 242.9s unbounded ANALYZE measured on the staging database. + {"no db section", &Config{}, 10000}, + {"db section, limit unset", &Config{DB: &dbconfig.DBConfig{}}, 10000}, + {"explicit limit", &Config{DB: &dbconfig.DBConfig{AnalysisLimit: 1000}}, 1000}, + {"disabled", &Config{DB: &dbconfig.DBConfig{AnalysisLimit: -1}}, -1}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := tc.cfg.AnalysisLimit(); got != tc.want { + t.Errorf("AnalysisLimit() = %d, want %d", got, tc.want) + } + }) + } +} -func hasStat1(t *testing.T, s *Store) bool { - t.Helper() - var n int - if err := s.db.QueryRow( - `SELECT count(*) FROM sqlite_master WHERE type='table' AND name='sqlite_stat1'`).Scan(&n); err != nil { - t.Fatalf("query sqlite_master: %v", err) +// The default is not a free choice: 400 and 1000 were measured to leave the +// channel-query plan unchanged on the staging database, so a well-meant edit +// back to SQLite's documented 400 would quietly return this to a no-op. +func TestAnalysisLimitDefaultIsHighEnoughToMatter_Issue2058(t *testing.T) { + const measuredIneffective = 1000 + if got := (&Config{}).AnalysisLimit(); got <= measuredIneffective { + t.Errorf("default analysis_limit is %d; %d and below were measured to leave the plan unchanged on a 9.4 GB database", + got, measuredIneffective) + } +} + +func TestAnalysisLimitSurvivesTheConfigFile_Issue2058(t *testing.T) { + // The knob is only useful if it survives the config file. The field lives in + // internal/dbconfig, so a wrong json tag there would leave the accessor + // returning the default however the operator set it. + path := filepath.Join(t.TempDir(), "config.json") + if err := os.WriteFile(path, []byte(`{"db":{"analysisLimit":123}}`), 0o600); err != nil { + t.Fatal(err) + } + cfg, err := LoadConfig(path) + if err != nil { + t.Fatalf("LoadConfig: %v", err) + } + if got := cfg.AnalysisLimit(); got != 123 { + t.Errorf("analysisLimit did not survive the config file: got %d, want 123", got) + } +} + +func TestPostgresOwnerAnalyzesTelemetry(t *testing.T) { + if testBackend(t) != dbconfig.Postgres { + t.Skip("PostgreSQL planner matrix not selected") + } + s := newTestStore(t) + owner := testAdmin(t, s) + if _, err := owner.Exec(`INSERT INTO transmissions(raw_hex,hash,first_seen) SELECT 'aa', n::text,'2026-01-01T00:00:00Z' FROM generate_series(1,1000) n`); err != nil { + t.Fatal(err) + } + if _, err := owner.Exec(`ANALYZE transmissions`); err != nil { + t.Fatal(err) + } + var estimate float64 + if err := owner.QueryRow(`SELECT reltuples FROM pg_class WHERE oid='transmissions'::regclass`).Scan(&estimate); err != nil { + t.Fatal(err) + } + if estimate != 1000 { + t.Fatalf("planner row estimate=%v, want 1000", estimate) + } +} +func TestPostgresRuntimeCannotAlterPlannerTargets(t *testing.T) { + if testBackend(t) != dbconfig.Postgres { + t.Skip("PostgreSQL planner matrix not selected") + } + s := newTestStore(t) + if _, err := s.db.Exec(`ALTER TABLE transmissions ALTER COLUMN payload_type SET STATISTICS 1000`); err == nil { + t.Fatal("runtime role changed owner planner policy") } - return n > 0 } -// This is the guard against going back to PRAGMA optimize, which is what the -// first draft of this change used. Measured against the 9.4 GB staging database: -// optimize analyzes only tables the calling connection has itself queried during -// the session, so from a maintenance call it writes nothing and sqlite_stat1 -// never appears. A fresh store here has queried nothing either, so this test -// fails on that mistake instead of passing on a no-op. func TestRefreshPlannerStatsWritesStatistics_Issue2058(t *testing.T) { + if testBackend(t) != dbconfig.SQLite { + t.Skip("SQLite planner matrix not selected") + } + s := newTestStore(t) defer s.Close() @@ -51,6 +113,10 @@ func TestRefreshPlannerStatsWritesStatistics_Issue2058(t *testing.T) { } func TestRefreshPlannerStatsAppliesTheLimit_Issue2058(t *testing.T) { + if testBackend(t) != dbconfig.SQLite { + t.Skip("SQLite planner matrix not selected") + } + s := newTestStore(t) defer s.Close() @@ -70,6 +136,10 @@ func TestRefreshPlannerStatsAppliesTheLimit_Issue2058(t *testing.T) { } func TestRefreshPlannerStatsNegativeLimitIsANoop_Issue2058(t *testing.T) { + if testBackend(t) != dbconfig.SQLite { + t.Skip("SQLite planner matrix not selected") + } + s := newTestStore(t) defer s.Close() @@ -82,6 +152,10 @@ func TestRefreshPlannerStatsNegativeLimitIsANoop_Issue2058(t *testing.T) { } func TestRefreshPlannerStatsIsRepeatable_Issue2058(t *testing.T) { + if testBackend(t) != dbconfig.SQLite { + t.Skip("SQLite planner matrix not selected") + } + s := newTestStore(t) defer s.Close() @@ -97,61 +171,11 @@ func TestRefreshPlannerStatsIsRepeatable_Issue2058(t *testing.T) { } } -func TestAnalysisLimitConfigDefault_Issue2058(t *testing.T) { - cases := []struct { - name string - cfg *Config - want int - }{ - // Zero means "no limit" to SQLite, so an unset config must not be - // passed through as 0: that would turn a 2 second refresh into the - // 242.9s unbounded ANALYZE measured on the staging database. - {"no db section", &Config{}, 10000}, - {"db section, limit unset", &Config{DB: &dbconfig.DBConfig{}}, 10000}, - {"explicit limit", &Config{DB: &dbconfig.DBConfig{AnalysisLimit: 1000}}, 1000}, - {"disabled", &Config{DB: &dbconfig.DBConfig{AnalysisLimit: -1}}, -1}, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - if got := tc.cfg.AnalysisLimit(); got != tc.want { - t.Errorf("AnalysisLimit() = %d, want %d", got, tc.want) - } - }) - } -} - -// The default is not a free choice: 400 and 1000 were measured to leave the -// channel-query plan unchanged on the staging database, so a well-meant edit -// back to SQLite's documented 400 would quietly return this to a no-op. -func TestAnalysisLimitDefaultIsHighEnoughToMatter_Issue2058(t *testing.T) { - const measuredIneffective = 1000 - if got := (&Config{}).AnalysisLimit(); got <= measuredIneffective { - t.Errorf("default analysis_limit is %d; %d and below were measured to leave the plan unchanged on a 9.4 GB database", - got, measuredIneffective) - } -} - -func TestAnalysisLimitSurvivesTheConfigFile_Issue2058(t *testing.T) { - // The knob is only useful if it survives the config file. The field lives in - // internal/dbconfig, so a wrong json tag there would leave the accessor - // returning the default however the operator set it. - path := filepath.Join(t.TempDir(), "config.json") - if err := os.WriteFile(path, []byte(`{"db":{"analysisLimit":123}}`), 0o600); err != nil { - t.Fatal(err) - } - cfg, err := LoadConfig(path) - if err != nil { - t.Fatalf("LoadConfig: %v", err) - } - if got := cfg.AnalysisLimit(); got != 123 { - t.Errorf("analysisLimit did not survive the config file: got %d, want 123", got) +func TestEnsurePlannerStatsBuildsWhenAbsent_Issue2058(t *testing.T) { + if testBackend(t) != dbconfig.SQLite { + t.Skip("SQLite planner matrix not selected") } -} - -// EnsurePlannerStats closes the 2 minute window the startup stagger opens, and -// must do so exactly once per database rather than once per restart. -func TestEnsurePlannerStatsBuildsWhenAbsent_Issue2058(t *testing.T) { s := newTestStore(t) defer s.Close() @@ -164,6 +188,10 @@ func TestEnsurePlannerStatsBuildsWhenAbsent_Issue2058(t *testing.T) { } func TestEnsurePlannerStatsSkipsWhenPresent_Issue2058(t *testing.T) { + if testBackend(t) != dbconfig.SQLite { + t.Skip("SQLite planner matrix not selected") + } + s := newTestStore(t) defer s.Close() @@ -177,6 +205,10 @@ func TestEnsurePlannerStatsSkipsWhenPresent_Issue2058(t *testing.T) { } func TestEnsurePlannerStatsRespectsDisabled_Issue2058(t *testing.T) { + if testBackend(t) != dbconfig.SQLite { + t.Skip("SQLite planner matrix not selected") + } + s := newTestStore(t) defer s.Close() @@ -188,11 +220,11 @@ func TestEnsurePlannerStatsRespectsDisabled_Issue2058(t *testing.T) { } } -// The whole design rests on this: statistics live in the file, so the boot-time -// build is a one-off per database. If a future change ever wrote them somewhere -// per-process, EnsurePlannerStats would silently run a full ANALYZE on every -// restart and this test is what says so. func TestPlannerStatsSurviveReopen_Issue2058(t *testing.T) { + if testBackend(t) != dbconfig.SQLite { + t.Skip("SQLite planner matrix not selected") + } + dir := t.TempDir() dbPath := dir + "/reopen.db" @@ -221,11 +253,11 @@ func TestPlannerStatsSurviveReopen_Issue2058(t *testing.T) { } } -// An operator watching a first deploy sees ingest stop for minutes. The warning -// is what tells them it is an ANALYZE and not a hang, so it is worth pinning: -// measured on staging, the build held the write connection 3m43.9s and the -// observations table took zero rows for four minutes. func TestEnsurePlannerStatsWarnsBeforeBuilding_Issue2058(t *testing.T) { + if testBackend(t) != dbconfig.SQLite { + t.Skip("SQLite planner matrix not selected") + } + s := newTestStore(t) defer s.Close() @@ -247,9 +279,11 @@ func TestEnsurePlannerStatsWarnsBeforeBuilding_Issue2058(t *testing.T) { } } -// The counterpart: a restart must not log the warning, or every boot looks like -// it is about to stall. func TestEnsurePlannerStatsIsQuietWhenPresent_Issue2058(t *testing.T) { + if testBackend(t) != dbconfig.SQLite { + t.Skip("SQLite planner matrix not selected") + } + s := newTestStore(t) defer s.Close() s.RefreshPlannerStats(10000) @@ -265,3 +299,13 @@ func TestEnsurePlannerStatsIsQuietWhenPresent_Issue2058(t *testing.T) { t.Errorf("warned about building on a database that already has statistics: %s", out) } } + +func hasStat1(t *testing.T, s *Store) bool { + t.Helper() + var n int + if err := s.db.QueryRow( + `SELECT count(*) FROM sqlite_master WHERE type='table' AND name='sqlite_stat1'`).Scan(&n); err != nil { + t.Fatalf("query sqlite_master: %v", err) + } + return n > 0 +} diff --git a/cmd/ingestor/region_keys.go b/cmd/ingestor/region_keys.go index 298aba930..26b9c5f41 100644 --- a/cmd/ingestor/region_keys.go +++ b/cmd/ingestor/region_keys.go @@ -74,7 +74,7 @@ func regionNameAcceptable(name string) (string, bool) { // (#1865). Always present; the column is part of the schema. // - node_declared_regions, an optional table a deployment may fill by other // means. Absent on a stock install, so its absence is not an error: the -// probe below asks sqlite_master first rather than letting "no such table" +// probe below asks information_schema first rather than letting "no such table" // abort a refresh that the first source could still answer. // // A node counts once per name however many times it declares it, so the @@ -129,7 +129,7 @@ func (s *Store) declaredRegionSources() ([]declaredRegionStat, error) { rows.Close() var present string - if err := s.db.QueryRow(`SELECT name FROM sqlite_master WHERE type='table' AND name='node_declared_regions'`).Scan(&present); err == nil && present != "" { + if err := s.db.QueryRow(s.nativeSQL("SELECT name FROM sqlite_master WHERE type='table' AND name='node_declared_regions'", `SELECT table_name FROM information_schema.tables WHERE table_schema=current_schema() AND table_name='node_declared_regions'`)).Scan(&present); err == nil && present != "" { ndr, err := s.db.Query(` WITH ranked AS ( SELECT target, observed_at, regions_csv, diff --git a/cmd/ingestor/region_keys_test.go b/cmd/ingestor/region_keys_test.go index 209e47b68..927c37050 100644 --- a/cmd/ingestor/region_keys_test.go +++ b/cmd/ingestor/region_keys_test.go @@ -397,22 +397,9 @@ func TestDeclaredRegionSourcesIgnoresWildcard(t *testing.T) { // wants the optional source has to bring it. func insertDeclaredRegionsRow(t *testing.T, s *Store, target, observedAt, regionsCSV string) { t.Helper() - if _, err := s.db.Exec(` - CREATE TABLE IF NOT EXISTS node_declared_regions ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - target TEXT NOT NULL, - rx_pubkey TEXT NOT NULL, - observed_at TEXT NOT NULL, - ingested_at TEXT NOT NULL, - regions_csv TEXT NOT NULL, - truncated INTEGER NOT NULL DEFAULT 0, - UNIQUE(target, rx_pubkey, observed_at) - )`); err != nil { - t.Fatal(err) - } if _, err := s.db.Exec( `INSERT INTO node_declared_regions (target, rx_pubkey, observed_at, ingested_at, regions_csv, truncated) - VALUES (?, 'rx', ?, ?, ?, 0)`, target, observedAt, observedAt, regionsCSV); err != nil { + VALUES ($1, 'rx', $2, $3, $4, 0)`, target, observedAt, observedAt, regionsCSV); err != nil { t.Fatal(err) } } @@ -639,7 +626,7 @@ func TestDeclaredRegionSourcesMergesBothSources(t *testing.T) { func seedNodeWithConfiguredScope(t *testing.T, s *Store, pubkey, scopes, at string) { t.Helper() if _, err := s.db.Exec( - `INSERT INTO nodes (public_key, configured_scope, configured_scope_at) VALUES (?, ?, ?)`, + `INSERT INTO nodes (public_key, configured_scope, configured_scope_at) VALUES ($1, $2, $3)`, pubkey, scopes, at); err != nil { t.Fatal(err) } diff --git a/cmd/ingestor/regions_gate_test.go b/cmd/ingestor/regions_gate_test.go index a9dde1b3b..b2e4fc10c 100644 --- a/cmd/ingestor/regions_gate_test.go +++ b/cmd/ingestor/regions_gate_test.go @@ -67,7 +67,7 @@ func TestClientRegionsGateOffDoesNotFallThroughToObserverPath(t *testing.T) { t.Fatalf("feature OFF: expected 0 node_declared_regions rows, got %d", n) } var observerRows int - if err := store.db.QueryRow(`SELECT COUNT(*) FROM observers WHERE id = ?`, testCompanionPK).Scan(&observerRows); err != nil { + if err := store.db.QueryRow(`SELECT COUNT(*) FROM observers WHERE id = $1`, testCompanionPK).Scan(&observerRows); err != nil { t.Fatal(err) } if observerRows != 0 { diff --git a/cmd/ingestor/relay_touch_test.go b/cmd/ingestor/relay_touch_test.go index 53fa23480..7bbf8869f 100644 --- a/cmd/ingestor/relay_touch_test.go +++ b/cmd/ingestor/relay_touch_test.go @@ -29,7 +29,7 @@ func seedRelayNode(t *testing.T, s *Store, pubkey, name, lastSeen string) { func nodeLastSeen(t *testing.T, s *Store, pubkey string) string { t.Helper() var ls string - if err := s.db.QueryRow(`SELECT COALESCE(last_seen,'') FROM nodes WHERE public_key=?`, pubkey).Scan(&ls); err != nil { + if err := s.db.QueryRow(`SELECT COALESCE(last_seen,'') FROM nodes WHERE public_key=$1`, pubkey).Scan(&ls); err != nil { t.Fatalf("read last_seen for %s: %v", pubkey, err) } return ls diff --git a/cmd/ingestor/resolved_path_test.go b/cmd/ingestor/resolved_path_test.go index 308868506..cbc07da79 100644 --- a/cmd/ingestor/resolved_path_test.go +++ b/cmd/ingestor/resolved_path_test.go @@ -95,7 +95,7 @@ func TestInsertTransmissionWritesResolvedPath(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "ingest.db") - store, err := OpenStore(dbPath) + store, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore: %v", err) } @@ -103,7 +103,7 @@ func TestInsertTransmissionWritesResolvedPath(t *testing.T) { // Seed nodes with unique 1-byte prefixes. if _, err := store.db.Exec( - `INSERT INTO nodes (public_key, name) VALUES (?, ?), (?, ?)`, + `INSERT INTO nodes (public_key, name) VALUES ($1, $2), ($3, $4)`, "aaaaaaaaaa", "from-node", "bbbbbbbbbb", "first-hop", ); err != nil { @@ -138,7 +138,7 @@ func TestInsertTransmissionWritesResolvedPath(t *testing.T) { var rp sql.NullString if err := store.db.QueryRow( - `SELECT resolved_path FROM observations WHERE transmission_id = (SELECT id FROM transmissions WHERE hash = ?)`, + `SELECT resolved_path FROM observations WHERE transmission_id = (SELECT id FROM transmissions WHERE hash = $1)`, "h-1547", ).Scan(&rp); err != nil { t.Fatalf("query: %v", err) @@ -374,14 +374,14 @@ func TestInsertTransmissionDoesNotClobberResolvedPathOnAllNil(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "ingest.db") - store, err := OpenStore(dbPath) + store, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatalf("OpenStore: %v", err) } defer store.Close() if _, err := store.db.Exec( - `INSERT INTO nodes (public_key, name) VALUES (?, ?), (?, ?)`, + `INSERT INTO nodes (public_key, name) VALUES ($1, $2), ($3, $4)`, "aaaaaaaaaa", "from-node", "bbbbbbbbbb", "first-hop", ); err != nil { @@ -412,7 +412,7 @@ func TestInsertTransmissionDoesNotClobberResolvedPathOnAllNil(t *testing.T) { // Sanity: first write populated resolved_path. var first sql.NullString if err := store.db.QueryRow( - `SELECT resolved_path FROM observations WHERE transmission_id = (SELECT id FROM transmissions WHERE hash = ?)`, + `SELECT resolved_path FROM observations WHERE transmission_id = (SELECT id FROM transmissions WHERE hash = $1)`, "h-clobber", ).Scan(&first); err != nil { t.Fatalf("first query: %v", err) @@ -432,7 +432,7 @@ func TestInsertTransmissionDoesNotClobberResolvedPathOnAllNil(t *testing.T) { var after sql.NullString if err := store.db.QueryRow( - `SELECT resolved_path FROM observations WHERE transmission_id = (SELECT id FROM transmissions WHERE hash = ?)`, + `SELECT resolved_path FROM observations WHERE transmission_id = (SELECT id FROM transmissions WHERE hash = $1)`, "h-clobber", ).Scan(&after); err != nil { t.Fatalf("post-reingest query: %v", err) diff --git a/cmd/ingestor/retained_status_test.go b/cmd/ingestor/retained_status_test.go index 64a7ffc47..5ad07a574 100644 --- a/cmd/ingestor/retained_status_test.go +++ b/cmd/ingestor/retained_status_test.go @@ -26,7 +26,7 @@ func seedObserver(t *testing.T, s *Store, id string, ageDays int) string { t.Fatal(err) } old := time.Now().UTC().AddDate(0, 0, -ageDays).Format(time.RFC3339) - if _, err := s.db.Exec(`UPDATE observers SET last_seen = ? WHERE id = ?`, old, id); err != nil { + if _, err := s.db.Exec(`UPDATE observers SET last_seen = $1 WHERE id = $2`, old, id); err != nil { t.Fatal(err) } return old @@ -35,7 +35,7 @@ func seedObserver(t *testing.T, s *Store, id string, ageDays int) string { func observerLastSeen(t *testing.T, s *Store, id string) string { t.Helper() var got string - if err := s.db.QueryRow(`SELECT last_seen FROM observers WHERE id = ?`, id).Scan(&got); err != nil { + if err := s.db.QueryRow(`SELECT last_seen FROM observers WHERE id = $1`, id).Scan(&got); err != nil { t.Fatal(err) } return got @@ -57,7 +57,7 @@ func TestRetainedStatusDoesNotAdvanceLastSeen(t *testing.T) { func TestRetainedStatusDoesNotReactivateInactiveObserver(t *testing.T) { store := newTestStore(t) seedObserver(t, store, "obs-retained", 60) - if _, err := store.db.Exec(`UPDATE observers SET inactive = 1 WHERE id = ?`, "obs-retained"); err != nil { + if _, err := store.db.Exec(`UPDATE observers SET inactive = 1 WHERE id = $1`, "obs-retained"); err != nil { t.Fatal(err) } @@ -66,7 +66,7 @@ func TestRetainedStatusDoesNotReactivateInactiveObserver(t *testing.T) { nil, nil, &Config{}) var inactive int - if err := store.db.QueryRow(`SELECT inactive FROM observers WHERE id = ?`, "obs-retained").Scan(&inactive); err != nil { + if err := store.db.QueryRow(`SELECT inactive FROM observers WHERE id = $1`, "obs-retained").Scan(&inactive); err != nil { t.Fatal(err) } if inactive != 1 { @@ -85,7 +85,7 @@ func TestRetainedStatusDoesNotCreateUnknownObserver(t *testing.T) { nil, nil, &Config{}) var count int - if err := store.db.QueryRow(`SELECT COUNT(*) FROM observers WHERE id = ?`, "obs-never-seen").Scan(&count); err != nil { + if err := store.db.QueryRow(`SELECT COUNT(*) FROM observers WHERE id = $1`, "obs-never-seen").Scan(&count); err != nil { t.Fatal(err) } if count != 0 { @@ -105,7 +105,7 @@ func TestRetainedStatusDoesNotInsertMetricsSample(t *testing.T) { var count int if err := store.db.QueryRow( - `SELECT COUNT(*) FROM observer_metrics WHERE observer_id = ?`, "obs-retained", + `SELECT COUNT(*) FROM observer_metrics WHERE observer_id = $1`, "obs-retained", ).Scan(&count); err != nil { t.Fatal(err) } @@ -127,7 +127,7 @@ func TestRetainedStatusStillUpdatesMetadata(t *testing.T) { var firmware string var noiseFloor float64 if err := store.db.QueryRow( - `SELECT firmware, noise_floor FROM observers WHERE id = ?`, "obs-retained", + `SELECT firmware, noise_floor FROM observers WHERE id = $1`, "obs-retained", ).Scan(&firmware, &noiseFloor); err != nil { t.Fatal(err) } @@ -162,7 +162,7 @@ func TestLiveStatusStillCreatesUnknownObserver(t *testing.T) { nil, nil, &Config{}) var count int - if err := store.db.QueryRow(`SELECT COUNT(*) FROM observers WHERE id = ?`, "obs-fresh").Scan(&count); err != nil { + if err := store.db.QueryRow(`SELECT COUNT(*) FROM observers WHERE id = $1`, "obs-fresh").Scan(&count); err != nil { t.Fatal(err) } if count != 1 { diff --git a/cmd/ingestor/retention_yield_test.go b/cmd/ingestor/retention_yield_test.go new file mode 100644 index 000000000..3ad45d8ca --- /dev/null +++ b/cmd/ingestor/retention_yield_test.go @@ -0,0 +1,103 @@ +package main + +import ( + "context" + "fmt" + "testing" + "time" +) + +func TestPruneYieldsAfterCommittedFullBatches(t *testing.T) { + s := openPruneStore(t, "yield.db") + const aged = pruneBatchTransmissions*2 + 7 + seedRetentionFanout(t, s, aged, aged) + yields := 0 + removed, err := s.pruneOldPackets(5, func() { + yields++ + if !writerMu.TryLock() { + t.Error("scheduling action still holds writerMu") + return + } + writerMu.Unlock() + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + conn, err := s.db.Conn(ctx) + if err != nil { + t.Error("scheduling action still occupies the single DB connection", err) + return + } + var remaining int + err = conn.QueryRowContext(ctx, `SELECT count(*) FROM transmissions WHERE first_seen < '2021-01-01'`).Scan(&remaining) + conn.Close() + if err != nil || remaining != aged-yields*pruneBatchTransmissions { + t.Error("scheduling action preceded a committed full batch", remaining, err) + return + } + done := make(chan error, 1) + sequence := yields + go func() { + _, err := s.InsertTransmission(&PacketData{ + RawHex: "0A00D69FD7A5A7475DB07337749AE61FA53A4788E976", + Hash: fmt.Sprint("scheduled-live-", sequence), Timestamp: time.Now().UTC().Format(time.RFC3339), + RouteType: 2, PayloadType: 2, PathJSON: "[]", DecodedJSON: "{}", + }) + done <- err + }() + select { + case err := <-done: + if err != nil { + t.Error(err) + } + case <-ctx.Done(): + t.Error("foreground ingestion could not finish between committed batches") + } + }) + if err != nil || removed != aged { + t.Fatalf("removed=%d err=%v", removed, err) + } + if yields != 2 { + t.Errorf("scheduling action called %d times; want once after each of two full committed batches", yields) + } + if countRows(t, s, "transmissions") != 2 || countRows(t, s, "observations") != 2 { + t.Error("foreground rows were not ingested and preserved between batches") + } +} + +func seedRetentionFanout(tb testing.TB, s *Store, transmissions, observations int) { + tb.Helper() + s.WaitForAsyncMigrations() + tx, err := s.db.Begin() + if err != nil { + tb.Fatal(err) + } + defer tx.Rollback() + parents, err := tx.Prepare(`INSERT INTO transmissions(id,raw_hex,hash,first_seen,last_seen) VALUES($1,'00',$2,'2020-01-01T00:00:00Z',1)`) + if err != nil { + tb.Fatal(err) + } + defer parents.Close() + children, err := tx.Prepare(`INSERT INTO observations(transmission_id,observer_idx,path_json,timestamp) VALUES($1,$2,'[]',1)`) + if err != nil { + tb.Fatal(err) + } + defer children.Close() + for i := 1; i <= transmissions; i++ { + if _, err := parents.Exec(i, fmt.Sprint("retention-", i)); err != nil { + tb.Fatal(err) + } + } + for i := 0; i < observations; i++ { + if _, err := children.Exec(i%transmissions+1, i/transmissions); err != nil { + tb.Fatal(err) + } + } + + if err := tx.Commit(); err != nil { + tb.Fatal(err) + } + if s.Backend() == "postgres" { + if _, err := testAdmin(tb, s).Exec(`SELECT setval('transmissions_id_seq',$1,true)`, transmissions); err != nil { + tb.Fatal(err) + } + } +} diff --git a/cmd/ingestor/rf_gate_test.go b/cmd/ingestor/rf_gate_test.go index 4b6f8dee7..8f05afe6f 100644 --- a/cmd/ingestor/rf_gate_test.go +++ b/cmd/ingestor/rf_gate_test.go @@ -76,7 +76,7 @@ func TestClientRfSamplesGateOffDoesNotFallThroughToObserverPath(t *testing.T) { t.Fatalf("feature OFF: expected 0 client_rf_samples rows, got %d", n) } var observerRows int - if err := store.db.QueryRow(`SELECT COUNT(*) FROM observers WHERE id = ?`, testCompanionPK).Scan(&observerRows); err != nil { + if err := store.db.QueryRow(`SELECT COUNT(*) FROM observers WHERE id = $1`, testCompanionPK).Scan(&observerRows); err != nil { t.Fatal(err) } if observerRows != 0 { diff --git a/cmd/ingestor/scope_match_tally.go b/cmd/ingestor/scope_match_tally.go index 378b563f7..813cbf029 100644 --- a/cmd/ingestor/scope_match_tally.go +++ b/cmd/ingestor/scope_match_tally.go @@ -20,20 +20,7 @@ import ( // scope_match_totals is one row. The ingestor reads it into the counters at // startup and writes them back periodically and on shutdown, which makes the // tally cumulative since `since_unix` instead of since boot. It is deliberately -// NOT in internal/dbschema: the server neither reads this table nor detects it, -// and putting it in AssertReady would make an old DB without the table fail the -// server's startup check for data the server never looks at. - -const scopeMatchTotalsSchema = ` - CREATE TABLE IF NOT EXISTS scope_match_totals ( - id INTEGER PRIMARY KEY CHECK (id = 1), - since_unix INTEGER NOT NULL, - unique_matches INTEGER NOT NULL, - explicit_over_derived INTEGER NOT NULL, - ambiguous INTEGER NOT NULL, - none_matches INTEGER NOT NULL, - updated_unix INTEGER NOT NULL - )` +// included in the canonical telemetry schema owned by the migration command. // LoadScopeMatchTotals seeds the in-process counters from the stored row so // counting continues where the previous process stopped. On a DB that has no @@ -68,7 +55,7 @@ func (s *Store) SaveScopeMatchTotals() error { _, err := s.db.Exec(` INSERT INTO scope_match_totals (id, since_unix, unique_matches, explicit_over_derived, ambiguous, none_matches, updated_unix) - VALUES (1, ?, ?, ?, ?, ?, ?) + VALUES (1, `+s.parameter(1)+`, `+s.parameter(2)+`, `+s.parameter(3)+`, `+s.parameter(4)+`, `+s.parameter(5)+`, `+s.parameter(6)+`) ON CONFLICT(id) DO UPDATE SET since_unix = excluded.since_unix, unique_matches = excluded.unique_matches, diff --git a/cmd/ingestor/scope_match_tally_test.go b/cmd/ingestor/scope_match_tally_test.go index e62292da1..3332797dd 100644 --- a/cmd/ingestor/scope_match_tally_test.go +++ b/cmd/ingestor/scope_match_tally_test.go @@ -37,7 +37,7 @@ func TestScopeMatchTallySurvivesReopen(t *testing.T) { resetScopeMatchCounters(t) dbPath := t.TempDir() + "/tally.db" - s, err := OpenStore(dbPath) + s, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatal(err) } @@ -58,7 +58,7 @@ func TestScopeMatchTallySurvivesReopen(t *testing.T) { scopeMatchCounters.none.Store(0) scopeMatchCounters.sinceUnix.Store(0) - s2, err := OpenStore(dbPath) + s2, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatal(err) } @@ -89,7 +89,7 @@ func TestScopeMatchTallyAnchorsSinceOnFirstOpen(t *testing.T) { resetScopeMatchCounters(t) before := time.Now().Unix() - s, err := OpenStore(t.TempDir() + "/tally.db") + s, err := openPostgresTestStore(t, t.TempDir()+"/tally.db") if err != nil { t.Fatal(err) } @@ -115,7 +115,7 @@ func TestScopeMatchTallyContinuesAfterRestore(t *testing.T) { resetScopeMatchCounters(t) dbPath := t.TempDir() + "/tally.db" - s, err := OpenStore(dbPath) + s, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatal(err) } @@ -126,7 +126,7 @@ func TestScopeMatchTallyContinuesAfterRestore(t *testing.T) { } s.Close() - s2, err := OpenStore(dbPath) + s2, err := openPostgresTestStore(t, dbPath) if err != nil { t.Fatal(err) } @@ -154,7 +154,7 @@ func TestScopeMatchTallyContinuesAfterRestore(t *testing.T) { func TestScopeMatchTallySaveIsIdempotent(t *testing.T) { resetScopeMatchCounters(t) - s, err := OpenStore(t.TempDir() + "/tally.db") + s, err := openPostgresTestStore(t, t.TempDir()+"/tally.db") if err != nil { t.Fatal(err) } diff --git a/cmd/ingestor/sig_validate_ingest_test.go b/cmd/ingestor/sig_validate_ingest_test.go index ab09e1fb3..2443606e4 100644 --- a/cmd/ingestor/sig_validate_ingest_test.go +++ b/cmd/ingestor/sig_validate_ingest_test.go @@ -4,8 +4,10 @@ import ( "crypto/ed25519" "encoding/binary" "encoding/hex" + "encoding/json" "strings" "testing" + "unicode/utf8" ) // buildAdvertHex constructs a full ADVERT packet hex string. @@ -47,7 +49,7 @@ func makeAppdata(name string) []byte { func TestSigValidation_ValidAdvertStored(t *testing.T) { dbPath := t.TempDir() + "/test.db" - store, err := OpenStoreWithInterval(dbPath, 300) + store, err := openPostgresTestStoreInterval(t, dbPath, 300) if err != nil { t.Fatal(err) } @@ -73,7 +75,7 @@ func TestSigValidation_ValidAdvertStored(t *testing.T) { func TestSigValidation_TamperedSignatureDropped(t *testing.T) { dbPath := t.TempDir() + "/test.db" - store, err := OpenStoreWithInterval(dbPath, 300) + store, err := openPostgresTestStoreInterval(t, dbPath, 300) if err != nil { t.Fatal(err) } @@ -138,7 +140,7 @@ func TestSigValidation_TamperedSignatureDropped(t *testing.T) { func TestSigValidation_TruncatedAppdataDropped(t *testing.T) { dbPath := t.TempDir() + "/test.db" - store, err := OpenStoreWithInterval(dbPath, 300) + store, err := openPostgresTestStoreInterval(t, dbPath, 300) if err != nil { t.Fatal(err) } @@ -166,9 +168,118 @@ func TestSigValidation_TruncatedAppdataDropped(t *testing.T) { } } +func TestSigValidation_DroppedPacketText(t *testing.T) { + cases := []struct { + name, nodeName, observerName, wantNode, wantObserver string + }{ + {"valid multibyte", strings.Repeat("é", 16), "Observer", strings.Repeat("é", 16), "Observer"}, + {"split two-byte rune", strings.Repeat("x", 31) + "é", "Observer", strings.Repeat("x", 31), "Observer"}, + {"split replacement rune", strings.Repeat("x", 30) + "\xff", "Observer", strings.Repeat("x", 30), "Observer"}, + {"split four-byte rune", strings.Repeat("x", 31) + "🌍", "Observer", strings.Repeat("x", 31), "Observer"}, + {"malformed name", "No\xffde", "Observer", "No\ufffdde", "Observer"}, + {"observer NUL", "Node", "Obs\x00erver", "Node", "Observer"}, + {"long observer", "Node", strings.Repeat("é", 129), "Node", strings.Repeat("é", maxObserverTextLen)}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + store := newTestStore(t) + pub, priv, err := ed25519.GenerateKey(nil) + if err != nil { + t.Fatal(err) + } + rawHex := buildAdvertHex(pub, priv, 1700000000, makeAppdata(tc.nodeName)) + raw, err := hex.DecodeString(rawHex) + if err != nil { + t.Fatal(err) + } + raw[38] ^= 1 // Invalidate the signature without changing the name bytes. + rawHex = hex.EncodeToString(raw) + payload, err := json.Marshal(struct { + Raw string `json:"raw"` + Origin string `json:"origin"` + }{rawHex, tc.observerName}) + if err != nil { + t.Fatal(err) + } + handleMessage(store, "test", MQTTSource{Name: "test"}, newMockMsg("meshcore/US/obs1/packet", string(payload)), nil, nil, &Config{}) + + var hash, storedRaw, nodeName, observerName string + if err := store.db.QueryRow("SELECT hash, raw_hex, node_name, observer_name FROM dropped_packets").Scan(&hash, &storedRaw, &nodeName, &observerName); err != nil { + t.Fatalf("rejected packet audit missing: %v (write errors=%d)", err, store.Stats.WriteErrors.Load()) + } + if storedRaw != rawHex || hash != ComputeContentHash(rawHex) { + t.Error("audit changed raw packet evidence or its hash") + } + if nodeName != tc.wantNode || !utf8.ValidString(nodeName) || len(nodeName) > 32 { + t.Errorf("node name=%q (%d bytes), want %q and valid UTF-8 within 32 bytes", nodeName, len(nodeName), tc.wantNode) + } + if observerName != tc.wantObserver || !utf8.ValidString(observerName) || utf8.RuneCountInString(observerName) > maxObserverTextLen { + t.Errorf("observer name=%q, want %q within observer text limit", observerName, tc.wantObserver) + } + if drops, errors := store.Stats.SignatureDrops.Load(), store.Stats.WriteErrors.Load(); drops != 1 || errors != 0 { + t.Errorf("signature drops=%d write errors=%d, want 1 and 0", drops, errors) + } + var transmissions int + if err := store.db.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&transmissions); err != nil || transmissions != 0 { + t.Fatalf("invalid signature stored as transmission: count=%d err=%v", transmissions, err) + } + }) + } +} + +func TestSigValidation_AcceptedNodeText(t *testing.T) { + pub, priv, err := ed25519.GenerateKey(nil) + if err != nil { + t.Fatal(err) + } + cases := []struct { + name, nodeName, want string + update bool + }{ + {"insert split two-byte rune", strings.Repeat("x", 31) + "é", strings.Repeat("x", 31), false}, + {"update split replacement rune", strings.Repeat("x", 30) + "\xff", strings.Repeat("x", 30), true}, + {"update split four-byte rune", strings.Repeat("x", 31) + "🌍", strings.Repeat("x", 31), true}, + {"insert valid multibyte", strings.Repeat("é", 16), strings.Repeat("é", 16), false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + store := newTestStore(t) + wantUpserts := int64(1) + if tc.update { + seedHex := buildAdvertHex(pub, priv, 1700000000, makeAppdata("Original")) + handleMessage(store, "test", MQTTSource{Name: "test"}, newMockMsg("meshcore/US/obs1/packet", `{"raw":"`+seedHex+`","origin":"Observer"}`), nil, nil, &Config{}) + if store.Stats.NodeUpserts.Load() != 1 { + t.Fatal("seed node not inserted") + } + wantUpserts++ + } + rawHex := buildAdvertHex(pub, priv, 1700000001, makeAppdata(tc.nodeName)) + msg := newMockMsg("meshcore/US/obs1/packet", `{"raw":"`+rawHex+`","origin":"Observer"}`) + handleMessage(store, "test", MQTTSource{Name: "test"}, msg, nil, nil, &Config{}) + + var nodeName, storedRaw string + if err := store.db.QueryRow("SELECT name FROM nodes WHERE public_key = "+store.parameter(1), hex.EncodeToString(pub)).Scan(&nodeName); err != nil { + t.Fatalf("accepted node missing: %v (write errors=%d)", err, store.Stats.WriteErrors.Load()) + } + if nodeName != tc.want || !utf8.ValidString(nodeName) || len(nodeName) > 32 { + t.Errorf("node name=%q (%d bytes), want %q and valid UTF-8 within 32 bytes", nodeName, len(nodeName), tc.want) + } + if err := store.db.QueryRow("SELECT raw_hex FROM transmissions WHERE hash = "+store.parameter(1), ComputeContentHash(rawHex)).Scan(&storedRaw); err != nil || storedRaw != rawHex { + t.Fatalf("accepted packet evidence changed: err=%v", err) + } + if upserts, errors := store.Stats.NodeUpserts.Load(), store.Stats.WriteErrors.Load(); upserts != wantUpserts || errors != 0 { + t.Errorf("node upserts=%d write errors=%d, want %d and 0", upserts, errors, wantUpserts) + } + if drops := store.Stats.SignatureDrops.Load(); drops != 0 { + t.Errorf("valid signature dropped: %d", drops) + } + }) + } +} + func TestSigValidation_DisabledByConfig(t *testing.T) { dbPath := t.TempDir() + "/test.db" - store, err := OpenStoreWithInterval(dbPath, 300) + store, err := openPostgresTestStoreInterval(t, dbPath, 300) if err != nil { t.Fatal(err) } @@ -204,7 +315,7 @@ func TestSigValidation_DisabledByConfig(t *testing.T) { func TestSigValidation_DropCounterIncrements(t *testing.T) { dbPath := t.TempDir() + "/test.db" - store, err := OpenStoreWithInterval(dbPath, 300) + store, err := openPostgresTestStoreInterval(t, dbPath, 300) if err != nil { t.Fatal(err) } @@ -236,7 +347,7 @@ func TestSigValidation_DropCounterIncrements(t *testing.T) { func TestSigValidation_LogContainsFields(t *testing.T) { // This test verifies the dropped_packets row has all required fields dbPath := t.TempDir() + "/test.db" - store, err := OpenStoreWithInterval(dbPath, 300) + store, err := openPostgresTestStoreInterval(t, dbPath, 300) if err != nil { t.Fatal(err) } @@ -289,15 +400,15 @@ func TestSigValidation_LogContainsFields(t *testing.T) { func TestPruneDroppedPackets(t *testing.T) { dbPath := t.TempDir() + "/test.db" - store, err := OpenStoreWithInterval(dbPath, 300) + store, err := openPostgresTestStoreInterval(t, dbPath, 300) if err != nil { t.Fatal(err) } defer store.Close() // Insert an old dropped packet - store.db.Exec(`INSERT INTO dropped_packets (hash, reason, dropped_at) VALUES ('old', 'test', datetime('now', '-60 days'))`) - store.db.Exec(`INSERT INTO dropped_packets (hash, reason, dropped_at) VALUES ('new', 'test', datetime('now'))`) + store.db.Exec(testNativeSQL(`INSERT INTO dropped_packets (hash, reason, dropped_at) VALUES ('old', 'test', datetime('now','-60 days'))`, `INSERT INTO dropped_packets (hash, reason, dropped_at) VALUES ('old', 'test', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC' - interval '60 days','YYYY-MM-DD HH24:MI:SS'))`)) + store.db.Exec(testNativeSQL(`INSERT INTO dropped_packets (hash, reason, dropped_at) VALUES ('new', 'test', datetime('now'))`, `INSERT INTO dropped_packets (hash, reason, dropped_at) VALUES ('new', 'test', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC','YYYY-MM-DD HH24:MI:SS'))`)) n, err := store.PruneDroppedPackets(30) if err != nil { diff --git a/cmd/ingestor/sqlite_maintenance.go b/cmd/ingestor/sqlite_maintenance.go new file mode 100644 index 000000000..d33ebd787 --- /dev/null +++ b/cmd/ingestor/sqlite_maintenance.go @@ -0,0 +1,215 @@ +package main + +import ( + "fmt" + "github.com/meshcore-analyzer/dbconfig" + "log" + "os" + "strings" + "time" +) + +func (s *Store) CheckAutoVacuum(cfg *Config) { + if s.Backend() != dbconfig.SQLite { + return + } + + var autoVacuum int + if err := s.db.QueryRow("PRAGMA auto_vacuum").Scan(&autoVacuum); err != nil { + log.Printf("[db] warning: could not read auto_vacuum: %v", err) + return + } + + if autoVacuum == 2 { + log.Printf("[db] auto_vacuum=INCREMENTAL") + return + } + + modes := map[int]string{0: "NONE", 1: "FULL", 2: "INCREMENTAL"} + mode := modes[autoVacuum] + if mode == "" { + mode = fmt.Sprintf("UNKNOWN(%d)", autoVacuum) + } + + log.Printf("[db] auto_vacuum=%s — DB needs one-time VACUUM to enable incremental auto-vacuum. "+ + "Set db.vacuumOnStartup: true in config to migrate (will block startup for several minutes on large DBs). "+ + "See https://github.com/Kpa-clawbot/CoreScope/issues/919", mode) + + if cfg.DB != nil && cfg.DB.VacuumOnStartup { + // WARNING: Full VACUUM creates a temporary copy of the entire DB file. + // Requires ~2× the DB file size in free disk space or it will fail. + log.Printf("[db] vacuumOnStartup=true — starting one-time full VACUUM (ensure 2x DB size free disk space)...") + start := time.Now() + + if _, err := s.instrumentedExec("vacuum", "PRAGMA auto_vacuum = INCREMENTAL"); err != nil { + log.Printf("[db] VACUUM failed: could not set auto_vacuum: %v", err) + return + } + if _, err := s.instrumentedExec("vacuum", "VACUUM"); err != nil { + log.Printf("[db] VACUUM failed: %v", err) + return + } + + elapsed := time.Since(start) + log.Printf("[db] VACUUM complete in %v — auto_vacuum is now INCREMENTAL", elapsed.Round(time.Millisecond)) + } +} + +func (s *Store) RunIncrementalVacuum(pages int) { + if s.Backend() != dbconfig.SQLite { + return + } + + // Tagged for /api/perf writer-lock visibility (#1340). + if _, err := s.instrumentedExec("vacuum", fmt.Sprintf("PRAGMA incremental_vacuum(%d)", pages)); err != nil { + log.Printf("[vacuum] incremental_vacuum error: %v", err) + } +} + +func (s *Store) RefreshPlannerStats(analysisLimit int) bool { + if s.Backend() != dbconfig.SQLite { + return false + } + + if analysisLimit < 0 { + return false + } + first := !s.hasPlannerStats() + start := time.Now() + // Tagged for /api/perf writer-lock visibility (#1340). + if _, err := s.instrumentedExec("analyze", fmt.Sprintf("PRAGMA analysis_limit=%d", analysisLimit)); err != nil { + log.Printf("[analyze] could not set analysis_limit: %v", err) + return false + } + if _, err := s.instrumentedExec("analyze", "ANALYZE"); err != nil { + log.Printf("[analyze] ANALYZE failed: %v", err) + return false + } + elapsed := time.Since(start).Round(time.Millisecond) + s.recordPlannerStatsRefresh(time.Now()) + if first { + log.Printf("[analyze] planner statistics built in %v (analysis_limit=%d, first run against this database)", elapsed, analysisLimit) + } else { + log.Printf("[analyze] planner statistics refreshed in %v (analysis_limit=%d)", elapsed, analysisLimit) + } + return true +} + +func (s *Store) EnsurePlannerStats(analysisLimit int) bool { + if s.Backend() != dbconfig.SQLite { + return false + } + + if s.hasPlannerStats() { + return false + } + log.Printf("[analyze] this database has no planner statistics; building them now. " + + "ANALYZE holds the single write connection until it finishes (3m43.9s measured on 9.4 GB, cold), " + + "so ingest will buffer and catch up. Once per database, not once per restart.") + return s.RefreshPlannerStats(analysisLimit) +} + +func (s *Store) hasPlannerStats() bool { + if s.Backend() != dbconfig.SQLite { + return false + } + + var n int + if err := s.db.QueryRow( + `SELECT count(*) FROM sqlite_master WHERE type='table' AND name='sqlite_stat1'`).Scan(&n); err != nil { + return false + } + return n > 0 +} + +func (s *Store) Checkpoint() int { + if s.Backend() != dbconfig.SQLite { + return 0 + } + waitStart := time.Now() + writerMu.Lock() + wait := time.Since(waitStart) + holdStart := time.Now() + defer func() { writerMu.Unlock(); recordWriterTiming("checkpoint", wait, time.Since(holdStart), "Checkpoint") }() + + var busy, walFrames, checkpointed int + if err := s.db.QueryRow("PRAGMA wal_checkpoint(TRUNCATE)").Scan(&busy, &walFrames, &checkpointed); err != nil { + log.Printf("[db] WAL checkpoint error: %v", err) + return 0 + } + if walFrames > 0 { + log.Printf("[db] WAL checkpoint: %d/%d frames checkpointed (blocked=%v)", checkpointed, walFrames, busy != 0) + } + return checkpointed +} + +// plannerStatsInterval is how often the planner statistics are refreshed, and +// plannerStatsStagger how long after startup the first refresh waits at least. +const ( + plannerStatsInterval = 24 * time.Hour + plannerStatsStagger = 2 * time.Minute +) + +// nextPlannerStatsRefresh returns how long to wait before the next refresh, +// given when the previous one succeeded (zero if never): until it is +// plannerStatsInterval old, but at least plannerStatsStagger and at most +// plannerStatsInterval. +// +// Before #2146 the first refresh ran plannerStatsStagger after every start. +// On a cold page cache that ANALYZE took 6 to 9 minutes on a 10-11 GB +// database, holding the single write connection, so each restart stalled +// ingest that long for statistics that were hours old at most. +func nextPlannerStatsRefresh(last, now time.Time) time.Duration { + if last.IsZero() { + return plannerStatsStagger + } + d := last.Add(plannerStatsInterval).Sub(now) + if d < plannerStatsStagger { + return plannerStatsStagger + } + if d > plannerStatsInterval { + return plannerStatsInterval + } + return d +} + +// plannerStatsStampPath is the file next to the database that records when +// the planner statistics were last refreshed. +func (s *Store) plannerStatsStampPath() string { + return s.path + ".planner-stats-refreshed" +} + +// lastPlannerStatsRefresh returns when RefreshPlannerStats last succeeded on +// this database, or the zero time when that is not known: the later of the +// time this process remembers and the time recorded in the file. +func (s *Store) lastPlannerStatsRefresh() time.Time { + var last time.Time + if ns := s.plannerStatsAt.Load(); ns != 0 { + last = time.Unix(0, ns) + } + b, err := os.ReadFile(s.plannerStatsStampPath()) + if err != nil { + return last + } + t, err := time.Parse(time.RFC3339, strings.TrimSpace(string(b))) + if err != nil || t.Before(last) { + return last + } + return t +} + +// recordPlannerStatsRefresh remembers when the statistics were refreshed and +// writes it next to the database for the next start. If the write fails, this +// process still waits the full interval; only the first refresh after the +// next restart comes early. +func (s *Store) recordPlannerStatsRefresh(at time.Time) { + s.plannerStatsAt.Store(at.UnixNano()) + tmp := s.plannerStatsStampPath() + ".tmp" + if err := os.WriteFile(tmp, []byte(at.UTC().Format(time.RFC3339)), 0o644); err != nil { + log.Printf("[analyze] could not record the refresh time: %v", err) + return + } + if err := os.Rename(tmp, s.plannerStatsStampPath()); err != nil { + log.Printf("[analyze] could not record the refresh time: %v", err) + } +} diff --git a/cmd/ingestor/stats_file.go b/cmd/ingestor/stats_file.go index 6ff094f77..19d676cee 100644 --- a/cmd/ingestor/stats_file.go +++ b/cmd/ingestor/stats_file.go @@ -6,6 +6,7 @@ import ( "encoding/json" "log" "os" + "path/filepath" "sync" "time" @@ -47,7 +48,7 @@ type IngestorStatsSnapshot struct { // the server's /api/perf/io endpoint under .ingestor (#1120 — "Both // ingestor and server"). Optional; absent on non-Linux hosts. ProcIO *PerfIOSample `json:"procIO,omitempty"` - // WriterPerf is the per-component SQLite writer-lock latency + // WriterPerf is the per-component serialized writer-lock latency // snapshot (#1340) — wait_ms / hold_ms / contention_total tagged // by component (neighbor_builder, mqtt_handler, prune_packets, // prune_observers, prune_metrics, vacuum). Surfaced by the server @@ -109,15 +110,17 @@ type SourceLivenessSnapshot struct { // statsFilePath returns the writable path the ingestor will publish stats to. // Override via env CORESCOPE_INGESTOR_STATS for tests / non-default deploys. // -// SECURITY: the default lives in /tmp which is world-writable. The writer uses -// O_NOFOLLOW + 0o600 so a pre-planted symlink cannot be used to clobber an -// arbitrary file via this path. Operators who want stronger guarantees should -// point CORESCOPE_INGESTOR_STATS at a private directory (e.g. /var/lib/corescope/). +// The default lives in the private state directory. The writer retains +// O_NOFOLLOW and private permissions for an explicitly overridden path. func statsFilePath() string { if p := os.Getenv("CORESCOPE_INGESTOR_STATS"); p != "" { return p } - return "/tmp/corescope-ingestor-stats.json" + stateDir := os.Getenv("CORESCOPE_STATE_DIR") + if stateDir == "" { + stateDir = "data" + } + return filepath.Join(stateDir, "ingestor-stats.json") } // writeStatsAtomic writes b to path via a tmp-then-rename, refusing to follow @@ -261,6 +264,9 @@ func StartStatsFileWriter(s *Store, interval time.Duration) (stop func()) { t := time.NewTicker(interval) defer t.Stop() path := statsFilePath() + if os.Getenv("CORESCOPE_INGESTOR_STATS") == "" && s.stateDir != "" { + path = filepath.Join(s.stateDir, "ingestor-stats.json") + } // Track previous procIO sample so we can compute per-second deltas // across ticks (#1120 follow-up: ingestor /proc/self/io exposure). prevIO := readProcSelfIOFn() diff --git a/cmd/ingestor/stats_file_test.go b/cmd/ingestor/stats_file_test.go index 59d59f0ba..431e104e6 100644 --- a/cmd/ingestor/stats_file_test.go +++ b/cmd/ingestor/stats_file_test.go @@ -53,7 +53,7 @@ func TestStatsFileWriter_PublishesProcIO(t *testing.T) { statsPath := filepath.Join(dir, "ingestor-stats.json") t.Setenv("CORESCOPE_INGESTOR_STATS", statsPath) - store, err := OpenStore(filepath.Join(dir, "test.db")) + store, err := openPostgresTestStore(t, filepath.Join(dir, "test.db")) if err != nil { t.Fatalf("OpenStore: %v", err) } diff --git a/cmd/ingestor/stats_file_timestamp_test.go b/cmd/ingestor/stats_file_timestamp_test.go index fd3a66597..abae95de8 100644 --- a/cmd/ingestor/stats_file_timestamp_test.go +++ b/cmd/ingestor/stats_file_timestamp_test.go @@ -38,7 +38,7 @@ func TestStatsFileWriter_SampledAtMatchesProcIOSampledAt(t *testing.T) { statsPath := filepath.Join(dir, "ingestor-stats.json") t.Setenv("CORESCOPE_INGESTOR_STATS", statsPath) - store, err := OpenStore(filepath.Join(dir, "test.db")) + store, err := openPostgresTestStore(t, filepath.Join(dir, "test.db")) if err != nil { t.Fatalf("OpenStore: %v", err) } diff --git a/cmd/ingestor/status_liveness_test.go b/cmd/ingestor/status_liveness_test.go index 6f96b07d1..a72b5aff2 100644 --- a/cmd/ingestor/status_liveness_test.go +++ b/cmd/ingestor/status_liveness_test.go @@ -55,7 +55,7 @@ func TestStaleStatusPayloadDoesNotCreateObserver(t *testing.T) { nil, nil, &Config{}) var count int - if err := store.db.QueryRow(`SELECT COUNT(*) FROM observers WHERE id = ?`, "obs-purged").Scan(&count); err != nil { + if err := store.db.QueryRow(`SELECT COUNT(*) FROM observers WHERE id = $1`, "obs-purged").Scan(&count); err != nil { t.Fatal(err) } if count != 0 { @@ -82,7 +82,7 @@ func TestOfflineStatusIsNotLiveness(t *testing.T) { func TestStaleStatusPayloadDoesNotReactivateInactiveObserver(t *testing.T) { store := newTestStore(t) seedObserver(t, store, "obs-zombie", 60) - if _, err := store.db.Exec(`UPDATE observers SET inactive = 1 WHERE id = ?`, "obs-zombie"); err != nil { + if _, err := store.db.Exec(`UPDATE observers SET inactive = 1 WHERE id = $1`, "obs-zombie"); err != nil { t.Fatal(err) } @@ -92,7 +92,7 @@ func TestStaleStatusPayloadDoesNotReactivateInactiveObserver(t *testing.T) { nil, nil, &Config{}) var inactive int - if err := store.db.QueryRow(`SELECT inactive FROM observers WHERE id = ?`, "obs-zombie").Scan(&inactive); err != nil { + if err := store.db.QueryRow(`SELECT inactive FROM observers WHERE id = $1`, "obs-zombie").Scan(&inactive); err != nil { t.Fatal(err) } if inactive != 1 { @@ -172,7 +172,7 @@ func TestStaleStatusPayloadStillUpdatesMetadata(t *testing.T) { nil, nil, &Config{}) var firmware string - if err := store.db.QueryRow(`SELECT firmware FROM observers WHERE id = ?`, "obs-zombie").Scan(&firmware); err != nil { + if err := store.db.QueryRow(`SELECT firmware FROM observers WHERE id = $1`, "obs-zombie").Scan(&firmware); err != nil { t.Fatal(err) } if firmware != "v1.14.1" { @@ -193,7 +193,7 @@ func TestStaleStatusPayloadDoesNotInsertMetricsSample(t *testing.T) { var count int if err := store.db.QueryRow( - `SELECT COUNT(*) FROM observer_metrics WHERE observer_id = ?`, "obs-zombie", + `SELECT COUNT(*) FROM observer_metrics WHERE observer_id = $1`, "obs-zombie", ).Scan(&count); err != nil { t.Fatal(err) } diff --git a/cmd/ingestor/storage_adoption.go b/cmd/ingestor/storage_adoption.go new file mode 100644 index 000000000..9145ee227 --- /dev/null +++ b/cmd/ingestor/storage_adoption.go @@ -0,0 +1,147 @@ +package main + +import ( + "context" + "database/sql" + "errors" + "net/url" + "os" + "path/filepath" + "time" + + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/dbschema/legacy" + "github.com/meshcore-analyzer/pgutil" +) + +// Adoption never initializes a missing telemetry file. Setup creates a fresh +// installation; this path records only an existing, validated native target. +func adoptLegacyStorage(raw dbconfig.StorageInputs) error { + selectionPath, err := storageSelectionPath(raw) + if err != nil { + return err + } + if _, err := dbconfig.ReadSelection(selectionPath); err == nil { + return nil + } else if !errors.Is(err, dbconfig.ErrSelectionMissing) { + return err + } + if raw.Backend == "" && raw.EnvBackend == "" && raw.DBPath == "" && raw.DatabaseURL == "" && raw.ReaderDatabaseURL == "" && raw.WriterDatabaseURL == "" && raw.UsersDatabaseURL == "" && raw.ApprovedChannelsDatabaseURL == "" { + raw.DBPath = filepath.Join(raw.BaseDir, "data", "meshcore.db") + } + raw.StateDir = filepath.Dir(selectionPath) + storage, err := dbconfig.ResolveStorage(raw) + if err != nil { + return err + } + selected, err := dbconfig.NewSelection(storage) + if err != nil { + return err + } + if storage.Backend == dbconfig.SQLite { + db, err := openExistingSQLite(storage.DBPath) + if err != nil { + return err + } + err = dbconfig.AssertSQLiteImportComplete(db) + if err == nil { + err = legacy.CheckLegacySource(db) + } + db.Close() + if err != nil { + return err + } + if _, err := os.Stat(storage.UsersDBPath); err == nil { + accounts, err := openExistingSQLite(storage.UsersDBPath) + if err != nil { + return err + } + var version int + if err = dbconfig.AssertSQLiteImportComplete(accounts); err != nil { + accounts.Close() + return err + } + err = accounts.QueryRow(`SELECT MAX(version) FROM schema_version`).Scan(&version) + accounts.Close() + if err != nil || version < 1 || version > 6 { + return errors.New("existing SQLite accounts schema is unsupported; use setup or offline recovery") + } + } else if errors.Is(err, os.ErrNotExist) { + // Absent legacy accounts are uninitialized, not an initialized empty store. + // Only explicit setup may create them and fill this selection target. + selected.Accounts = nil + } else { + return err + } + } else { + if storage.UsersDatabaseURL == "" && storage.ApprovedChannelsDatabaseURL == "" { + legacyAccounts := raw.UsersDBPath + if legacyAccounts == "" { + legacyAccounts = filepath.Join(storage.StateDir, "users.db") + } + if _, err := os.Stat(legacyAccounts); err == nil { + return errors.New("existing SQLite accounts require offline conversion before adopting PostgreSQL") + } else if !errors.Is(err, os.ErrNotExist) { + return err + } + } + db, err := pgutil.Open(storage.WriterDatabaseURL, false) + if err != nil { + return err + } + err = dbschema.AssertPostgresReady(db) + if err == nil { + err = dbschema.AssertWriter(db) + } + db.Close() + if err != nil { + return err + } + accountURL := storage.UsersDatabaseURL + if accountURL == "" { + accountURL = storage.ApprovedChannelsDatabaseURL + } + if accountURL != "" { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + same, err := pgutil.SameDatabase(ctx, storage.WriterDatabaseURL, accountURL) + cancel() + if err != nil { + return err + } + if same { + return errors.New("accounts must not use the telemetry database") + } + accounts, err := pgutil.Open(accountURL, true) + if err != nil { + return err + } + var ready bool + err = accounts.QueryRow(`SELECT ready FROM corescope_schema WHERE kind='accounts' AND version=1`).Scan(&ready) + accounts.Close() + if err != nil || !ready { + return errors.New("PostgreSQL accounts import/bootstrap is incomplete") + } + } + } + + _, err = dbconfig.AdoptSelection(selectionPath, selected) + return err +} + +func openExistingSQLite(path string) (*sql.DB, error) { + info, err := os.Stat(path) + if err != nil || !info.Mode().IsRegular() || info.Size() == 0 { + return nil, errors.New("existing SQLite storage is missing or empty; run explicit setup instead of creating a new runtime database") + } + dsn, err := dbconfig.SQLiteURI(path, url.Values{"mode": {"ro"}, "_query_only": {"on"}, "_busy_timeout": {"5000"}}) + if err != nil { + return nil, err + } + db, err := sql.Open("sqlite3", dsn) + if err != nil { + return nil, err + } + db.SetMaxOpenConns(1) + return db, nil +} diff --git a/cmd/ingestor/storage_adoption_test.go b/cmd/ingestor/storage_adoption_test.go new file mode 100644 index 000000000..ce9d882b4 --- /dev/null +++ b/cmd/ingestor/storage_adoption_test.go @@ -0,0 +1,125 @@ +package main + +import ( + "errors" + "os" + "path/filepath" + "testing" + + "github.com/meshcore-analyzer/dbconfig" +) + +func TestRuntimeMissingSQLiteCannotCreateReplacement(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "uninitialized", "meshcore.db") + raw := dbconfig.StorageInputs{BaseDir: dir, Backend: dbconfig.SQLite, DBPath: path} + if err := adoptLegacyStorage(raw); err == nil { + t.Fatal("missing source was treated as fresh") + } + if _, err := os.Stat(filepath.Dir(path)); !errors.Is(err, os.ErrNotExist) { + t.Fatal("runtime created replacement storage", err) + } +} + +func TestRuntimeAdoptsExistingSQLiteAndPinsRecordedChoice(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "meshcore.db") + seed, err := OpenStore(path) + if err != nil { + t.Fatal(err) + } + seed.Close() + raw := dbconfig.StorageInputs{BaseDir: dir, DBPath: path} + if err := adoptLegacyStorage(raw); err != nil { + t.Fatal(err) + } + raw.Backend = dbconfig.Postgres + raw.EnvBackend = dbconfig.Postgres + raw.DatabaseURL = "postgresql://unused@localhost/unselected" + storage, lease, err := resolveRuntimeStorage(raw) + if err != nil { + t.Fatal(err) + } + defer lease.Close() + if storage.Backend != dbconfig.SQLite || storage.DBPath != path || storage.UsersDBPath != "" { + t.Fatal("obsolete bootstrap choices overrode recorded installation") + } + selected, err := dbconfig.ReadSelection(dbconfig.SelectionPath(dir)) + if err != nil { + t.Fatal(err) + } + if selected.Accounts != nil { + t.Fatal("absent accounts marked initialized") + } + if _, err := dbconfig.BeginSelectionUpdate(dbconfig.SelectionPath(dir), selected.Generation); !errors.Is(err, dbconfig.ErrSelectionBusy) { + t.Fatal("runtime did not hold the selection lease", err) + } +} + +func TestRuntimeCannotAdoptStagedIncompleteSQLite(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "staged.db") + seed, err := OpenStore(path) + if err != nil { + t.Fatal(err) + } + if _, err = seed.db.Exec(`CREATE TABLE ` + dbconfig.SQLiteImportPendingTable + `(phase TEXT)`); err != nil { + t.Fatal(err) + } + seed.Close() + raw := dbconfig.StorageInputs{BaseDir: dir, StateDir: dir, DBPath: path} + if err := adoptLegacyStorage(raw); !errors.Is(err, dbconfig.ErrSQLiteImportIncomplete) { + t.Fatal("incomplete target accepted", err) + } + if _, err := dbconfig.ReadSelection(dbconfig.SelectionPath(dir)); !errors.Is(err, dbconfig.ErrSelectionMissing) { + t.Fatal("incomplete target recorded", err) + } +} + +func TestRecordedSQLiteCannotCreateReplacement(t *testing.T) { + for _, empty := range []bool{false, true} { + name := "missing" + if empty { + name = "empty" + } + t.Run(name, func(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "meshcore.db") + seed, err := OpenStore(path) + if err != nil { + t.Fatal(err) + } + seed.Close() + raw := dbconfig.StorageInputs{BaseDir: dir, DBPath: path} + if err := adoptLegacyStorage(raw); err != nil { + t.Fatal(err) + } + if err := os.Rename(path, path+".preserved"); err != nil { + t.Fatal(err) + } + if empty { + if err := os.WriteFile(path, nil, 0600); err != nil { + t.Fatal(err) + } + } + storage, lease, err := resolveRuntimeStorage(raw) + if err != nil { + t.Fatal(err) + } + defer lease.Close() + store, err := openStoreStorage(storage, nil, 300) + if err == nil { + store.Close() + t.Error("recorded runtime silently initialized replacement telemetry") + } + info, statErr := os.Stat(path) + if empty { + if statErr != nil || info.Size() != 0 { + t.Error("runtime modified the empty recorded target", statErr) + } + } else if !errors.Is(statErr, os.ErrNotExist) { + t.Error("runtime created a missing recorded target", statErr) + } + }) + } +} diff --git a/cmd/ingestor/storage_backend.go b/cmd/ingestor/storage_backend.go new file mode 100644 index 000000000..120e96fd5 --- /dev/null +++ b/cmd/ingestor/storage_backend.go @@ -0,0 +1,72 @@ +package main + +import ( + "database/sql" + "errors" + "net/url" + "os" + + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/dbschema/legacy" +) + +func (s *Store) Backend() dbconfig.Backend { + if s.backend == "" { + return dbconfig.SQLite + } + return s.backend +} +func (s *Store) parameter(n int) string { return s.Backend().Parameter(n) } +func (s *Store) nativeSQL(sqlite, postgres string) string { + if s.Backend() == dbconfig.SQLite { + return sqlite + } + return postgres +} + +// OpenStoreStorage is the runtime boundary after the shared selector resolves +// raw inputs. Compatibility constructors still accept paths or explicit URLs. +func OpenStoreStorage(storage dbconfig.Storage, cfg *DBConfig) (*Store, error) { + return openStoreStorage(storage, cfg, 300) +} + +// Retain the shared durability/connection options. Existing runtime targets use +// rw (never rwc); persistent pragmas run only after schema validation succeeds. +func sqliteTelemetryWriterDSN(path string, create bool) (string, error) { + if !create { + info, err := os.Stat(path) + if err != nil || !info.Mode().IsRegular() || info.Size() == 0 { + return "", errors.New("selected SQLite telemetry is missing or empty; restore the initialized store") + } + } + dsn, err := legacy.WriterDSN(path) + if err != nil { + return "", err + } + u, err := url.Parse(dsn) + if err != nil { + return "", err + } + q := u.Query() + q.Set("_txlock", "immediate") + if !create { + q.Set("mode", "rw") + q.Del("_journal_mode") + q.Del("_auto_vacuum") + } + u.RawQuery = q.Encode() + return u.String(), nil +} + +func validateExistingTelemetry(db *sql.DB) error { + if err := dbconfig.AssertSQLiteImportComplete(db); err != nil { + return err + } + if err := dbschema.AssertSQLiteReady(db); err == nil { + return nil + } + // A supported older v3 store may need the normal writer upgrade. This also + // rejects empty/header-only/unrelated databases through the opened handle. + return legacy.CheckLegacySource(db) +} diff --git a/cmd/ingestor/storage_backend_test.go b/cmd/ingestor/storage_backend_test.go new file mode 100644 index 000000000..51d823cfb --- /dev/null +++ b/cmd/ingestor/storage_backend_test.go @@ -0,0 +1,100 @@ +package main + +import ( + "database/sql" + "github.com/meshcore-analyzer/dbconfig" + "os" + "path/filepath" + "testing" +) + +func TestOpenStoreNativeSQLiteWithoutPostgres(t *testing.T) { + t.Setenv("CORESCOPE_TEST_POSTGRES_URL", "") + path := filepath.Join(t.TempDir(), "telemetry.db") + s, err := OpenStore(path) + if err != nil { + t.Fatalf("native SQLite writer must work without PostgreSQL: %v", err) + } + defer s.Close() + s.WaitForAsyncMigrations() + for pragma, want := range map[string]string{"journal_mode": "wal", "synchronous": "2", "foreign_keys": "1"} { + var actual string + if err := s.db.QueryRow("PRAGMA " + pragma).Scan(&actual); err != nil || actual != want { + t.Fatalf("%s=%q want %q: %v", pragma, actual, want, err) + } + } + if s.path != path { + t.Fatalf("legacy queue anchor changed: %q", s.path) + } +} + +func TestSQLiteObserverTimeIndexCompletesThroughAsyncLedger(t *testing.T) { + s, err := OpenStore(filepath.Join(t.TempDir(), "observer-index.db")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + s.WaitForAsyncMigrations() + var count int + if err = s.db.QueryRow(`SELECT count(*) FROM sqlite_master WHERE type='index' AND name='idx_observations_observer_idx_timestamp'`).Scan(&count); err != nil || count != 1 { + t.Fatalf("native observer/time index count=%d: %v", count, err) + } + if status, err := s.AsyncMigrationStatus("obs_observer_ts_idx_v1"); err != nil || status != "done" { + t.Fatal("native index migration did not complete", status, err) + } +} + +func TestSQLiteMissingObserverIndexIsNotHiddenByCompletionMarker(t *testing.T) { + path := filepath.Join(t.TempDir(), "recovered-index.db") + s, err := OpenStore(path) + if err != nil { + t.Fatal(err) + } + s.WaitForAsyncMigrations() + if _, err = s.db.Exec(`DROP INDEX idx_observations_observer_idx_timestamp`); err != nil { + t.Fatal(err) + } + s.Close() + s, err = OpenStore(path) + if err != nil { + t.Fatal(err) + } + defer s.Close() + s.WaitForAsyncMigrations() + var count int + if err = s.db.QueryRow(`SELECT count(*) FROM sqlite_master WHERE type='index' AND name='idx_observations_observer_idx_timestamp'`).Scan(&count); err != nil || count != 1 { + t.Fatal("missing index concealed by done marker", count, err) + } +} + +func testBackend(t testing.TB) dbconfig.Backend { + t.Helper() + v := os.Getenv("CORESCOPE_TEST_BACKEND") + if v != "" && v != "sqlite" && v != "postgres" { + t.Fatalf("invalid CORESCOPE_TEST_BACKEND") + } + if v == "postgres" { + return dbconfig.Postgres + } + return dbconfig.SQLite +} +func testBackendValue() dbconfig.Backend { + if os.Getenv("CORESCOPE_TEST_BACKEND") == "postgres" { + return dbconfig.Postgres + } + return dbconfig.SQLite +} +func testNativeSQL(sqlite, postgres string) string { + if testBackendValue() == dbconfig.Postgres { + return postgres + } + return sqlite +} + +func scanTestPlan(rows *sql.Rows, detail *string) error { + if testBackendValue() == dbconfig.SQLite { + var id, parent, unused int + return rows.Scan(&id, &parent, &unused, detail) + } + return rows.Scan(detail) +} diff --git a/cmd/ingestor/storage_config.go b/cmd/ingestor/storage_config.go new file mode 100644 index 000000000..021b1e3d9 --- /dev/null +++ b/cmd/ingestor/storage_config.go @@ -0,0 +1,120 @@ +package main + +import ( + "errors" + "io" + "path/filepath" + "strings" + + "github.com/meshcore-analyzer/dbconfig" +) + +type storageFlags struct { + Backend dbconfig.Backend + DBPath, DatabaseURL, StateDir string +} + +// storageInputs gathers bootstrap values without guessing an installation's +// state. The shared persisted selector applies before ResolveStorage runs. +func (c *Config) storageInputs(baseDir string, flags storageFlags, getenv func(string) string) (dbconfig.StorageInputs, error) { + base, err := filepath.Abs(baseDir) + if err != nil { + return dbconfig.StorageInputs{}, errors.New("resolve storage base directory") + } + in := dbconfig.StorageInputs{BaseDir: base, DBPath: c.DBPath, StateDir: envOrValue(getenv, "CORESCOPE_STATE_DIR", c.StateDir), + EnvBackend: dbconfig.Backend(getenv("CORESCOPE_DB_BACKEND")), + DatabaseURL: envOrValue(getenv, "CORESCOPE_DATABASE_URL", c.DatabaseURL), + ReaderDatabaseURL: getenv("CORESCOPE_READER_DATABASE_URL"), WriterDatabaseURL: getenv("CORESCOPE_WRITER_DATABASE_URL"), + UsersDatabaseURL: getenv("CORESCOPE_USERS_DATABASE_URL"), ApprovedChannelsDatabaseURL: getenv("CORESCOPE_APPROVED_CHANNELS_DATABASE_URL")} + if p := getenv("DB_PATH"); p != "" { + in.DBPath = p + } + if c.DB != nil { + in.Backend = c.DB.Backend + } + if c.UserManagement != nil { + in.UsersDBPath = strings.TrimSpace(c.UserManagement.DBPath) + in.UsersDatabaseURL = envOrValue(getenv, "CORESCOPE_USERS_DATABASE_URL", c.UserManagement.DatabaseURL) + in.ApprovedChannelsDatabaseURL = envOrValue(getenv, "CORESCOPE_APPROVED_CHANNELS_DATABASE_URL", c.UserManagement.ApprovedChannelsDatabaseURL) + } + if flags.Backend != "" { + in.Backend = flags.Backend + } + if flags.DBPath != "" { + in.DBPath = flags.DBPath + } + if flags.DatabaseURL != "" { + in.DatabaseURL = flags.DatabaseURL + in.WriterDatabaseURL = flags.DatabaseURL + } + if flags.StateDir != "" { + in.StateDir = flags.StateDir + } + // The ingestor receives the process working directory as its base; explicit + // legacy paths retain their historical working-directory interpretation. + for _, p := range []*string{&in.DBPath, &in.UsersDBPath} { + if *p == "" || strings.Contains(*p, "://") { + continue + } + if filepath.VolumeName(*p) != "" && !filepath.IsAbs(*p) { + return in, errors.New("drive-relative storage paths are ambiguous") + } + *p, err = filepath.Abs(*p) + if err != nil { + return in, errors.New("resolve SQLite storage path") + } + } + return in, nil +} + +func envOrValue(getenv func(string) string, key, fallback string) string { + if value := strings.TrimSpace(getenv(key)); value != "" { + return value + } + return fallback +} + +// Discovery uses the stable legacy anchor, including after a backend switch. +func storageSelectionPath(raw dbconfig.StorageInputs) (string, error) { + state := raw.StateDir + if state == "" { + if strings.Contains(raw.DBPath, "://") || strings.HasPrefix(raw.DBPath, "postgres:") || strings.HasPrefix(raw.DBPath, "postgresql:") { + return "", errors.New("a database URL cannot be used as the legacy state directory anchor") + } + path := raw.DBPath + if path == "" { + path = filepath.Join(raw.BaseDir, "data", "meshcore.db") + } + state = filepath.Dir(path) + } + if strings.Contains(state, "://") || strings.ContainsRune(state, 0) { + return "", errors.New("invalid storage state directory") + } + if !filepath.IsAbs(state) { + state = filepath.Join(raw.BaseDir, state) + } + return dbconfig.SelectionPath(filepath.Clean(state)), nil +} +func resolveRuntimeStorage(raw dbconfig.StorageInputs) (dbconfig.Storage, io.Closer, error) { + path, err := storageSelectionPath(raw) + if err != nil { + return dbconfig.Storage{}, nil, err + } + selected, lease, err := dbconfig.OpenSelection(path) + if err != nil { + return dbconfig.Storage{}, nil, err + } + fail := func(err error) (dbconfig.Storage, io.Closer, error) { + lease.Close() + return dbconfig.Storage{}, nil, err + } + raw, err = selected.ApplyTo(raw) + if err != nil { + return fail(err) + } + storage, err := dbconfig.ResolveStorage(raw) + if err != nil { + return fail(err) + } + return storage, lease, nil +} diff --git a/cmd/ingestor/storage_config_test.go b/cmd/ingestor/storage_config_test.go new file mode 100644 index 000000000..390e7d917 --- /dev/null +++ b/cmd/ingestor/storage_config_test.go @@ -0,0 +1,88 @@ +package main + +import ( + "path/filepath" + "testing" + + "github.com/meshcore-analyzer/dbconfig" +) + +func TestStorageInputsKeepBootstrapRawAndLegacyPrecedence(t *testing.T) { + base := t.TempDir() + cfg := &Config{DBPath: "configured.db", UserManagement: &UserManagementConfig{DBPath: "accounts.db"}} + env := func(k string) string { + if k == "DB_PATH" { + return "environment.db" + } + return "" + } + in, err := cfg.storageInputs(base, storageFlags{}, env) + if err != nil { + t.Fatal(err) + } + want, _ := filepath.Abs("environment.db") + if in.DBPath != want || in.StateDir != "" || in.Backend != "" || in.FreshInstall { + t.Fatalf("raw values/defaults changed: %+v", in) + } + in, err = cfg.storageInputs(base, storageFlags{DBPath: "cli.db"}, env) + if err != nil { + t.Fatal(err) + } + want, _ = filepath.Abs("cli.db") + if in.DBPath != want { + t.Fatal("CLI path did not win") + } + in, err = (&Config{}).storageInputs(base, storageFlags{}, func(string) string { return "" }) + if err != nil { + t.Fatal(err) + } + if _, err = dbconfig.ResolveStorage(in); err == nil { + t.Fatal("unknown existing install was guessed fresh") + } + in.FreshInstall = true + got, err := dbconfig.ResolveStorage(in) + if err != nil { + t.Fatal(err) + } + if got.Backend != dbconfig.SQLite || got.DBPath != filepath.Join(base, "data", "meshcore.db") { + t.Fatalf("fresh default: %+v", got) + } +} + +func TestStorageInputsWriterURLAndStatePrecedence(t *testing.T) { + base := t.TempDir() + cfg := &Config{DB: &DBConfig{Backend: dbconfig.Postgres}, DatabaseURL: "postgresql://reader@localhost/config", StateDir: "configured"} + env := func(k string) string { + switch k { + case "CORESCOPE_DATABASE_URL": + return "postgresql://reader@localhost/generic" + case "CORESCOPE_WRITER_DATABASE_URL": + return "postgresql://reader@localhost/reader" + case "CORESCOPE_STATE_DIR": + return "state" + } + return "" + } + in, err := cfg.storageInputs(base, storageFlags{}, env) + if err != nil { + t.Fatal(err) + } + got, err := dbconfig.ResolveStorage(in) + if err != nil { + t.Fatal(err) + } + if got.WriterDatabaseURL != env("CORESCOPE_WRITER_DATABASE_URL") || got.StateDir != filepath.Join(base, "state") { + t.Fatal("reader environment precedence changed") + } + in, err = cfg.storageInputs(base, storageFlags{DatabaseURL: "postgresql://reader@localhost/cli", StateDir: "cli-state"}, env) + if err != nil { + t.Fatal(err) + } + got, err = dbconfig.ResolveStorage(in) + if err != nil { + t.Fatal(err) + } + if got.ReaderDatabaseURL != "postgresql://reader@localhost/cli" || got.WriterDatabaseURL != "postgresql://reader@localhost/cli" || got.StateDir != filepath.Join(base, "cli-state") { + t.Fatal("CLI overrides lost") + } +} diff --git a/cmd/ingestor/storage_runtime_open_test.go b/cmd/ingestor/storage_runtime_open_test.go new file mode 100644 index 000000000..e182d164b --- /dev/null +++ b/cmd/ingestor/storage_runtime_open_test.go @@ -0,0 +1,121 @@ +package main + +import ( + "crypto/sha256" + "database/sql" + "errors" + "os" + "path/filepath" + "testing" + + "github.com/meshcore-analyzer/dbconfig" +) + +func TestSelectedTelemetryRejectsWrongOrUninitializedSchemaWithoutMutation(t *testing.T) { + for _, query := range []string{`PRAGMA user_version=1`, `CREATE TABLE unrelated(value TEXT); INSERT INTO unrelated VALUES('keep')`} { + path := filepath.Join(t.TempDir(), "existing.db") + db, err := sql.Open("sqlite3", path) + if err != nil { + t.Fatal(err) + } + if _, err = db.Exec(query); err != nil { + t.Fatal(err) + } + db.Close() + before, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if s, err := OpenStoreStorage(dbconfig.Storage{Backend: dbconfig.SQLite, DBPath: path}, nil); err == nil { + s.Close() + t.Fatal("unrelated or uninitialized store accepted") + } + after, err := os.ReadFile(path) + if err != nil || sha256.Sum256(before) != sha256.Sum256(after) { + t.Fatal("rejected telemetry store changed", err) + } + } +} + +func TestSelectedTelemetryCannotRecreateRemovedVolume(t *testing.T) { + dir := filepath.Join(t.TempDir(), "missing-volume") + if s, err := OpenStoreStorage(dbconfig.Storage{Backend: dbconfig.SQLite, DBPath: filepath.Join(dir, "telemetry.db")}, nil); err == nil { + s.Close() + t.Fatal("missing volume accepted") + } + if _, err := os.Stat(dir); !errors.Is(err, os.ErrNotExist) { + t.Fatal("runtime recreated missing volume", err) + } +} + +func TestSQLiteTelemetryTargetRemovedAfterDSNValidation(t *testing.T) { + for _, replaceEmpty := range []bool{false, true} { + path := filepath.Join(t.TempDir(), "telemetry.db") + seed, err := OpenStore(path) + if err != nil { + t.Fatal(err) + } + seed.Close() + dsn, err := sqliteTelemetryWriterDSN(path, false) + if err != nil { + t.Fatal(err) + } + if err = os.Rename(path, path+".preserved"); err != nil { + t.Fatal(err) + } + if replaceEmpty { + if err = os.WriteFile(path, nil, 0600); err != nil { + t.Fatal(err) + } + } + db, err := sql.Open("sqlite3", dsn) + if err != nil { + t.Fatal(err) + } + if err = validateExistingTelemetry(db); err == nil { + t.Fatal("raced missing/empty target accepted") + } + db.Close() + info, err := os.Stat(path) + if replaceEmpty { + if err != nil || info.Size() != 0 { + t.Fatal("raced empty file was initialized", err) + } + } else if !errors.Is(err, os.ErrNotExist) { + t.Fatal("raced missing file was recreated", err) + } + } +} + +func TestSelectedTelemetryRetainsRowsSequenceAndDurability(t *testing.T) { + path := filepath.Join(t.TempDir(), "telemetry.db") + seed, err := OpenStore(path) + if err != nil { + t.Fatal(err) + } + if _, err = seed.InsertTransmission(&PacketData{Hash: "preserved", RawHex: "AA", Timestamp: "2026-01-01T00:00:00Z", PathJSON: "[]"}); err != nil { + t.Fatal(err) + } + if _, err = seed.db.Exec(`INSERT INTO transmissions(id,raw_hex,hash,first_seen) VALUES(900,'AA','retired','2026-01-01T00:00:00Z'); DELETE FROM transmissions WHERE id=900`); err != nil { + t.Fatal(err) + } + seed.Close() + s, err := OpenStoreStorage(dbconfig.Storage{Backend: dbconfig.SQLite, DBPath: path}, nil) + if err != nil { + t.Fatal(err) + } + defer s.Close() + var count, sequence int + if err = s.db.QueryRow(`SELECT count(*) FROM transmissions WHERE hash='preserved'`).Scan(&count); err != nil || count != 1 { + t.Fatal("stored row lost", count, err) + } + if err = s.db.QueryRow(`SELECT seq FROM sqlite_sequence WHERE name='transmissions'`).Scan(&sequence); err != nil || sequence != 900 { + t.Fatal("allocator reset", sequence, err) + } + for pragma, want := range map[string]string{"journal_mode": "wal", "synchronous": "2", "foreign_keys": "1"} { + var got string + if err = s.db.QueryRow("PRAGMA " + pragma).Scan(&got); err != nil || got != want { + t.Fatalf("%s=%s: %v", pragma, got, err) + } + } +} diff --git a/cmd/migrate/account_setup.go b/cmd/migrate/account_setup.go new file mode 100644 index 000000000..b139b11ca --- /dev/null +++ b/cmd/migrate/account_setup.go @@ -0,0 +1,186 @@ +package main + +import ( + "context" + "errors" + "os" + "reflect" + + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/pgutil" + "github.com/meshcore-analyzer/users" +) + +// The existing journal identifies this one same-backend transition without a +// separate initialization flag or sidecar. Its telemetry target never changes. +func accountInitialization(current, next dbconfig.Selection) bool { + return current.Generation != "" && current.Accounts == nil && next.Accounts != nil && current.Backend == next.Backend && current.StateDir == next.StateDir && reflect.DeepEqual(current.Telemetry, next.Telemetry) +} + +func initializeAccounts(ctx context.Context, o storageOptions, current dbconfig.Selection, raw dbconfig.StorageInputs) error { + update, err := dbconfig.BeginSelectionUpdate(o.SelectionFile, current.Generation) + if err != nil { + return err + } + defer update.Close() + // Failed validation before staging has done no account work. A real process + // crash still leaves the journal for an explicit metadata-only abort. + defer func() { + if update.Target().Generation == "" { + _ = update.Abort() + } + }() + if err := validateRuntimeSelection(ctx, current, raw); err != nil { + return err + } + bound, err := current.ApplyTo(raw) + if err != nil { + return err + } + bound.ExistingBackend = "" // This explicit setup, alone, supplies a new account default. + if current.Backend == dbconfig.SQLite { + bound.UsersDBPath = raw.UsersDBPath + } else { + if o.UsersOwnerURL == "" || raw.UsersDatabaseURL == "" { + return errors.New("first PostgreSQL account setup requires both account owner and runtime writer URLs") + } + bound.UsersDatabaseURL = o.UsersOwnerURL + bound.ApprovedChannelsDatabaseURL = "" + } + storage, err := dbconfig.ResolveStorage(bound) + if err != nil { + return err + } + next, err := dbconfig.NewSelection(storage) + if err != nil { + return err + } + if !accountInitialization(current, next) { + return errors.New("account setup cannot change the selected telemetry target") + } + if current.Backend == dbconfig.Postgres { + if err := validateInitialOwnerTargets(ctx, next, storage); err != nil { + return err + } + } + if err := update.Stage(next); err != nil { + return err + } + return finishAccountSetup(ctx, o, update, raw, false) +} + +func finishAccountSetup(ctx context.Context, o storageOptions, update *dbconfig.SelectionUpdate, raw dbconfig.StorageInputs, resume bool) error { + current, next := update.Current(), update.Target() + if !accountInitialization(current, next) { + return errors.New("pending journal is not a supported first account initialization") + } + if err := validateRuntimeSelection(ctx, current, raw); err != nil { + return err + } + if next.Backend == dbconfig.SQLite { + path := next.Accounts.SQLitePath + telemetry, err := os.Stat(current.Telemetry.SQLitePath) + if err != nil { + return err + } + info, statErr := os.Stat(path) + if statErr == nil && os.SameFile(telemetry, info) { + return errors.New("account setup cannot use the telemetry file") + } + if statErr != nil && !errors.Is(statErr, os.ErrNotExist) { + return statErr + } + if resume { + if statErr != nil { + return errors.New("pending account target is missing; restore it or abort this uncommitted initialization before explicit setup; it was not recreated") + } + db, err := openSQLite(path, "ro") + if err != nil { + return err + } + err = assertSQLiteData(ctx, db, "accounts") + db.Close() + if err != nil { + return errors.New("account initialization did not reach a ready target; preserve it, abort the uncommitted update and rerun explicit setup") + } + } else if err := applySQLiteTarget(ctx, path, "accounts"); err != nil { + return err + } + db, err := openSQLite(path, "ro") + if err != nil { + return err + } + err = assertSQLiteData(ctx, db, "accounts") + db.Close() + if err != nil { + return err + } + } else { + if o.UsersOwnerURL == "" || raw.UsersDatabaseURL == "" { + return errors.New("first PostgreSQL account setup requires both account owner and runtime writer URLs") + } + ownerInputs := raw + ownerInputs.UsersDatabaseURL = o.UsersOwnerURL + ownerInputs.ApprovedChannelsDatabaseURL = "" + bound, err := next.ApplyTo(ownerInputs) + if err != nil { + return err + } + owners, err := dbconfig.ResolveStorage(bound) + if err != nil { + return err + } + same, err := pgutil.SameDatabase(ctx, owners.WriterDatabaseURL, owners.UsersDatabaseURL) + if err != nil { + return err + } + if same { + return errors.New("accounts require a separate PostgreSQL database") + } + db, err := pgutil.Open(owners.UsersDatabaseURL, false) + if err != nil { + return err + } + readyErr := users.AssertPostgresReady(db) + if readyErr != nil { + if resume { + db.Close() + return errors.New("account initialization did not reach a ready PostgreSQL target; preserve it, abort the uncommitted update and rerun setup") + } + var objects int + err = db.QueryRowContext(ctx, importDestinationObjects).Scan(&objects) + if err == nil && objects != 0 { + err = errors.New("account target is neither a ready store nor a fresh empty PostgreSQL schema") + } + if err == nil { + err = users.ApplyPostgres(db) + } + } + if err == nil { + tables, layoutErr := layouts("accounts") + if layoutErr != nil { + err = layoutErr + } else { + err = validatePostgresLayout(db, "accounts", tables) + } + } + db.Close() + if err != nil { + return err + } + owners.WriterDatabaseURL = "" // Existing telemetry permissions are not part of first account setup. + if err := grantRuntimeTargets(ctx, next, owners, raw); err != nil { + return err + } + } + if err := validateRuntimeSelection(ctx, next, raw); err != nil { + return err + } + if o.beforeCommit != nil { + if err := o.beforeCommit(); err != nil { + return err + } + } + _, err := update.Commit(next) + return err +} diff --git a/cmd/migrate/account_setup_test.go b/cmd/migrate/account_setup_test.go new file mode 100644 index 000000000..0aeba5a27 --- /dev/null +++ b/cmd/migrate/account_setup_test.go @@ -0,0 +1,244 @@ +package main + +import ( + "bytes" + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/users" +) + +func TestLegacyAccountAbsenceFirstSetupAndRecordedLoss(t *testing.T) { + for _, key := range []string{"CORESCOPE_DB_BACKEND", "DB_PATH", "CORESCOPE_DATABASE_URL", "CORESCOPE_USERS_OWNER_DATABASE_URL", "CORESCOPE_USERS_DATABASE_URL", "CORESCOPE_APPROVED_CHANNELS_DATABASE_URL"} { + t.Setenv(key, "") + } + source := telemetrySource(t) + accounts := filepath.Join(t.TempDir(), "first-accounts.sqlite") + dir := t.TempDir() + o := storageOptions{Action: "adopt", Backend: "sqlite", Offline: true, SelectionFile: filepath.Join(dir, "storage-selection.json"), SQLitePath: source, UsersSQLitePath: accounts, ConfigDir: t.TempDir()} + if err := runStorageAction(context.Background(), o, &bytes.Buffer{}); err != nil { + t.Fatal(err) + } + selected, err := dbconfig.ReadSelection(o.SelectionFile) + if err != nil || selected.Accounts != nil { + t.Fatal("legacy absence was initialized", err) + } + if _, err := os.Stat(accounts); !os.IsNotExist(err) { + t.Fatal("legacy adoption created account file", err) + } + _, lease, err := dbconfig.OpenSelection(o.SelectionFile) + if err != nil { + t.Fatal(err) + } + o.Action = "setup" + if err := runStorageAction(context.Background(), o, &bytes.Buffer{}); !errors.Is(err, dbconfig.ErrSelectionBusy) { + t.Fatal("first initialization lacked exclusive lease", err) + } + lease.Close() + if _, err := os.Stat(accounts); !os.IsNotExist(err) { + t.Fatal("busy setup created account file", err) + } + if err := runStorageAction(context.Background(), o, &bytes.Buffer{}); err != nil { + t.Fatal("first explicit setup", err) + } + initialized, err := dbconfig.ReadSelection(o.SelectionFile) + if err != nil || initialized.Accounts == nil || initialized.Accounts.SQLitePath != accounts || initialized.Generation == selected.Generation { + t.Fatal("initialized target not recorded", err) + } + db, err := openSQLite(accounts, "ro") + if err != nil { + t.Fatal(err) + } + if err := users.AssertSQLiteReady(db); err != nil { + t.Fatal(err) + } + db.Close() + if err := os.Remove(accounts); err != nil { + t.Fatal(err) + } + if err := runStorageAction(context.Background(), o, &bytes.Buffer{}); err == nil { + t.Fatal("recorded missing accounts recreated") + } + if _, err := os.Stat(accounts); !os.IsNotExist(err) { + t.Fatal("lost account file replaced", err) + } + after, err := dbconfig.ReadSelection(o.SelectionFile) + if err != nil || after.Generation != initialized.Generation { + t.Fatal("failed setup changed recorded target", err) + } +} + +func TestLegacyDisabledExistingAccountStoreIsRecorded(t *testing.T) { + source, accounts := telemetrySource(t), accountSource(t, 5) + dir := t.TempDir() + config := filepath.Join(dir, "config.json") + if err := os.WriteFile(config, []byte(`{"userManagement":{"enabled":false}}`), 0600); err != nil { + t.Fatal(err) + } + o := storageOptions{Action: "adopt", Backend: "sqlite", Offline: true, SelectionFile: filepath.Join(dir, "storage-selection.json"), SQLitePath: source, UsersSQLitePath: accounts, ConfigFile: config} + if err := runStorageAction(context.Background(), o, &bytes.Buffer{}); err != nil { + t.Fatal(err) + } + selected, err := dbconfig.ReadSelection(o.SelectionFile) + if err != nil || selected.Accounts == nil || selected.Accounts.SQLitePath != accounts { + t.Fatal("disabled initialized store was omitted", err) + } + db, err := openSQLite(accounts, "ro") + if err != nil { + t.Fatal(err) + } + defer db.Close() + var id int + if err := db.QueryRow(`SELECT id FROM mail_events WHERE mail_id=4`).Scan(&id); err != nil || id != 900 { + t.Fatal("existing account data changed", id, err) + } +} + +func TestSQLiteAccountSetupResumeUsesOnlyItsJournal(t *testing.T) { + for _, missing := range []bool{false, true} { + t.Run(map[bool]string{false: "complete", true: "lost-target"}[missing], func(t *testing.T) { + source := telemetrySource(t) + accounts := filepath.Join(t.TempDir(), "accounts.sqlite") + dir := t.TempDir() + o := storageOptions{Action: "adopt", Backend: "sqlite", Offline: true, SelectionFile: filepath.Join(dir, "storage-selection.json"), SQLitePath: source, UsersSQLitePath: accounts, ConfigDir: t.TempDir()} + if err := runStorageAction(context.Background(), o, &bytes.Buffer{}); err != nil { + t.Fatal(err) + } + o.Action = "setup" + o.beforeCommit = func() error { return errors.New("stop before account selection") } + if err := runStorageAction(context.Background(), o, &bytes.Buffer{}); err == nil { + t.Fatal("injected cutover failure passed") + } + status, err := dbconfig.InspectSelection(o.SelectionFile) + if err != nil || status.State != "pending" || status.SourceBackend != dbconfig.SQLite || status.TargetBackend != dbconfig.SQLite { + t.Fatal("same-backend journal status", status, err) + } + if missing { + if err := os.Remove(accounts); err != nil { + t.Fatal(err) + } + } + t.Setenv("CORESCOPE_DATABASE_URL", "must-not-connect") + t.Setenv("CORESCOPE_USERS_OWNER_DATABASE_URL", "must-not-connect") + t.Setenv("CORESCOPE_USERS_DATABASE_URL", "must-not-connect") + o.Action = "resume" + o.JobID = status.JobID + o.beforeCommit = nil + err = runStorageAction(context.Background(), o, &bytes.Buffer{}) + if missing { + if err == nil || !strings.Contains(err.Error(), "not recreated") { + t.Fatal("lost staged account target was recreated", err) + } + if _, err := os.Stat(accounts); !os.IsNotExist(err) { + t.Fatal("missing staged target replaced") + } + o.Action = "abort" + if err := runStorageAction(context.Background(), o, &bytes.Buffer{}); err != nil { + t.Fatal(err) + } + } else if err != nil { + t.Fatal("SQLite-only account resume", err) + } + }) + } +} + +func TestUnstagedJournalNeedsNoDatabaseForAbort(t *testing.T) { + source := telemetrySource(t) + if err := applySQLiteTarget(context.Background(), source, "telemetry"); err != nil { + t.Fatal(err) + } + dir := t.TempDir() + selected, err := dbconfig.NewSelection(dbconfig.Storage{Backend: dbconfig.SQLite, StateDir: dir, DBPath: source}) + if err != nil { + t.Fatal(err) + } + path := dbconfig.SelectionPath(dir) + if _, err := dbconfig.AdoptSelection(path, selected); err != nil { + t.Fatal(err) + } + update, err := dbconfig.BeginSelectionUpdate(path, selected.Generation) + if err != nil { + t.Fatal(err) + } + id := update.ID() + update.Close() + t.Setenv("CORESCOPE_DATABASE_URL", "must-not-connect") + o := storageOptions{Action: "resume", Offline: true, SelectionFile: path, JobID: id, ConfigDir: t.TempDir()} + if err := runStorageAction(context.Background(), o, &bytes.Buffer{}); err == nil || !strings.Contains(err.Error(), "no staged destination") { + t.Fatal("unstaged job resume guessed a target", err) + } + o.Action = "abort" + if err := runStorageAction(context.Background(), o, &bytes.Buffer{}); err != nil { + t.Fatal("metadata-only abort", err) + } +} + +func TestPostgresAccountFirstSetupRequiresCredentialsAndResumes(t *testing.T) { + owner, accountOwner := postgresDatabase(t), postgresDatabase(t) + db := openImportDB(t, owner) + if err := dbschema.ApplyPostgres(db, nil); err != nil { + t.Fatal(err) + } + reader, writer := restrictedRole(t, owner), restrictedRole(t, owner) + dir := t.TempDir() + selected, err := dbconfig.NewSelection(dbconfig.Storage{Backend: dbconfig.Postgres, StateDir: dir, WriterDatabaseURL: owner}) + if err != nil { + t.Fatal(err) + } + raw := dbconfig.StorageInputs{BaseDir: dir, ReaderDatabaseURL: reader, WriterDatabaseURL: writer} + if err := grantRuntimeTargets(context.Background(), selected, dbconfig.Storage{WriterDatabaseURL: owner}, raw); err != nil { + t.Fatal(err) + } + selection := dbconfig.SelectionPath(dir) + if _, err := dbconfig.AdoptSelection(selection, selected); err != nil { + t.Fatal(err) + } + t.Setenv("CORESCOPE_DATABASE_URL", owner) + t.Setenv("CORESCOPE_READER_DATABASE_URL", reader) + t.Setenv("CORESCOPE_WRITER_DATABASE_URL", writer) + t.Setenv("CORESCOPE_USERS_OWNER_DATABASE_URL", "") + t.Setenv("CORESCOPE_USERS_DATABASE_URL", "") + t.Setenv("CORESCOPE_APPROVED_CHANNELS_DATABASE_URL", "") + o := storageOptions{Action: "setup", Offline: true, SelectionFile: selection, ConfigDir: t.TempDir()} + if err := runStorageAction(context.Background(), o, &bytes.Buffer{}); err != nil { + t.Fatal("ordinary PG setup invented an account target", err) + } + unchanged, err := dbconfig.ReadSelection(selection) + if err != nil || unchanged.Accounts != nil || unchanged.Generation != selected.Generation { + t.Fatal("ordinary PG setup changed absence", err) + } + t.Setenv("CORESCOPE_USERS_OWNER_DATABASE_URL", accountOwner) + if err := runStorageAction(context.Background(), o, &bytes.Buffer{}); err == nil { + t.Fatal("account initialization lacked runtime credentials") + } + unchanged, err = dbconfig.ReadSelection(selection) + if err != nil || unchanged.Accounts != nil { + t.Fatal("failed preflight changed selection", err) + } + t.Setenv("CORESCOPE_USERS_DATABASE_URL", restrictedRole(t, accountOwner)) + o.beforeCommit = func() error { return errors.New("interrupt account commit") } + if err := runStorageAction(context.Background(), o, &bytes.Buffer{}); err == nil { + t.Fatal("injected account commit passed") + } + status, err := dbconfig.InspectSelection(selection) + if err != nil || status.SourceBackend != dbconfig.Postgres || status.TargetBackend != dbconfig.Postgres { + t.Fatal("PG account journal", status, err) + } + o.Action = "resume" + o.JobID = status.JobID + o.beforeCommit = nil + if err := runStorageAction(context.Background(), o, &bytes.Buffer{}); err != nil { + t.Fatal("PG account initialization resume", err) + } + after, err := dbconfig.ReadSelection(selection) + if err != nil || after.Accounts == nil || after.Generation == selected.Generation { + t.Fatal("account target not atomically recorded", err) + } +} diff --git a/cmd/migrate/account_v6_roundtrip_test.go b/cmd/migrate/account_v6_roundtrip_test.go new file mode 100644 index 000000000..d07b07176 --- /dev/null +++ b/cmd/migrate/account_v6_roundtrip_test.go @@ -0,0 +1,120 @@ +package main + +import ( + "context" + "path/filepath" + "testing" + + "github.com/meshcore-analyzer/users" +) + +func TestBothSQLiteV6AccountLayoutsRoundTrip(t *testing.T) { + for _, native := range []bool{false, true} { + name := "upstream-companion-v6" + if native { + name = "fork-identity-v6" + } + t.Run(name, func(t *testing.T) { + source := accountSource(t, 6) + db, err := openSQLite(source, "rw") + if err != nil { + t.Fatal(err) + } + if native { + if err := users.ApplySQLite(db); err != nil { + db.Close() + t.Fatal(err) + } + // Reconstruct the previous native v6 exact shape: identity tables, + // no companion columns/tables. No source is weakened or guessed. + for _, q := range []string{ + `DROP TABLE companion_links`, `DROP TABLE link_challenges`, + `ALTER TABLE sessions DROP COLUMN kind`, `ALTER TABLE sessions DROP COLUMN label`, `ALTER TABLE sessions DROP COLUMN scopes`, + `UPDATE schema_version SET version=6`, + `UPDATE sqlite_sequence SET seq=10000 WHERE name='users'`, + } { + if _, err := db.Exec(q); err != nil { + db.Close() + t.Fatal(err) + } + } + } else { + for _, q := range []string{ + `UPDATE sessions SET kind='device',label='Radio',scopes='rx,settings' WHERE id=90`, + `INSERT INTO companion_links(pubkey,user_id,name,linked_at) VALUES('fixture-key',0,'Companion',1)`, + `INSERT INTO link_challenges(challenge_hash,user_id,pubkey,expires_at) VALUES('fixture-challenge',0,'fixture-key',9999999999)`, + } { + if _, err := db.Exec(q); err != nil { + db.Close() + t.Fatal(err) + } + } + } + if err := users.ValidateSQLiteSourceLayout(db); err != nil { + db.Close() + t.Fatal(err) + } + db.Close() + before, err := sourceFingerprint(source) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + dsn := postgresDatabase(t) + if _, err := importSQLite(ctx, importOptions{Source: source, DatabaseURL: dsn, StateDir: t.TempDir(), Kind: "accounts", BatchSize: 1}); err != nil { + t.Fatal(err) + } + if _, err := finalizeImport(ctx, dsn, "accounts"); err != nil { + t.Fatal(err) + } + after, err := sourceFingerprint(source) + if err != nil || before != after { + t.Fatal("source changed", err) + } + dest := filepath.Join(t.TempDir(), "accounts.sqlite") + if report, err := reversePostgres(ctx, reverseOptions{DatabaseURL: dsn, Destination: dest, StateDir: t.TempDir(), Kind: "accounts", BatchSize: 1}); err != nil || !report.Verified { + t.Fatal("reverse", err) + } + db, err = openSQLite(dest, "rw") + if err != nil { + t.Fatal(err) + } + defer db.Close() + if err := users.AssertSQLiteReady(db); err != nil { + t.Fatal(err) + } + var token, kind, label, scopes string + if err := db.QueryRow(`SELECT token_hash,kind,label,scopes FROM sessions WHERE id=90`).Scan(&token, &kind, &label, &scopes); err != nil { + t.Fatal(err) + } + if token != "preserved-token" { + t.Fatal("session token changed") + } + if native && (kind != "web" || label != "" || scopes != "") { + t.Fatal("native v6 session changed") + } + if !native && (kind != "device" || label != "Radio" || scopes != "rx,settings") { + t.Fatal("companion v6 device session changed") + } + for _, table := range []string{"companion_links", "link_challenges"} { + var count int + if err := db.QueryRow(`SELECT count(*) FROM ` + table).Scan(&count); err != nil { + t.Fatal(err) + } + want := 1 + if native { + want = 0 + } + if count != want { + t.Fatalf("%s rows=%d want=%d", table, count, want) + } + } + if native { + var id int64 + if err := db.QueryRow(`INSERT INTO users(email,display_name,password_hash,created_at) VALUES('after@example.invalid','','hash',2) RETURNING id`).Scan(&id); err != nil || id <= 10000 { + t.Fatal("identity high water changed", id, err) + } + } + }) + } +} diff --git a/cmd/migrate/backend_test.go b/cmd/migrate/backend_test.go new file mode 100644 index 000000000..40e5f7d09 --- /dev/null +++ b/cmd/migrate/backend_test.go @@ -0,0 +1,41 @@ +package main + +import ( + "fmt" + "github.com/meshcore-analyzer/pgutil/pgtest" + "os" + "testing" +) + +func TestMain(m *testing.M) { + backend := os.Getenv("CORESCOPE_TEST_BACKEND") + if backend != "" && backend != "sqlite" && backend != "postgres" { + fmt.Fprintln(os.Stderr, "unsupported CORESCOPE_TEST_BACKEND") + os.Exit(1) + } + if backend == "postgres" && os.Getenv("CORESCOPE_TEST_POSTGRES_URL") == "" { + fmt.Fprintln(os.Stderr, "explicit PostgreSQL matrix requires CORESCOPE_TEST_POSTGRES_URL") + os.Exit(1) + } + os.Exit(m.Run()) +} + +func requirePostgres(t testing.TB) { + t.Helper() + switch os.Getenv("CORESCOPE_TEST_BACKEND") { + case "", "sqlite": + t.Skip("select CORESCOPE_TEST_BACKEND=postgres for the required PostgreSQL matrix") + case "postgres": + if os.Getenv("CORESCOPE_TEST_POSTGRES_URL") == "" { + t.Fatal("explicit PostgreSQL matrix requires CORESCOPE_TEST_POSTGRES_URL") + } + default: + t.Fatal("unsupported CORESCOPE_TEST_BACKEND") + } +} +func postgresSchema(t testing.TB) string { t.Helper(); requirePostgres(t); return pgtest.NewSchema(t) } +func postgresDatabase(t testing.TB) string { + t.Helper() + requirePostgres(t) + return pgtest.NewDatabase(t) +} diff --git a/cmd/migrate/cli_test.go b/cmd/migrate/cli_test.go new file mode 100644 index 000000000..3e79c41c3 --- /dev/null +++ b/cmd/migrate/cli_test.go @@ -0,0 +1,105 @@ +package main + +import ( + "bytes" + "context" + "database/sql" + "strings" + "testing" + + "github.com/meshcore-analyzer/dbschema" +) + +func TestCheckReadyDoesNotBootstrap(t *testing.T) { + dsn := postgresSchema(t) + t.Setenv("CORESCOPE_DATABASE_URL", dsn) + t.Setenv("CORESCOPE_USERS_DATABASE_URL", "") + var output bytes.Buffer + if err := runCommand(context.Background(), []string{"-check-ready"}, &output); err == nil { + t.Fatal("empty schema reported ready") + } + var n int + if err := openImportDB(t, dsn).QueryRow(`SELECT count(*) FROM information_schema.tables WHERE table_schema=current_schema()`).Scan(&n); err != nil { + t.Fatal(err) + } + if n != 0 { + t.Fatal("readiness check created tables") + } + if err := runCommand(context.Background(), nil, &output); err != nil { + t.Fatal(err) + } + if err := runCommand(context.Background(), []string{"-check-ready"}, &output); err != nil { + t.Fatal(err) + } + if strings.Contains(output.String(), dsn) { + t.Fatal("CLI output leaked a database URL") + } +} + +func TestCLIImportRequiresOfflineAndDoesNotFinalizeOnAccountFailure(t *testing.T) { + source := telemetrySource(t) + telemetry, accounts := postgresDatabase(t), postgresDatabase(t) + t.Setenv("CORESCOPE_DATABASE_URL", telemetry) + t.Setenv("CORESCOPE_USERS_DATABASE_URL", accounts) + var output bytes.Buffer + args := []string{"-from-sqlite", source, "-state-dir", t.TempDir()} + if err := runCommand(context.Background(), args, &output); err == nil { + t.Fatal("import accepted without explicit offline confirmation") + } + args = append(args, "-offline", "-users-from-sqlite", "missing-account-source.db") + if err := runCommand(context.Background(), args, &output); err == nil { + t.Fatal("missing account source was ignored") + } + if err := dbschema.AssertReady(openImportDB(t, telemetry)); err == nil { + t.Fatal("telemetry became ready despite failed account import") + } +} + +func TestCheckImportMarkerBindsExactSource(t *testing.T) { + dsn, source := postgresSchema(t), telemetrySource(t) + t.Setenv("CORESCOPE_DATABASE_URL", dsn) + t.Setenv("CORESCOPE_USERS_DATABASE_URL", "") + var output bytes.Buffer + args := []string{"-check-import-kind=telemetry", "-from-sqlite", source} + if err := runCommand(context.Background(), args, &output); err == nil { + t.Fatal("empty database passed imported-source check") + } + if _, err := importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: dsn, StateDir: t.TempDir(), Kind: "telemetry"}); err != nil { + t.Fatal(err) + } + if err := runCommand(context.Background(), args, &output); err == nil { + t.Fatal("unfinalized import passed bootstrap guard") + } + if _, err := finalizeImport(context.Background(), dsn, "telemetry"); err != nil { + t.Fatal(err) + } + if err := runCommand(context.Background(), args, &output); err != nil { + t.Fatal(err) + } + args[len(args)-1] = accountSource(t, 1) + if err := runCommand(context.Background(), args, &output); err == nil { + t.Fatal("different legacy source passed bootstrap guard") + } +} + +func TestWALModeCleanShutdownImportAndGuard(t *testing.T) { + source := telemetrySource(t) + db, err := sql.Open("sqlite3", source) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`PRAGMA journal_mode=WAL`); err != nil { + t.Fatal(err) + } + db.Close() + dsn := postgresSchema(t) + t.Setenv("CORESCOPE_DATABASE_URL", dsn) + t.Setenv("CORESCOPE_USERS_DATABASE_URL", "") + var output bytes.Buffer + if err := runCommand(context.Background(), []string{"-offline", "-from-sqlite", source, "-state-dir", t.TempDir()}, &output); err != nil { + t.Fatal(err) + } + if err := runCommand(context.Background(), []string{"-check-import-kind=telemetry", "-from-sqlite", source}, &output); err != nil { + t.Fatal(err) + } +} diff --git a/cmd/migrate/datetime_test.go b/cmd/migrate/datetime_test.go new file mode 100644 index 000000000..f1c2308bf --- /dev/null +++ b/cmd/migrate/datetime_test.go @@ -0,0 +1,152 @@ +package main + +import ( + "context" + "database/sql" + "errors" + "strings" + "testing" + + "github.com/meshcore-analyzer/dbschema" +) + +func dateSource(t *testing.T, declared string, values []any) string { + t.Helper() + source := telemetrySource(t) + db, err := openSQLite(source, "rw") + if err != nil { + t.Fatal(err) + } + defer db.Close() + if declared != "DATETIME" { + var ddl string + if err := db.QueryRow(`SELECT sql FROM sqlite_master WHERE name='dropped_packets'`).Scan(&ddl); err != nil { + t.Fatal(err) + } + if !strings.Contains(ddl, "dropped_at DATETIME") { + t.Fatal("legacy date declaration changed") + } + if _, err := db.Exec(`DROP TABLE dropped_packets`); err != nil { + t.Fatal(err) + } + if _, err := db.Exec(strings.Replace(ddl, "dropped_at DATETIME", "dropped_at "+declared, 1)); err != nil { + t.Fatal(err) + } + } + for i, value := range values { + if _, err := db.Exec(`INSERT INTO dropped_packets(id,reason,dropped_at) VALUES(?,?,?)`, i, "synthetic", value); err != nil { + t.Fatal(err) + } + } + if err := db.Close(); err != nil { + t.Fatal(err) + } + return source +} + +func TestImportPreservesDeclaredDateTextAndResume(t *testing.T) { + values := []any{ + nil, + "2026-10-09T01:02:03.123456789-04:30", + "2026-10-09 01:02:03.1200+00:00", + "2026-10-09T01:02:03Z", + "2026-10-09", + "", + "not a timestamp", + } + for _, declared := range []string{"DATETIME", "DATE", "TIMESTAMP"} { + t.Run(declared, func(t *testing.T) { + source := dateSource(t, declared, values) + before, err := sourceFingerprint(source) + if err != nil { + t.Fatal(err) + } + dsn := postgresSchema(t) + target := openImportDB(t, dsn) + options := importOptions{Source: source, DatabaseURL: dsn, StateDir: t.TempDir(), Kind: "telemetry", BatchSize: 2} + options.afterBatch = func() error { + var rows int + if err := target.QueryRow(`SELECT count(*) FROM dropped_packets`).Scan(&rows); err != nil { + return err + } + if rows == 2 { + return context.Canceled + } + return nil + } + if _, err := importSQLite(context.Background(), options); !errors.Is(err, context.Canceled) { + t.Fatalf("expected interruption after preserving two date rows: %v", err) + } + if err := dbschema.AssertReady(target); err == nil { + t.Fatal("interrupted date import became ready") + } + options.Resume, options.afterBatch = true, nil + report, err := importSQLite(context.Background(), options) + if err != nil || !report.Verified { + t.Fatalf("resume date import: verified=%v error=%v", report.Verified, err) + } + if _, err := finalizeImport(context.Background(), dsn, "telemetry"); err != nil { + t.Fatal(err) + } + if err := dbschema.AssertReady(target); err != nil { + t.Fatal(err) + } + rows, err := target.Query(`SELECT id,dropped_at FROM dropped_packets ORDER BY id`) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + count := 0 + for rows.Next() { + var id int + var value sql.NullString + if err := rows.Scan(&id, &value); err != nil { + t.Fatal(err) + } + if id != count || id >= len(values) { + t.Fatalf("date row identity changed: %d", id) + } + if want, ok := values[id].(string); ok { + if !value.Valid || value.String != want { + t.Errorf("date text changed at row %d: got %q want %q", id, value.String, want) + } + } else if value.Valid { + t.Errorf("NULL date became non-NULL at row %d", id) + } + count++ + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + if count != len(values) { + t.Fatalf("date rows duplicated or lost: %d", count) + } + after, err := sourceFingerprint(source) + if err != nil || before != after { + t.Fatal("date import changed the original SQLite source", err) + } + }) + } +} + +func TestImportRefusesNumericStorageInDeclaredDateColumn(t *testing.T) { + for _, value := range []any{int64(1700000000), 2.5, "12345"} { + source := dateSource(t, "DATETIME", []any{value}) + before, err := sourceFingerprint(source) + if err != nil { + t.Fatal(err) + } + dsn := postgresSchema(t) + report, err := importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: dsn, StateDir: t.TempDir(), Kind: "telemetry"}) + if err == nil || !strings.Contains(err.Error(), "non-text value") || report.Verified { + t.Fatalf("non-text date storage was silently coerced: verified=%v error=%v", report.Verified, err) + } + if err := dbschema.AssertReady(openImportDB(t, dsn)); err == nil { + t.Fatal("invalid date source became ready") + } + after, err := sourceFingerprint(source) + if err != nil || before != after { + t.Fatal("refused date import changed the source", err) + } + } +} diff --git a/cmd/migrate/destination_empty_test.go b/cmd/migrate/destination_empty_test.go new file mode 100644 index 000000000..519e674c9 --- /dev/null +++ b/cmd/migrate/destination_empty_test.go @@ -0,0 +1,37 @@ +package main + +import ( + "context" + "strings" + "testing" +) + +func TestFreshImportRefusesStandaloneSequence(t *testing.T) { + dsn := postgresSchema(t) + db := openImportDB(t, dsn) + if _, err := db.Exec(`CREATE SEQUENCE retained_sequence START WITH 77`); err != nil { + t.Fatal(err) + } + source := accountSource(t, 5) + before, err := fingerprint(context.Background(), source) + if err != nil { + t.Fatal(err) + } + _, err = importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: dsn, StateDir: t.TempDir(), Kind: "accounts"}) + if err == nil || !strings.Contains(err.Error(), "not empty") { + t.Fatalf("fresh import accepted a sequence-only occupied target: %v", err) + } + var value int + var called bool + if err := db.QueryRow(`SELECT last_value,is_called FROM retained_sequence`).Scan(&value, &called); err != nil || value != 77 || called { + t.Fatal("fresh import refusal changed the existing sequence", err) + } + var tables int + if err := db.QueryRow(`SELECT count(*) FROM information_schema.tables WHERE table_schema=current_schema()`).Scan(&tables); err != nil || tables != 0 { + t.Fatal("fresh import refusal created destination tables", err) + } + after, err := fingerprint(context.Background(), source) + if err != nil || before != after { + t.Fatal("fresh import refusal changed the retained source", err) + } +} diff --git a/cmd/migrate/generated_columns_test.go b/cmd/migrate/generated_columns_test.go new file mode 100644 index 000000000..56ed32272 --- /dev/null +++ b/cmd/migrate/generated_columns_test.go @@ -0,0 +1,81 @@ +package main + +import ( + "context" + "database/sql" + "path/filepath" + "strings" + "testing" + + "github.com/meshcore-analyzer/users" +) + +func TestImportRejectsGeneratedAccountColumnsBeforeNormalization(t *testing.T) { + for _, storage := range []string{"STORED", "VIRTUAL"} { + for _, column := range []string{"unsupported_note", "last_login_at"} { + t.Run(storage+"/"+column, func(t *testing.T) { + source := filepath.Join(t.TempDir(), "users.db") + db, err := sql.Open("sqlite3", source) + if err != nil { + t.Fatal(err) + } + defer db.Close() + if _, err := db.Exec(`CREATE TABLE schema_version(version INTEGER NOT NULL); INSERT INTO schema_version VALUES(1)`); err != nil { + t.Fatal(err) + } + for _, stmt := range users.LegacyMigrations()[0] { + if strings.HasPrefix(stmt, "CREATE TABLE users (") { + if column == "last_login_at" { + stmt = strings.Replace(stmt, "last_login_at INTEGER,", "last_login_at INTEGER GENERATED ALWAYS AS (created_at + 1) "+storage+",", 1) + } else { + stmt = strings.TrimSuffix(strings.TrimSpace(stmt), ")") + ", unsupported_note TEXT GENERATED ALWAYS AS (display_name || ':retained') " + storage + ")" + } + } + if _, err := db.Exec(stmt); err != nil { + t.Fatal(err) + } + } + if _, err := db.Exec(`INSERT INTO users(id,email,display_name,password_hash,role,status,created_at) VALUES(42,'synthetic@example.invalid','synthetic','hash','user','active',1)`); err != nil { + t.Fatal(err) + } + var value string + if err := db.QueryRow(`SELECT ` + quote(column) + ` FROM users WHERE id=42`).Scan(&value); err != nil || value == "" { + t.Fatal("generated fixture value missing", err) + } + if err := db.Close(); err != nil { + t.Fatal(err) + } + before, err := sourceFingerprint(source) + if err != nil { + t.Fatal(err) + } + dsn, state := postgresSchema(t), t.TempDir() + report, err := importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: dsn, StateDir: state, Kind: "accounts"}) + if err == nil || !strings.Contains(err.Error(), "generated or hidden") { + t.Errorf("generated source must be explicitly refused before normalization: verified=%v error=%v", report.Verified, err) + } + target := openImportDB(t, dsn) + var tables int + if err := target.QueryRow(`SELECT COUNT(*) FROM information_schema.tables WHERE table_schema=current_schema()`).Scan(&tables); err != nil || tables != 0 { + t.Errorf("unsupported source changed the target: tables=%d error=%v", tables, err) + } + if err := users.AssertReady(target); err == nil { + t.Error("unsupported source became runtime-ready") + } + normalized, err := openSQLite(filepath.Join(state, "accounts", "normalized.sqlite"), "ro") + if err != nil { + t.Fatal(err) + } + defer normalized.Close() + var version int + if err := normalized.QueryRow(`SELECT version FROM schema_version`).Scan(&version); err != nil || version != 1 { + t.Errorf("unsupported source reached account normalization: version=%d error=%v", version, err) + } + after, err := sourceFingerprint(source) + if err != nil || before != after { + t.Fatal("original source changed", err) + } + }) + } + } +} diff --git a/cmd/migrate/go.mod b/cmd/migrate/go.mod index 1d24e0a43..abaa9c9b0 100644 --- a/cmd/migrate/go.mod +++ b/cmd/migrate/go.mod @@ -1,10 +1,48 @@ module github.com/corescope/migrate -go 1.22 +go 1.25.0 require ( + github.com/jackc/pgx/v5 v5.7.6 github.com/mattn/go-sqlite3 v1.14.52 + github.com/meshcore-analyzer/dbconfig v0.0.0 github.com/meshcore-analyzer/dbschema v0.0.0 + github.com/meshcore-analyzer/pgutil v0.0.0 + github.com/meshcore-analyzer/sqliteutil v0.0.0 + github.com/meshcore-analyzer/users v0.0.0 +) + +require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/ncruces/go-strftime v0.1.9 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + modernc.org/libc v1.55.3 // indirect + modernc.org/mathutil v1.6.0 // indirect + modernc.org/memory v1.8.0 // indirect + modernc.org/sqlite v1.34.5 // indirect +) + +require ( + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/meshcore-analyzer/packetpath v0.0.0 + golang.org/x/crypto v0.37.0 // indirect + golang.org/x/sync v0.13.0 // indirect + golang.org/x/sys v0.32.0 // indirect + golang.org/x/text v0.24.0 // indirect ) replace github.com/meshcore-analyzer/dbschema => ../../internal/dbschema + +replace github.com/meshcore-analyzer/users => ../../internal/users + +replace github.com/meshcore-analyzer/pgutil => ../../internal/pgutil + +replace github.com/meshcore-analyzer/packetpath => ../../internal/packetpath + +replace github.com/meshcore-analyzer/dbconfig => ../../internal/dbconfig + +replace github.com/meshcore-analyzer/sqliteutil => ../../internal/sqliteutil diff --git a/cmd/migrate/go.sum b/cmd/migrate/go.sum index 75bf6eb53..eb0d9a0d8 100644 --- a/cmd/migrate/go.sum +++ b/cmd/migrate/go.sum @@ -1,2 +1,73 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo= +github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.7.6 h1:rWQc5FwZSPX58r1OQmkuaNicxdmExaEz5A2DO2hUuTk= +github.com/jackc/pgx/v5 v5.7.6/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-sqlite3 v1.14.52 h1:wVbm2Qnf4OXkqhBTSPuCRZDRnxfbVrrmiCEroVdog8U= github.com/mattn/go-sqlite3 v1.14.52/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w= +github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= +github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= +golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= +golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= +golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= +golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/cc/v4 v4.21.4 h1:3Be/Rdo1fpr8GrQ7IVw9OHtplU4gWbb+wNgeoBMmGLQ= +modernc.org/cc/v4 v4.21.4/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ= +modernc.org/ccgo/v4 v4.19.2 h1:lwQZgvboKD0jBwdaeVCTouxhxAyN6iawF3STraAal8Y= +modernc.org/ccgo/v4 v4.19.2/go.mod h1:ysS3mxiMV38XGRTTcgo0DQTeTmAO4oCmJl1nX9VFI3s= +modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE= +modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ= +modernc.org/gc/v2 v2.4.1 h1:9cNzOqPyMJBvrUipmynX0ZohMhcxPtMccYgGOJdOiBw= +modernc.org/gc/v2 v2.4.1/go.mod h1:wzN5dK1AzVGoH6XOzc3YZ+ey/jPgYHLuVckd62P0GYU= +modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U= +modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w= +modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= +modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= +modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E= +modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU= +modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= +modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= +modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc= +modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss= +modernc.org/sqlite v1.34.5 h1:Bb6SR13/fjp15jt70CL4f18JIN7p7dnMExd+UFnF15g= +modernc.org/sqlite v1.34.5/go.mod h1:YLuNmX9NKs8wRNK2ko1LW1NGYcc9FkBO69JOt1AR9JE= +modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= +modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/cmd/migrate/grants.go b/cmd/migrate/grants.go new file mode 100644 index 000000000..372418ecb --- /dev/null +++ b/cmd/migrate/grants.go @@ -0,0 +1,123 @@ +package main + +import ( + "context" + "errors" + "strings" + + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/pgutil" +) + +// Same narrow policy as docker/postgres-grants.sql, applied to the verified +// unselected destination before runtime validation and the atomic selection +// commit. Runtime role names come from their explicit credential URLs; no +// password or connection URL enters a SQL statement or child process argv. +func grantRuntimeTargets(ctx context.Context, selected dbconfig.Selection, owners dbconfig.Storage, raw dbconfig.StorageInputs) error { + bound, err := selected.ApplyTo(raw) + if err != nil { + return err + } + runtime, err := dbconfig.ResolveStorage(bound) + if err != nil { + return err + } + role := func(dsn string) (string, error) { + if dsn == "" { + return "", errors.New("prepare the runtime role credentials before backend selection") + } + config, err := pgutil.ParseConfig(dsn) + if err != nil { + return "", err + } + return quote(config.User), nil + } + reader, err := role(runtime.ReaderDatabaseURL) + if err != nil { + return err + } + writer, err := role(runtime.WriterDatabaseURL) + if err != nil { + return err + } + if reader == writer { + return errors.New("PostgreSQL telemetry reader and writer must use separate restricted roles") + } + type grants struct { + kind, owner, schema string + roles []string + queries []string + } + telemetry := grants{kind: "telemetry", owner: owners.WriterDatabaseURL, schema: quote(selected.Telemetry.Postgres.Schema), roles: []string{reader, writer}} + telemetry.queries = []string{ + `GRANT SELECT ON ALL TABLES IN SCHEMA ` + telemetry.schema + ` TO ` + reader, + `GRANT SELECT ON ALL SEQUENCES IN SCHEMA ` + telemetry.schema + ` TO ` + reader, + `GRANT SELECT ON corescope_schema,packets_v TO ` + writer, + `GRANT USAGE,SELECT ON ALL SEQUENCES IN SCHEMA ` + telemetry.schema + ` TO ` + writer, + } + tables, err := layouts("telemetry") + if err != nil { + return err + } + for _, table := range tables { + telemetry.queries = append(telemetry.queries, `GRANT SELECT,INSERT,UPDATE,DELETE ON TABLE `+quote(table.Name)+` TO `+writer) + } + stores := []grants{telemetry} + if selected.Accounts != nil { + account, err := role(runtime.UsersDatabaseURL) + if err != nil { + return err + } + entry := grants{kind: "accounts", owner: owners.UsersDatabaseURL, schema: quote(selected.Accounts.Postgres.Schema), roles: []string{account}} + entry.queries = []string{`GRANT SELECT ON ALL TABLES IN SCHEMA ` + entry.schema + ` TO ` + account, `GRANT USAGE,SELECT ON ALL SEQUENCES IN SCHEMA ` + entry.schema + ` TO ` + account} + tables, err := layouts("accounts") + if err != nil { + return err + } + for _, table := range tables { + entry.queries = append(entry.queries, `GRANT SELECT,INSERT,UPDATE,DELETE ON TABLE `+quote(table.Name)+` TO `+account) + } + if runtime.ApprovedChannelsDatabaseURL != "" { + channel, err := role(runtime.ApprovedChannelsDatabaseURL) + if err != nil { + return err + } + if channel == account { + return errors.New("account writer and channel projection reader must use separate restricted roles") + } + entry.roles = append(entry.roles, channel) + entry.queries = append(entry.queries, `GRANT SELECT ON approved_channels,linked_companions,corescope_schema TO `+channel) + } + stores = append(stores, entry) + } + for _, store := range stores { + if store.owner == "" { + continue + } // Account-only initialization leaves telemetry grants unchanged. + db, err := pgutil.Open(store.owner, false) + if err != nil { + return err + } + tx, err := db.BeginTx(ctx, nil) + if err != nil { + db.Close() + return err + } + queries := append([]string{`REVOKE CREATE ON SCHEMA ` + store.schema + ` FROM PUBLIC`, `GRANT USAGE ON SCHEMA ` + store.schema + ` TO ` + strings.Join(store.roles, ",")}, store.queries...) + for _, query := range queries { + if _, err = tx.ExecContext(ctx, query); err != nil { + break + } + } + if err == nil { + err = tx.Commit() + } else { + tx.Rollback() + } + db.Close() + if err != nil { + return err + } + } + return nil +} diff --git a/cmd/migrate/identity_layout_test.go b/cmd/migrate/identity_layout_test.go new file mode 100644 index 000000000..b46d52f91 --- /dev/null +++ b/cmd/migrate/identity_layout_test.go @@ -0,0 +1,114 @@ +package main + +import ( + "context" + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/users" + "path/filepath" + "testing" +) + +func TestNewSQLiteIdentitySourceLayouts(t *testing.T) { + for _, kind := range []string{"telemetry", "accounts"} { + t.Run(kind, func(t *testing.T) { + source := filepath.Join(t.TempDir(), "identity # % é.sqlite") + db, err := openSQLite(source, "rwc") + if err != nil { + t.Fatal(err) + } + if kind == "telemetry" { + err = dbschema.ApplySQLite(db, nil) + } else { + err = users.ApplySQLite(db) + } + if err != nil { + db.Close() + t.Fatal(err) + } + var sourceVersion int + if kind == "accounts" { + if err := db.QueryRow(`SELECT version FROM schema_version`).Scan(&sourceVersion); err != nil { + db.Close() + t.Fatal(err) + } + } + if kind == "accounts" { + for _, q := range []string{ + `INSERT INTO users(id,email,display_name,password_hash,created_at) VALUES(5,'test@example.invalid','','hash',1)`, + `INSERT INTO mail_log(id,user_id,to_email,purpose,sent_at,last_event_at) VALUES(4,5,'test@example.invalid','test',1,1)`, + `INSERT INTO mail_events(id,mail_id,event,at) VALUES(-9,4,'zero',1),(0,4,'negative',2),(900,4,'delivered',3)`, + `UPDATE sqlite_sequence SET seq=9000 WHERE name='mail_events'`, + } { + if _, err := db.Exec(q); err != nil { + t.Fatal(err) + } + } + } + db.Close() + before, err := sourceFingerprint(source) + if err != nil { + t.Fatal(err) + } + tables, err := layouts(kind) + if err != nil { + t.Fatal(err) + } + _, normalized, err := prepareSource(context.Background(), importOptions{Source: source, StateDir: t.TempDir(), Kind: kind, targetDigest: "test-only"}, tables, true) + if err != nil { + t.Fatal(err) + } + after, err := sourceFingerprint(source) + if err != nil || after != before { + t.Fatal("source changed", err) + } + copyDB, err := openSQLite(normalized, "ro") + if err != nil { + t.Fatal(err) + } + defer copyDB.Close() + if kind == "accounts" { + var n, seq, version int + if err := copyDB.QueryRow(`SELECT count(*) FROM mail_events WHERE id IN (-9,0,900)`).Scan(&n); err != nil || n != 3 { + t.Fatal("explicit event IDs changed", n, err) + } + if err := copyDB.QueryRow(`SELECT seq FROM sqlite_sequence WHERE name='mail_events'`).Scan(&seq); err != nil || seq != 9000 { + t.Fatal("event sequence changed", seq, err) + } + if err := copyDB.QueryRow(`SELECT version FROM schema_version`).Scan(&version); err != nil || version != sourceVersion { + t.Fatal("SQLite source version changed", version, err) + } + } + }) + } +} + +func TestForwardNewSQLiteAccountIDsAndSequence(t *testing.T) { + dsn := postgresSchema(t) + source := accountSource(t, 5) + db, err := openSQLite(source, "rw") + if err != nil { + t.Fatal(err) + } + if err := users.ApplySQLite(db); err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`UPDATE sqlite_sequence SET seq=9000 WHERE name='mail_events'`); err != nil { + t.Fatal(err) + } + db.Close() + report, err := importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: dsn, StateDir: t.TempDir(), Kind: "accounts"}) + if err != nil || !report.Verified { + t.Fatal("forward native SQLite accounts", err) + } + pg := openImportDB(t, dsn) + var id, version int + if err := pg.QueryRow(`SELECT id FROM mail_events WHERE mail_id=4`).Scan(&id); err != nil || id != 900 { + t.Fatal("mail event identity", id, err) + } + if err := pg.QueryRow(`SELECT version FROM schema_version`).Scan(&version); err != nil || version != users.CurrentSchemaVersion { + t.Fatal("PG version", version, err) + } + if err := pg.QueryRow(`INSERT INTO mail_events(mail_id,event,at) VALUES(4,'later',44) RETURNING id`).Scan(&id); err != nil || id != 9001 { + t.Fatal("mail sequence", id, err) + } +} diff --git a/cmd/migrate/import.go b/cmd/migrate/import.go new file mode 100644 index 000000000..76484286e --- /dev/null +++ b/cmd/migrate/import.go @@ -0,0 +1,900 @@ +package main + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "database/sql" + "encoding/binary" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "hash" + "math" + "os" + "path/filepath" + "slices" + "strings" + "unicode/utf8" + + "github.com/jackc/pgx/v5" + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/pgutil" + "github.com/meshcore-analyzer/users" +) + +type importOptions struct { + Source, DatabaseURL, StateDir, Kind string + BatchSize int + Resume bool + afterBatch func() error + targetDigest string +} +type tableReport struct { + Table string `json:"table"` + Rows int64 `json:"rows"` + SHA256 string `json:"sha256"` +} +type importReport struct { + Kind string `json:"kind"` + Verified bool `json:"verified"` + Tables []tableReport `json:"tables"` + ExcludedDerived []derivedTableReport `json:"excluded_derived,omitempty"` +} +type sourceManifest struct { + Version int `json:"version"` + RunID string `json:"run_id"` + Kind string `json:"kind"` + SourceDigest string `json:"source_digest"` + RawDigest string `json:"raw_digest"` + NormalizedDigest string `json:"normalized_digest"` + TargetDigest string `json:"target_digest"` + ExcludedDerived []derivedTableReport `json:"excluded_derived,omitempty"` +} + +const importerLock int64 = 7289041410 + +const importDestinationObjects = `SELECT count(*) FROM pg_class WHERE relnamespace=current_schema()::regnamespace AND relkind IN ('r','p','v','m','S','f')` + +func fingerprint(ctx context.Context, path string) (string, error) { + fp, err := sourceFingerprintContext(ctx, path) + if err != nil { + return "", err + } + return fmt.Sprintf("%x:%x", fp[0], fp[1]), nil +} + +func prepareSource(ctx context.Context, o importOptions, tables []importTable, allowNew bool) (sourceManifest, string, error) { + var manifest sourceManifest + if o.Source == "" || o.StateDir == "" { + return manifest, "", errors.New("source and state directory are required for import") + } + digest, err := fingerprint(ctx, o.Source) + if err != nil { + return manifest, "", err + } + dir := filepath.Join(o.StateDir, o.Kind) + raw := filepath.Join(dir, "recovery.sqlite") + normalized := filepath.Join(dir, "normalized.sqlite") + path := filepath.Join(dir, "manifest.json") + data, err := os.ReadFile(path) + if err == nil { + if !o.Resume { + return manifest, "", errors.New("migration state already exists; use -resume with the same source and destination") + } + if err := json.Unmarshal(data, &manifest); err != nil { + return manifest, "", errors.New("invalid migration state manifest") + } + if manifest.Version != 1 || manifest.Kind != o.Kind || manifest.SourceDigest != digest || manifest.TargetDigest != o.targetDigest { + return manifest, "", errors.New("migration resume source or destination does not match the immutable manifest") + } + for p, want := range map[string]string{raw: manifest.RawDigest, normalized: manifest.NormalizedDigest} { + got, err := fingerprint(ctx, p) + if err != nil { + return manifest, "", err + } + if got != want { + return manifest, "", errors.New("migration recovery or normalized snapshot changed; resume refused") + } + } + excluded, err := recoveryDerivedExclusions(raw, o.Kind) + if err != nil { + return manifest, "", err + } + if !slices.Equal(excluded, manifest.ExcludedDerived) { + return manifest, "", errors.New("migration derived-schema report differs from immutable recovery; resume refused") + } + return manifest, normalized, nil + } + if !errors.Is(err, os.ErrNotExist) { + return manifest, "", err + } + entries, err := os.ReadDir(dir) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return manifest, "", err + } + if len(entries) != 0 { + return manifest, "", errors.New("preparation stopped before a complete manifest; preserve this state directory and the original SQLite files, then use a new state directory for this store only after confirming its PostgreSQL destination is empty") + } + if !allowNew { + return manifest, "", errors.New("no complete migration state manifest exists to resume") + } + if err := os.MkdirAll(dir, 0o700); err != nil { + return manifest, "", err + } + if err := snapshotSource(ctx, o.Source, raw); err != nil { + return manifest, "", err + } + if err := snapshotSource(ctx, raw, normalized); err != nil { + return manifest, "", err + } + db, err := openSQLite(normalized, "rw") + if err != nil { + return manifest, "", err + } + db.SetMaxOpenConns(1) + var excluded []derivedTableReport + if _, err = db.ExecContext(ctx, `PRAGMA trusted_schema=OFF`); err == nil { + if excluded, err = sqliteDerivedExclusions(db, o.Kind); err == nil { + err = normalizeSource(db, o.Kind, raw, tables) + } + } + closeErr := db.Close() + if err != nil { + return manifest, "", err + } + if closeErr != nil { + return manifest, "", closeErr + } + manifest = sourceManifest{Version: 1, Kind: o.Kind, SourceDigest: digest, TargetDigest: o.targetDigest, ExcludedDerived: excluded} + var random [16]byte + if _, err := rand.Read(random[:]); err != nil { + return manifest, "", err + } + manifest.RunID = hex.EncodeToString(random[:]) + if manifest.RawDigest, err = fingerprint(ctx, raw); err != nil { + return manifest, "", err + } + if manifest.NormalizedDigest, err = fingerprint(ctx, normalized); err != nil { + return manifest, "", err + } + after, err := fingerprint(ctx, o.Source) + if err != nil { + return manifest, "", err + } + if after != digest { + return manifest, "", errors.New("SQLite source changed during preparation; keep every instance writer stopped") + } + data, err = json.MarshalIndent(manifest, "", " ") + if err != nil { + return manifest, "", err + } + f, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if err != nil { + return manifest, "", err + } + _, writeErr := f.Write(data) + syncErr := f.Sync() + closeErr = f.Close() + if err := errors.Join(writeErr, syncErr, closeErr); err != nil { + return manifest, "", err + } + return manifest, normalized, nil +} + +func importSQLite(ctx context.Context, o importOptions) (importReport, error) { + report := importReport{Kind: o.Kind} + if err := ctx.Err(); err != nil { + return report, err + } + if o.BatchSize == 0 { + o.BatchSize = 1000 + } + if o.BatchSize < 1 || o.BatchSize > 10000 { + return report, errors.New("import batch size must be between 1 and 10000") + } + tables, err := layouts(o.Kind) + if err != nil { + return report, err + } + config, err := pgutil.ParseConfig(o.DatabaseURL) + if err != nil { + return report, err + } + owner, err := pgutil.Open(o.DatabaseURL, false) + if err != nil { + return report, err + } + defer owner.Close() + owner.SetMaxOpenConns(1) + var database, schema string + var databaseOID, schemaOID int64 + if err := owner.QueryRowContext(ctx, `SELECT current_database(),current_schema(), + (SELECT oid::bigint FROM pg_database WHERE datname=current_database()), + (SELECT oid::bigint FROM pg_namespace WHERE nspname=current_schema())`).Scan(&database, &schema, &databaseOID, &schemaOID); err != nil { + return report, err + } + // Container IPs can change while their persistent volume and configured DNS + // endpoint remain the same. Bind resume to that endpoint and logical OIDs; + // the destination's random import run ID is verified separately below. + identity, _ := json.Marshal([]any{database, schema, databaseOID, schemaOID, config.Host, config.Port}) + identityHash := sha256.Sum256(identity) + o.targetDigest = hex.EncodeToString(identityHash[:]) + conn, err := pgx.ConnectConfig(ctx, config) + if err != nil { + return report, errors.New("connect to PostgreSQL import destination failed") + } + defer conn.Close(context.Background()) + // One importer owns this database until completion/cancellation. Session lock + // works across committed COPY batches; closing the connection always releases it. + if err := lockImport(ctx, conn); err != nil { + return report, err + } + allowNew := !o.Resume + if o.Resume { + var hasImport bool + if err := conn.QueryRow(ctx, `SELECT to_regclass('corescope_import') IS NOT NULL`).Scan(&hasImport); err != nil { + return report, err + } + if !hasImport { + var occupied int + if err := conn.QueryRow(ctx, importDestinationObjects).Scan(&occupied); err != nil { + return report, err + } + if occupied != 0 { + return report, errors.New("resume destination has no import marker and is not empty; existing data was not changed") + } + allowNew = true + } + } + manifest, path, err := prepareSource(ctx, o, tables, allowNew) + if err != nil { + return report, err + } + report.ExcludedDerived = manifest.ExcludedDerived + source, err := openSQLite(path, "ro") + if err != nil { + return report, err + } + defer source.Close() + source.SetMaxOpenConns(1) + if _, err := source.ExecContext(ctx, `PRAGMA trusted_schema=OFF`); err != nil { + return report, err + } + + var exists bool + if err := conn.QueryRow(ctx, `SELECT to_regclass('corescope_import') IS NOT NULL`).Scan(&exists); err != nil { + return report, err + } + if !exists { + var occupied int + if err := conn.QueryRow(ctx, importDestinationObjects).Scan(&occupied); err != nil { + return report, err + } + if occupied != 0 { + return report, errors.New("import destination is not empty; existing data was not changed") + } + tx, err := owner.BeginTx(ctx, nil) + if err != nil { + return report, err + } + defer tx.Rollback() + if o.Kind == "telemetry" { + err = dbschema.ApplyForImportTx(tx, nil) + } else { + err = users.ApplyForImportTx(tx) + } + if err != nil { + return report, err + } + if _, err := tx.ExecContext(ctx, `CREATE TABLE corescope_import (id INTEGER PRIMARY KEY CHECK(id=1),run_id TEXT NOT NULL,kind TEXT NOT NULL,source_digest TEXT NOT NULL,normalized_digest TEXT NOT NULL,verified BOOLEAN NOT NULL DEFAULT false,report TEXT NOT NULL DEFAULT '') + ;CREATE TABLE corescope_import_progress (table_name TEXT PRIMARY KEY,rows_copied BIGINT NOT NULL,sha256 TEXT NOT NULL,complete BOOLEAN NOT NULL)`); err != nil { + return report, err + } + if _, err := tx.ExecContext(ctx, `INSERT INTO corescope_import(id,run_id,kind,source_digest,normalized_digest) VALUES(1,$1,$2,$3,$4)`, manifest.RunID, o.Kind, manifest.SourceDigest, manifest.NormalizedDigest); err != nil { + return report, err + } + if err := tx.Commit(); err != nil { + return report, err + } + } else if !o.Resume { + return report, errors.New("destination has an import in progress; use -resume") + } + var runID, kind, sourceDigest, normalizedDigest string + if err := conn.QueryRow(ctx, `SELECT run_id,kind,source_digest,normalized_digest FROM corescope_import WHERE id=1`).Scan(&runID, &kind, &sourceDigest, &normalizedDigest); err != nil { + return report, err + } + if runID != manifest.RunID || kind != o.Kind || sourceDigest != manifest.SourceDigest || normalizedDigest != manifest.NormalizedDigest { + return report, errors.New("destination import identity does not match this source/state directory") + } + var version int + var ready bool + if err := conn.QueryRow(ctx, `SELECT version,ready FROM corescope_schema WHERE kind=$1`, o.Kind).Scan(&version, &ready); err != nil { + return report, err + } + expectedVersion := users.PostgresSchemaVersion + if o.Kind == "telemetry" { + expectedVersion = dbschema.Version + } + if version != expectedVersion { + return report, errors.New("unsupported PostgreSQL schema version for import") + } + var recordedReport importReport + if ready { + // A process can stop between the two databases' readiness commits. Keep + // this completed store unchanged, but never trust readiness by itself. + tx, err := conn.Begin(ctx) + if err != nil { + return report, err + } + defer tx.Rollback(context.Background()) + lockTables := []string{"corescope_import", "corescope_import_progress", "corescope_schema"} + for _, table := range tables { + lockTables = append(lockTables, quote(table.Name)) + } + if _, err := tx.Exec(ctx, `LOCK TABLE `+strings.Join(lockTables, ",")+` IN SHARE MODE`); err != nil { + return report, err + } + var verified bool + var reportText string + if err := tx.QueryRow(ctx, `SELECT verified,report FROM corescope_import WHERE id=1`).Scan(&verified, &reportText); err != nil { + return report, err + } + if !verified || json.Unmarshal([]byte(reportText), &recordedReport) != nil || !recordedReport.Verified || recordedReport.Kind != o.Kind || len(recordedReport.Tables) != len(tables) { + return report, errors.New("ready destination has no complete verified import report; resume refused") + } + } + for _, table := range tables { + got, err := copyTable(ctx, source, owner, conn, table, o, ready) + if err != nil { + return report, fmt.Errorf("import %s: %w", table.Name, err) + } + report.Tables = append(report.Tables, got) + } + if err := reseed(ctx, source, conn, tables, o.Kind, ready); err != nil { + return report, err + } + // COPY does not populate planner statistics. Prepare only this store's + // imported tables before declaring verification complete; the cost belongs + // to migration timing, never to the steady-state ingestion benchmark. + if !ready { + for _, table := range tables { + if _, err := conn.Exec(ctx, `ANALYZE `+quote(table.Name)); err != nil { + return report, fmt.Errorf("analyze imported %s: %w", table.Name, err) + } + } + } + // Source stays frozen through verification, not merely while snapshotting. + current, err := fingerprint(ctx, o.Source) + if err != nil { + return report, err + } + if current != manifest.SourceDigest { + return report, errors.New("original SQLite source changed during import; keep services stopped") + } + normalized, err := fingerprint(ctx, path) + if err != nil { + return report, err + } + if normalized != manifest.NormalizedDigest { + return report, errors.New("normalized snapshot changed during import") + } + report.Verified = true + if ready { + if !slices.Equal(report.Tables, recordedReport.Tables) || !slices.Equal(report.ExcludedDerived, recordedReport.ExcludedDerived) { + return report, errors.New("ready destination differs from its verified import report; resume refused") + } + return report, nil + } + data, err := json.Marshal(report) + if err != nil { + return report, err + } + if _, err := conn.Exec(ctx, `UPDATE corescope_import SET verified=true,report=$1 WHERE id=1`, string(data)); err != nil { + return report, err + } + return report, nil +} + +func columnNames(table importTable) ([]string, []string) { + names := make([]string, len(table.Columns)) + quoted := make([]string, len(names)) + for i, col := range table.Columns { + names[i] = col.Name + quoted[i] = quote(col.Name) + } + return names, quoted +} + +func orderBy(db *sql.DB, table importTable, postgres, hiddenMailID bool) (string, error) { + // Sort by logical primary key with identical byte collation in both engines. + rows, err := db.Query(`SELECT a.attname FROM pg_index i JOIN pg_attribute a ON a.attrelid=i.indrelid AND a.attnum=ANY(i.indkey) + WHERE i.indrelid=to_regclass($1) AND i.indisprimary ORDER BY array_position(i.indkey,a.attnum)`, table.Name) + if err != nil { + return "", err + } + defer rows.Close() + var keys []string + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return "", err + } + keys = append(keys, name) + } + if err := rows.Err(); err != nil { + return "", err + } + if len(keys) == 0 { + return "", errors.New("destination table lacks a stable primary key") + } + var out []string + for _, name := range keys { + colType := "" + for _, col := range table.Columns { + if col.Name == name { + colType = col.Type + } + } + expr := quote(name) + if table.Name == "mail_events" && name == "id" && hiddenMailID { + expr = "rowid" + } + if colType == "TEXT" { + if postgres { + expr += ` COLLATE "C"` + } else { + expr += ` COLLATE BINARY` + } + } + if postgres { + expr += " NULLS FIRST" + } + out = append(out, expr) + } + return strings.Join(out, ","), nil +} + +func canonical(value any, kind string) (any, error) { + if value == nil { + return nil, nil + } + switch kind { + case "TEXT": + var s string + switch v := value.(type) { + case string: + s = v + case []byte: + s = string(v) + default: + return nil, errors.New("non-text value in a text column") + } + if !utf8.ValidString(s) || strings.ContainsRune(s, 0) { + return nil, errors.New("text contains invalid UTF-8 or NUL, which PostgreSQL cannot preserve") + } + return s, nil + case "BIGINT": + switch v := value.(type) { + case int64: + return v, nil + case int32: + return int64(v), nil + case int: + return int64(v), nil + case float64: + if math.Trunc(v) == v && v >= math.MinInt64 && v < 9223372036854775808.0 { + return int64(v), nil + } + } + return nil, errors.New("non-integral or out-of-range value in an integer column") + case "DOUBLE PRECISION": + switch v := value.(type) { + case float64: + return v, nil + case int64: + if int64(float64(v)) != v { + return nil, errors.New("integer cannot be represented exactly by the destination floating-point column") + } + return float64(v), nil + case int32: + return float64(v), nil + } + return nil, errors.New("non-numeric value in a floating-point column") + } + return nil, fmt.Errorf("unsupported destination column type %s", kind) +} + +func hashRow(h hash.Hash, values []any) { + for _, value := range values { + switch v := value.(type) { + case nil: + h.Write([]byte{0}) + case int64: + h.Write([]byte{1}) + var b [8]byte + binary.BigEndian.PutUint64(b[:], uint64(v)) + h.Write(b[:]) + case float64: + h.Write([]byte{2}) + var b [8]byte + binary.BigEndian.PutUint64(b[:], math.Float64bits(v)) + h.Write(b[:]) + case string: + h.Write([]byte{3}) + var b [8]byte + binary.BigEndian.PutUint64(b[:], uint64(len(v))) + h.Write(b[:]) + h.Write([]byte(v)) + } + } + h.Write([]byte{255}) +} + +func digestTarget(ctx context.Context, db *sql.DB, table importTable) (tableReport, error) { + report := tableReport{Table: table.Name} + _, quoted := columnNames(table) + order, err := orderBy(db, table, true, false) + if err != nil { + return report, err + } + rows, err := db.QueryContext(ctx, `SELECT `+strings.Join(quoted, ",")+` FROM `+quote(table.Name)+` ORDER BY `+order) + if err != nil { + return report, err + } + defer rows.Close() + h := sha256.New() + for rows.Next() { + values := make([]any, len(quoted)) + dest := make([]any, len(quoted)) + for i := range dest { + dest[i] = &values[i] + } + if err := rows.Scan(dest...); err != nil { + return report, err + } + for i, col := range table.Columns { + values[i], err = canonical(values[i], col.Type) + if err != nil { + return report, err + } + } + hashRow(h, values) + report.Rows++ + } + report.SHA256 = hex.EncodeToString(h.Sum(nil)) + return report, rows.Err() +} + +func copyTable(ctx context.Context, source, owner *sql.DB, conn *pgx.Conn, table importTable, o importOptions, verifyOnly bool) (tableReport, error) { + report := tableReport{Table: table.Name} + var copied int64 + var storedDigest string + var complete bool + err := conn.QueryRow(ctx, `SELECT rows_copied,sha256,complete FROM corescope_import_progress WHERE table_name=$1`, table.Name).Scan(&copied, &storedDigest, &complete) + if err != nil && !errors.Is(err, pgx.ErrNoRows) { + return report, err + } + if verifyOnly && !complete { + return report, errors.New("ready destination lacks complete import progress; resume refused") + } + existing, err := digestTarget(ctx, owner, table) + if err != nil { + return report, err + } + emptyHash := sha256.Sum256(nil) + if storedDigest == "" { + storedDigest = hex.EncodeToString(emptyHash[:]) + } + if existing.Rows != copied || existing.SHA256 != storedDigest { + return report, errors.New("destination rows differ from committed import progress; resume refused") + } + var exists int + if err := source.QueryRowContext(ctx, `SELECT count(*) FROM sqlite_master WHERE type='table' AND name=?`, table.Name).Scan(&exists); err != nil { + return report, err + } + h := sha256.New() + names, quoted := columnNames(table) + var batch [][]any + var batchBytes int + flush := func(done bool) error { + if len(batch) == 0 && !done { + return nil + } + tx, err := conn.Begin(ctx) + if err != nil { + return err + } + defer tx.Rollback(context.Background()) + if len(batch) > 0 { + if n, err := tx.CopyFrom(ctx, pgx.Identifier{table.Name}, names, pgx.CopyFromRows(batch)); err != nil { + return err + } else if n != int64(len(batch)) { + return errors.New("COPY row count mismatch") + } + } + digest := hex.EncodeToString(h.Sum(nil)) + if _, err := tx.Exec(ctx, `INSERT INTO corescope_import_progress(table_name,rows_copied,sha256,complete) VALUES($1,$2,$3,$4) + ON CONFLICT(table_name) DO UPDATE SET rows_copied=excluded.rows_copied,sha256=excluded.sha256,complete=excluded.complete`, table.Name, report.Rows, digest, done); err != nil { + return err + } + if err := tx.Commit(ctx); err != nil { + return err + } + wrote := len(batch) > 0 + batch = nil + batchBytes = 0 + if wrote && o.afterBatch != nil { + return o.afterBatch() + } + return nil + } + if exists != 0 { + hiddenMailID := false + if o.Kind == "accounts" && table.Name == "mail_events" { + var explicit int + if err := source.QueryRow(`SELECT count(*) FROM pragma_table_info('mail_events') WHERE name='id'`).Scan(&explicit); err != nil { + return report, err + } + hiddenMailID = explicit == 0 + } + order, err := orderBy(owner, table, false, hiddenMailID) + if err != nil { + return report, err + } + expressions := append([]string(nil), quoted...) + if hiddenMailID { + expressions[0] = "rowid" + } + // SQLite's unary + preserves the stored value/type while removing declared + // column metadata. Otherwise the driver parses DATE/DATETIME/TIMESTAMP + // values into time.Time before copy/resume hashing can preserve their text. + for i, expression := range expressions { + expressions[i] = "+" + expression + } + rows, err := source.QueryContext(ctx, `SELECT `+strings.Join(expressions, ",")+` FROM `+quote(table.Name)+` ORDER BY `+order) + if err != nil { + return report, err + } + defer rows.Close() + for rows.Next() { + values := make([]any, len(names)) + dest := make([]any, len(names)) + for i := range dest { + dest[i] = &values[i] + } + if err := rows.Scan(dest...); err != nil { + return report, err + } + rowBytes := 0 + for i, col := range table.Columns { + values[i], err = canonical(values[i], col.Type) + if err != nil { + return report, fmt.Errorf("column %s: %w", col.Name, err) + } + if s, ok := values[i].(string); ok { + rowBytes += len(s) + } else { + rowBytes += 8 + } + } + if rowBytes > maxImportRowBytes { + return report, errors.New("a source row exceeds the 64 MiB migration safety limit") + } + hashRow(h, values) + report.Rows++ + if report.Rows <= copied { + if report.Rows == copied && hex.EncodeToString(h.Sum(nil)) != storedDigest { + return report, errors.New("source prefix differs from import progress") + } + continue + } + if complete { + return report, errors.New("completed source table contains unexpected rows") + } + batch = append(batch, values) + batchBytes += rowBytes + if len(batch) >= o.BatchSize || batchBytes >= 8*1024*1024 { + if err := flush(false); err != nil { + return report, err + } + } + } + if err := rows.Err(); err != nil { + return report, err + } + rows.Close() + } + if report.Rows < copied { + return report, errors.New("source table shorter than committed progress") + } + if !verifyOnly { + if err := flush(true); err != nil { + return report, err + } + } + report.SHA256 = hex.EncodeToString(h.Sum(nil)) + final, err := digestTarget(ctx, owner, table) + if err != nil { + return report, err + } + if final != report { + return report, errors.New("row count or logical digest verification failed") + } + return report, nil +} + +func reseed(ctx context.Context, source *sql.DB, conn *pgx.Conn, tables []importTable, kind string, verifyOnly bool) error { + highWater := map[string]int64{} + var hasSequence int + if err := source.QueryRow(`SELECT count(*) FROM sqlite_master WHERE name='sqlite_sequence'`).Scan(&hasSequence); err != nil { + return err + } + if hasSequence > 0 { + rows, err := source.Query(`SELECT name,MAX(seq),count(*) FROM sqlite_sequence GROUP BY name`) + if err != nil { + return err + } + for rows.Next() { + var name string + var n int64 + var count int + if err := rows.Scan(&name, &n, &count); err != nil { + rows.Close() + return err + } + if count != 1 || n < 0 { + rows.Close() + return errors.New("source SQLite sequence state is invalid or duplicated") + } + highWater[name] = n + } + if err := rows.Err(); err != nil { + rows.Close() + return err + } + rows.Close() + } + references := identityReferences(kind) + for _, table := range tables { + for _, col := range table.Columns { + if !col.Identity { + continue + } + high := highWater[table.Name] + var n int64 + if err := conn.QueryRow(ctx, `SELECT COALESCE(MAX(`+quote(col.Name)+`),0) FROM `+quote(table.Name)).Scan(&n); err != nil { + return err + } + if n > high { + high = n + } + for _, ref := range references[table.Name] { + if err := conn.QueryRow(ctx, `SELECT COALESCE(MAX(`+quote(ref[1])+`),0) FROM `+quote(ref[0])).Scan(&n); err != nil { + return err + } + if n > high { + high = n + } + } + called := high > 0 + if !called { + high = 1 + } + if verifyOnly { + var sequence string + if err := conn.QueryRow(ctx, `SELECT pg_get_serial_sequence($1,$2)`, table.Name, col.Name).Scan(&sequence); err != nil { + return err + } + var actual int64 + var wasCalled bool + // pg_get_serial_sequence returns a safely quoted qualified identifier. + if err := conn.QueryRow(ctx, `SELECT last_value,is_called FROM `+sequence).Scan(&actual, &wasCalled); err != nil { + return err + } + if actual != high || wasCalled != called { + return fmt.Errorf("ready destination sequence for %s changed since import; resume refused", table.Name) + } + } else if _, err := conn.Exec(ctx, `SELECT setval(pg_get_serial_sequence($1,$2),$3,$4)`, table.Name, col.Name, high, called); err != nil { + return err + } + } + } + return nil +} + +// finalizeImport reports whether this call committed a closed-to-ready transition. +// The caller may roll back only transitions it owns if another store fails. +func finalizeImport(ctx context.Context, databaseURL, kind string) (bool, error) { + tables, err := layouts(kind) + if err != nil { + return false, err + } + db, err := pgutil.Open(databaseURL, false) + if err != nil { + return false, err + } + defer db.Close() + config, err := pgutil.ParseConfig(databaseURL) + if err != nil { + return false, err + } + conn, err := pgx.ConnectConfig(ctx, config) + if err != nil { + return false, errors.New("connect to PostgreSQL finalization destination failed") + } + defer conn.Close(context.Background()) + if err := lockImport(ctx, conn); err != nil { + return false, err + } + var verified bool + var recordedKind, reportText string + if err := conn.QueryRow(ctx, `SELECT kind,verified,report FROM corescope_import WHERE id=1`).Scan(&recordedKind, &verified, &reportText); err != nil { + return false, err + } + if !verified || recordedKind != kind { + return false, errors.New("cannot finalize an unverified or mismatched import") + } + var report importReport + if err := json.Unmarshal([]byte(reportText), &report); err != nil { + return false, err + } + if !report.Verified || report.Kind != kind || len(report.Tables) != len(tables) { + return false, errors.New("incomplete verification manifest") + } + tx, err := conn.Begin(ctx) + if err != nil { + return false, err + } + defer tx.Rollback(context.Background()) + var lockTables []string + for _, table := range tables { + lockTables = append(lockTables, quote(table.Name)) + } + if _, err := tx.Exec(ctx, `LOCK TABLE `+strings.Join(lockTables, ",")+` IN SHARE MODE`); err != nil { + return false, err + } + for i, table := range tables { + got, err := digestTarget(ctx, db, table) + if err != nil { + return false, err + } + if got != report.Tables[i] { + return false, fmt.Errorf("final verification failed for %s; target remains unready", table.Name) + } + } + var ready bool + version := users.PostgresSchemaVersion + if kind == "telemetry" { + version = dbschema.Version + } + if err := tx.QueryRow(ctx, `SELECT ready FROM corescope_schema WHERE kind=$1 AND version=$2`, kind, version).Scan(&ready); err != nil { + return false, errors.New("missing or unsupported readiness marker") + } + if ready { + return false, nil + } + result, err := tx.Exec(ctx, `UPDATE corescope_schema SET ready=true WHERE kind=$1 AND version=$2`, kind, version) + if err != nil { + return false, err + } + if result.RowsAffected() != 1 { + return false, errors.New("missing or unsupported readiness marker") + } + if err := tx.Commit(ctx); err != nil { + return false, err + } + return true, nil +} + +func lockImport(ctx context.Context, conn *pgx.Conn) error { + var locked bool + if err := conn.QueryRow(ctx, `SELECT pg_try_advisory_lock(hashtextextended(current_schema(),$1))`, importerLock).Scan(&locked); err != nil { + return err + } + if !locked { + return errors.New("another importer owns this destination; wait for it to finish before retrying") + } + return nil +} diff --git a/cmd/migrate/import_safety_test.go b/cmd/migrate/import_safety_test.go new file mode 100644 index 000000000..7203fbbbb --- /dev/null +++ b/cmd/migrate/import_safety_test.go @@ -0,0 +1,293 @@ +package main + +import ( + "context" + "database/sql" + "errors" + "fmt" + "path/filepath" + "strings" + "testing" + + "github.com/meshcore-analyzer/dbschema" +) + +func TestResumeRefusesDifferentDestination(t *testing.T) { + o := importOptions{Source: telemetrySource(t), DatabaseURL: postgresSchema(t), StateDir: t.TempDir(), Kind: "telemetry", BatchSize: 1, afterBatch: func() error { return context.Canceled }} + if _, err := importSQLite(context.Background(), o); !errors.Is(err, context.Canceled) { + t.Fatal(err) + } + o.Resume = true + o.afterBatch = nil + o.DatabaseURL = postgresSchema(t) + if _, err := importSQLite(context.Background(), o); err == nil { + t.Fatal("resume accepted a different PostgreSQL destination") + } +} + +func TestResumeAndFinalizationRejectTargetCorruption(t *testing.T) { + for _, partial := range []bool{false, true} { + t.Run(map[bool]string{false: "finalization", true: "resume"}[partial], func(t *testing.T) { + o := importOptions{Source: telemetrySource(t), DatabaseURL: postgresSchema(t), StateDir: t.TempDir(), Kind: "telemetry", BatchSize: 1} + if partial { + o.afterBatch = func() error { return context.Canceled } + } + _, err := importSQLite(context.Background(), o) + if partial { + if !errors.Is(err, context.Canceled) { + t.Fatal(err) + } + } else if err != nil { + t.Fatal(err) + } + db := openImportDB(t, o.DatabaseURL) + if _, err := db.Exec(`UPDATE observers SET name='tampered'`); err != nil { + t.Fatal(err) + } + if partial { + o.Resume = true + o.afterBatch = nil + _, err = importSQLite(context.Background(), o) + } else { + _, err = finalizeImport(context.Background(), o.DatabaseURL, o.Kind) + } + if err == nil { + t.Fatal("corrupted target was accepted") + } + if err := dbschema.AssertReady(db); err == nil { + t.Fatal("corrupted target became runtime-ready") + } + }) + } +} + +func TestImportPreservesRawCaseJSONAndFractionalScore(t *testing.T) { + source := telemetrySource(t) + db, err := sql.Open("sqlite3", source) + if err != nil { + t.Fatal(err) + } + for _, stmt := range []string{`INSERT INTO nodes(public_key,name,advert_count) VALUES('AbCd','MiXeD',123)`, `UPDATE observations SET path_json='{malformed',score=0.75 WHERE id=90`, `UPDATE transmissions SET channel_hash='public',payload_type=5 WHERE id=80`} { + if _, err := db.Exec(stmt); err != nil { + db.Close() + t.Fatal(err) + } + } + db.Close() + dsn := postgresSchema(t) + if _, err := importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: dsn, StateDir: t.TempDir(), Kind: "telemetry"}); err != nil { + t.Fatal(err) + } + target := openImportDB(t, dsn) + var key, jsonText, channel string + var count int + var score float64 + if err := target.QueryRow(`SELECT public_key,advert_count FROM nodes`).Scan(&key, &count); err != nil || key != "AbCd" || count != 123 { + t.Fatal("raw key/counter changed", err) + } + if err := target.QueryRow(`SELECT path_json,score FROM observations WHERE id=90`).Scan(&jsonText, &score); err != nil || jsonText != "{malformed" || score != 0.75 { + t.Fatal("raw JSON or fractional score changed", err) + } + if err := target.QueryRow(`SELECT channel_hash FROM transmissions WHERE id=80`).Scan(&channel); err != nil || channel != "public" { + t.Fatal("raw channel hash changed", err) + } +} + +func TestImportRejectsUnexpectedViewsTriggersAndFutureAccounts(t *testing.T) { + for _, statement := range []string{`CREATE VIEW unexpected AS SELECT * FROM users`, `CREATE TRIGGER surprise AFTER INSERT ON users BEGIN DELETE FROM tokens; END`, `UPDATE schema_version SET version=999`} { + source := accountSource(t, 5) + db, err := sql.Open("sqlite3", source) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(statement); err != nil { + t.Fatal(err) + } + db.Close() + dsn := postgresSchema(t) + _, err = importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: dsn, StateDir: t.TempDir(), Kind: "accounts"}) + if err == nil { + t.Fatal("unsupported source accepted") + } + var n int + if err := openImportDB(t, dsn).QueryRow(`SELECT count(*) FROM information_schema.tables WHERE table_schema=current_schema()`).Scan(&n); err != nil || n != 0 { + t.Fatal("unsupported source changed target", err) + } + } +} + +func TestAccountsReseedDeletedHistoricalReferences(t *testing.T) { + source := accountSource(t, 5) + db, err := sql.Open("sqlite3", source) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`INSERT INTO audit_log(at,actor_user_id,target_user_id,action) VALUES(1,700000,900000,'deleted'); UPDATE users SET activated_by=1000000`); err != nil { + t.Fatal(err) + } + db.Close() + dsn := postgresSchema(t) + if _, err := importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: dsn, StateDir: t.TempDir(), Kind: "accounts"}); err != nil { + t.Fatal(err) + } + var id int64 + if err := openImportDB(t, dsn).QueryRow(`INSERT INTO users(email,display_name,password_hash,created_at) VALUES('next@example.invalid','next','hash',1) RETURNING id`).Scan(&id); err != nil || id <= 1000000 { + t.Fatalf("historical account identity reused: %d,%v", id, err) + } +} + +func TestSourceFingerprintCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := sourceFingerprintContext(ctx, telemetrySource(t)) + if !errors.Is(err, context.Canceled) { + t.Fatalf("canceled source hashing: %v", err) + } +} + +func TestImportRefusesLegacyV2WithUpgradeGuidance(t *testing.T) { + source := telemetrySource(t) + db, err := sql.Open("sqlite3", source) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`ALTER TABLE observations RENAME COLUMN observer_idx TO observer_id`); err != nil { + t.Fatal(err) + } + db.Close() + _, err = importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: postgresSchema(t), StateDir: t.TempDir(), Kind: "telemetry"}) + if err == nil || !strings.Contains(err.Error(), "upgrade") { + t.Fatal("v2 refusal lacks explicit upgrade guidance") + } +} + +func TestAccountVersionMatrixAndMissingDeclaredTable(t *testing.T) { + for version := 0; version <= 5; version++ { + t.Run(fmt.Sprint(version), func(t *testing.T) { + var source string + if version > 0 { + source = accountSource(t, version) + } else { + source = filepath.Join(t.TempDir(), "empty.db") + db, err := sql.Open("sqlite3", source) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`CREATE TABLE schema_version(version INTEGER NOT NULL);INSERT INTO schema_version VALUES(0)`); err != nil { + t.Fatal(err) + } + db.Close() + } + report, err := importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: postgresSchema(t), StateDir: t.TempDir(), Kind: "accounts"}) + if err != nil || !report.Verified { + t.Fatalf("account version %d: %v", version, err) + } + }) + } + t.Run("missing-table", func(t *testing.T) { + source := accountSource(t, 5) + db, err := sql.Open("sqlite3", source) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`DROP TABLE mail_events`); err != nil { + t.Fatal(err) + } + db.Close() + if _, err := importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: postgresSchema(t), StateDir: t.TempDir(), Kind: "accounts"}); err == nil { + t.Fatal("incomplete declared account schema was accepted") + } + }) +} + +func TestNormalizeDoesNotRewriteObservationRows(t *testing.T) { + raw := telemetrySource(t) + db, err := sql.Open("sqlite3", raw) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<1000) + INSERT INTO observations(id,transmission_id,observer_idx,timestamp) SELECT x+1000,80,NULL,x FROM n`); err != nil { + t.Fatal(err) + } + db.Close() + working := filepath.Join(t.TempDir(), "working.sqlite") + if err := snapshotSource(context.Background(), raw, working); err != nil { + t.Fatal(err) + } + db, err = openSQLite(working, "rw") + if err != nil { + t.Fatal(err) + } + defer db.Close() + db.SetMaxOpenConns(1) + tables, err := layouts("telemetry") + if err != nil { + t.Fatal(err) + } + var before, after int64 + if err := db.QueryRow(`SELECT total_changes()`).Scan(&before); err != nil { + t.Fatal(err) + } + if err := normalizeSource(db, "telemetry", raw, tables); err != nil { + t.Fatal(err) + } + if err := db.QueryRow(`SELECT total_changes()`).Scan(&after); err != nil { + t.Fatal(err) + } + if after-before >= 50 { + t.Fatalf("normalization rewrote unchanged bulk rows: %d changes for 1003 observations", after-before) + } + t.Logf("normalization wrote %d rows for 1003 unchanged observations", after-before) +} + +func TestImportPopulatesPlannerStatsBeforeReadiness(t *testing.T) { + dsn := postgresSchema(t) + if _, err := importSQLite(context.Background(), importOptions{Source: telemetrySource(t), DatabaseURL: dsn, StateDir: t.TempDir(), Kind: "telemetry"}); err != nil { + t.Fatal(err) + } + db := openImportDB(t, dsn) + var estimate float64 + if err := db.QueryRow(`SELECT reltuples FROM pg_class WHERE oid=to_regclass('observations')`).Scan(&estimate); err != nil { + t.Fatal(err) + } + if estimate != 3 { + t.Fatalf("planner estimate before readiness=%v; want 3 imported observations", estimate) + } + if err := dbschema.AssertReady(db); err == nil { + t.Fatal("statistics preparation opened readiness before finalization") + } +} + +func TestReservedNameDoesNotHideSourceTrigger(t *testing.T) { + source := accountSource(t, 5) + db, err := sql.Open("sqlite3", source) + if err != nil { + t.Fatal(err) + } + for _, stmt := range []string{ + `CREATE TRIGGER hidden_source AFTER INSERT ON users BEGIN DELETE FROM tokens; END`, + `PRAGMA writable_schema=ON`, + `UPDATE sqlite_master SET name='sqlite_hidden_source',sql=replace(sql,'hidden_source','sqlite_hidden_source') WHERE type='trigger' AND name='hidden_source'`, + `PRAGMA writable_schema=OFF`, + } { + if _, err := db.Exec(stmt); err != nil { + db.Close() + t.Fatal(err) + } + } + db.Close() + db, err = openSQLite(source, "ro") + if err != nil { + t.Fatal(err) + } + var count int + if err := db.QueryRow(`SELECT count(*) FROM sqlite_master WHERE type='trigger' AND name='sqlite_hidden_source'`).Scan(&count); err != nil || count != 1 { + db.Close() + t.Fatalf("tampered schema fixture: count=%d err=%v", count, err) + } + db.Close() + if _, err := importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: postgresSchema(t), StateDir: t.TempDir(), Kind: "accounts"}); err == nil { + t.Fatal("reserved-name trigger bypassed source validation") + } +} diff --git a/cmd/migrate/import_test.go b/cmd/migrate/import_test.go new file mode 100644 index 000000000..01b6f9bad --- /dev/null +++ b/cmd/migrate/import_test.go @@ -0,0 +1,255 @@ +package main + +import ( + "context" + "database/sql" + "errors" + "path/filepath" + "testing" + + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/dbschema/legacy" + "github.com/meshcore-analyzer/pgutil" + "github.com/meshcore-analyzer/users" +) + +func telemetrySource(t *testing.T) string { + t.Helper() + path := filepath.Join(t.TempDir(), "source.db") + db, err := sql.Open("sqlite3", path) + if err != nil { + t.Fatal(err) + } + defer db.Close() + if err := legacy.ApplyBase(db); err != nil { + t.Fatal(err) + } + if err := legacy.Apply(db, t.Logf); err != nil { + t.Fatal(err) + } + for _, query := range []string{ + `INSERT INTO observers(rowid,id,name) VALUES(0,'observer-zero',NULL),(50,'observer-gap','')`, + `INSERT INTO transmissions(id,raw_hex,hash,first_seen,last_seen) VALUES(0,'00','zero','2026-01-01 00:00:00',0),(80,'00','gap','2026-01-02T00:00:00Z',1),(99999,'00','deleted','2026-01-01',0)`, + `DELETE FROM transmissions WHERE id=99999`, + `INSERT INTO observations(id,transmission_id,observer_idx,path_json,timestamp) VALUES(0,0,0,NULL,0),(90,80,999,'',1),(91,80,NULL,'[]',2)`, + } { + if _, err := db.Exec(query); err != nil { + t.Fatal(err) + } + } + return path +} + +func openImportDB(t *testing.T, dsn string) *sql.DB { + t.Helper() + db, err := pgutil.Open(dsn, false) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { db.Close() }) + return db +} + +func TestImportTelemetryPreservesIDsNullsAndHighWater(t *testing.T) { + source := telemetrySource(t) + before, err := sourceFingerprint(source) + if err != nil { + t.Fatal(err) + } + dsn := postgresSchema(t) + report, err := importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: dsn, StateDir: t.TempDir(), Kind: "telemetry", BatchSize: 2}) + if err != nil { + t.Fatalf("import valid telemetry: %v", err) + } + if !report.Verified || len(report.Tables) != len(dbschema.Tables) { + t.Fatalf("missing verification: %+v", report) + } + db := openImportDB(t, dsn) + if err := dbschema.AssertReady(db); err == nil { + t.Fatal("import accepted before explicit cutover finalization") + } + if _, err := finalizeImport(context.Background(), dsn, "telemetry"); err != nil { + t.Fatal(err) + } + if err := dbschema.AssertReady(db); err != nil { + t.Fatal(err) + } + var n int + if err := db.QueryRow(`SELECT count(*) FROM observations WHERE (id=0 AND observer_idx=0 AND path_json IS NULL) OR (id=90 AND observer_idx=999 AND path_json='') OR (id=91 AND observer_idx IS NULL)`).Scan(&n); err != nil { + t.Fatal(err) + } + if n != 3 { + t.Fatalf("lost observer identity/null/orphan links: %d", n) + } + var id int64 + if err := db.QueryRow(`INSERT INTO observers(id) VALUES('next-observer') RETURNING rowid`).Scan(&id); err != nil { + t.Fatal(err) + } + if id <= 999 { + t.Fatalf("observer identity reused historical reference: %d", id) + } + if err := db.QueryRow(`INSERT INTO transmissions(raw_hex,hash,first_seen) VALUES('00','next','2026-01-03') RETURNING id`).Scan(&id); err != nil { + t.Fatal(err) + } + if id <= 99999 { + t.Fatalf("transmission identity ignored deleted high water: %d", id) + } + after, err := sourceFingerprint(source) + if err != nil { + t.Fatal(err) + } + if before != after { + t.Fatal("migration changed original SQLite source") + } +} + +func TestImportInterruptResumeAndWrongSource(t *testing.T) { + source := telemetrySource(t) + dsn, state := postgresSchema(t), t.TempDir() + options := importOptions{Source: source, DatabaseURL: dsn, StateDir: state, Kind: "telemetry", BatchSize: 1, afterBatch: func() error { return context.Canceled }} + if _, err := importSQLite(context.Background(), options); !errors.Is(err, context.Canceled) { + t.Fatalf("expected cancellation after committed batch, got %v", err) + } + db := openImportDB(t, dsn) + if err := dbschema.AssertReady(db); err == nil { + t.Fatal("partially imported database accepted") + } + options.afterBatch = nil + if _, err := importSQLite(context.Background(), options); err == nil { + t.Fatal("partial import resumed without explicit resume") + } + options.Resume = true + options.Source = telemetrySource(t) + changed, err := sql.Open("sqlite3", options.Source) + if err != nil { + t.Fatal(err) + } + if _, err := changed.Exec(`UPDATE transmissions SET raw_hex='ff' WHERE id=0`); err != nil { + t.Fatal(err) + } + changed.Close() + if _, err := importSQLite(context.Background(), options); err == nil { + t.Fatal("resumed with a different source") + } + options.Source = source + report, err := importSQLite(context.Background(), options) + if err != nil { + t.Fatal(err) + } + if !report.Verified { + t.Fatal("resumed import unverified") + } + var n int + if err := db.QueryRow(`SELECT COUNT(*) FROM observations`).Scan(&n); err != nil { + t.Fatal(err) + } + if n != 3 { + t.Fatalf("resume duplicated/lost observations: %d", n) + } + if _, err := importSQLite(context.Background(), options); err != nil { + t.Fatalf("verified retry must be idempotent: %v", err) + } +} + +func TestImportRefusesUnknownSourceAndOccupiedTarget(t *testing.T) { + for _, unknown := range []bool{false, true} { + source, dsn := telemetrySource(t), postgresSchema(t) + if unknown { + db, err := sql.Open("sqlite3", source) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`ALTER TABLE transmissions ADD COLUMN future_secret TEXT`); err != nil { + t.Fatal(err) + } + db.Close() + } else { + db := openImportDB(t, dsn) + if _, err := db.Exec(`CREATE TABLE keep_me(value TEXT); INSERT INTO keep_me VALUES('retained')`); err != nil { + t.Fatal(err) + } + } + if _, err := importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: dsn, StateDir: t.TempDir(), Kind: "telemetry"}); err == nil { + t.Fatal("unsupported source/occupied destination accepted") + } + if !unknown { + var value string + if err := openImportDB(t, dsn).QueryRow(`SELECT value FROM keep_me`).Scan(&value); err != nil { + t.Fatal(err) + } + if value != "retained" { + t.Fatal("existing target changed") + } + } + } +} + +func accountSource(t *testing.T, version int) string { + t.Helper() + path := filepath.Join(t.TempDir(), "users.db") + db, err := sql.Open("sqlite3", path) + if err != nil { + t.Fatal(err) + } + defer db.Close() + if _, err := db.Exec(`CREATE TABLE schema_version(version INTEGER NOT NULL)`); err != nil { + t.Fatal(err) + } + for _, migration := range users.LegacyMigrations()[:version] { + for _, stmt := range migration { + if _, err := db.Exec(stmt); err != nil { + t.Fatal(err) + } + } + } + if _, err := db.Exec(`INSERT INTO schema_version VALUES(?)`, version); err != nil { + t.Fatal(err) + } + for _, stmt := range []string{ + `INSERT INTO users(id,email,display_name,password_hash,role,status,created_at) VALUES(0,'user@example.invalid','test','preserved-hash','user','active',1)`, + `INSERT INTO sessions(id,token_hash,user_id,csrf_token,created_at,expires_at,last_seen_at) VALUES(90,'preserved-token',0,'preserved-csrf',1,9999999999,2)`, + `INSERT INTO tokens(token_hash,user_id,purpose,expires_at,used_at) VALUES('one-use',0,'reset',9999999999,NULL)`, + `INSERT INTO mail_log(id,to_email,purpose,sent_at,last_event_at) VALUES(4,'user@example.invalid','activate',1,1)`, + `INSERT INTO mail_events(rowid,mail_id,event,at) VALUES(900,4,'delivered',1)`, + } { + if _, err := db.Exec(stmt); err != nil { + t.Fatal(err) + } + } + return path +} + +func TestImportLegacyAccountsPreservesAuthenticationAndEventOrder(t *testing.T) { + source, dsn := accountSource(t, 1), postgresSchema(t) + report, err := importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: dsn, StateDir: t.TempDir(), Kind: "accounts", BatchSize: 1}) + if err != nil { + t.Fatal(err) + } + if !report.Verified { + t.Fatal("accounts not verified") + } + db := openImportDB(t, dsn) + if err := users.AssertReady(db); err == nil { + t.Fatal("accounts accepted before finalization") + } + if _, err := finalizeImport(context.Background(), dsn, "accounts"); err != nil { + t.Fatal(err) + } + if err := users.AssertReady(db); err != nil { + t.Fatal(err) + } + var value string + if err := db.QueryRow(`SELECT password_hash FROM users WHERE id=0`).Scan(&value); err != nil || value != "preserved-hash" { + t.Fatalf("password hash changed: %v", err) + } + if err := db.QueryRow(`SELECT token_hash FROM sessions WHERE id=90 AND user_id=0`).Scan(&value); err != nil || value != "preserved-token" { + t.Fatalf("session changed: %v", err) + } + var n int + if err := db.QueryRow(`SELECT COUNT(*) FROM tokens WHERE token_hash='one-use' AND used_at IS NULL`).Scan(&n); err != nil || n != 1 { + t.Fatalf("one-time token changed: %v", err) + } + if err := db.QueryRow(`SELECT id FROM mail_events WHERE mail_id=4`).Scan(&n); err != nil || n != 900 { + t.Fatalf("mail event identity changed: %d %v", n, err) + } +} diff --git a/cmd/migrate/key_backfill_roundtrip_test.go b/cmd/migrate/key_backfill_roundtrip_test.go new file mode 100644 index 000000000..47be14160 --- /dev/null +++ b/cmd/migrate/key_backfill_roundtrip_test.go @@ -0,0 +1,60 @@ +package main + +import ( + "context" + "path/filepath" + "testing" +) + +func TestKeyBackfillStateAndRewriteFeedRoundTrip(t *testing.T) { + source := telemetrySource(t) + sqlite, err := openSQLite(source, "rw") + if err != nil { + t.Fatal(err) + } + for _, statement := range []string{ + `INSERT INTO tx_rewrite_feed(id,tx_id,created_at) VALUES(700,80,123)`, + `INSERT INTO key_backfill_state(kind,run_fp,cursor,upper,rewritten,started_at) VALUES('channel','opaque fingerprint',50,80,3,124)`, + `INSERT INTO key_backfill_done(kind,key_fp,done_at) VALUES('region','key fingerprint',125)`, + `INSERT INTO tx_rewrite_feed(id,tx_id,created_at) VALUES(900,80,126)`, + `DELETE FROM tx_rewrite_feed WHERE id=900`, + } { + if _, err := sqlite.Exec(statement); err != nil { + sqlite.Close() + t.Fatal(err) + } + } + sqlite.Close() + ctx := context.Background() + dsn := postgresDatabase(t) + if _, err := importSQLite(ctx, importOptions{Source: source, DatabaseURL: dsn, StateDir: t.TempDir(), Kind: "telemetry", BatchSize: 1}); err != nil { + t.Fatal(err) + } + if _, err := finalizeImport(ctx, dsn, "telemetry"); err != nil { + t.Fatal(err) + } + target := filepath.Join(t.TempDir(), "roundtrip.sqlite") + report, err := reversePostgres(ctx, reverseOptions{DatabaseURL: dsn, Destination: target, StateDir: t.TempDir(), Kind: "telemetry", BatchSize: 1}) + if err != nil || !report.Verified { + t.Fatal("reverse verification", err) + } + sqlite, err = openSQLite(target, "rw") + if err != nil { + t.Fatal(err) + } + defer sqlite.Close() + var count int + for _, query := range []string{ + `SELECT count(*) FROM tx_rewrite_feed WHERE id=700 AND tx_id=80 AND created_at=123`, + `SELECT count(*) FROM key_backfill_state WHERE kind='channel' AND run_fp='opaque fingerprint' AND cursor=50 AND upper=80 AND rewritten=3 AND started_at=124`, + `SELECT count(*) FROM key_backfill_done WHERE kind='region' AND key_fp='key fingerprint' AND done_at=125`, + } { + if err := sqlite.QueryRow(query).Scan(&count); err != nil || count != 1 { + t.Fatal("backfill state changed", err) + } + } + var id int64 + if err := sqlite.QueryRow(`INSERT INTO tx_rewrite_feed(tx_id,created_at) VALUES(80,127) RETURNING id`).Scan(&id); err != nil || id <= 900 { + t.Fatalf("feed identity reused deleted high water: %d,%v", id, err) + } +} diff --git a/cmd/migrate/layout.go b/cmd/migrate/layout.go new file mode 100644 index 000000000..43874e298 --- /dev/null +++ b/cmd/migrate/layout.go @@ -0,0 +1,300 @@ +package main + +import ( + "database/sql" + "errors" + "fmt" + "strings" + + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/dbschema/legacy" + "github.com/meshcore-analyzer/users" +) + +type importColumn struct { + Name string + Type string + Identity bool +} +type importTable struct { + Name string + Columns []importColumn +} + +func quote(name string) string { return `"` + strings.ReplaceAll(name, `"`, `""`) + `"` } + +func layouts(kind string) ([]importTable, error) { + var out []importTable + if kind == "telemetry" { + for _, table := range dbschema.Tables { + t := importTable{Name: table.Name} + for _, col := range table.Columns { + t.Columns = append(t.Columns, importColumn{col.Name, col.Type, col.Identity}) + } + out = append(out, t) + } + return out, nil + } + if kind != "accounts" { + return nil, errors.New("import kind must be telemetry or accounts") + } + // SQLite is used only by this offline importer. Derive the legacy account + // layout from the immutable migration metadata instead of duplicating it. + db, err := sql.Open("sqlite3", ":memory:") + if err != nil { + return nil, err + } + defer db.Close() + db.SetMaxOpenConns(1) + for _, migration := range users.LegacyMigrations() { + for _, stmt := range migration { + if _, err := db.Exec(stmt); err != nil { + return nil, err + } + } + } + identities := map[string]bool{"users": true, "sessions": true, "audit_log": true, "mail_log": true, "proposals": true, "mail_events": true} + for _, name := range users.Tables() { + t := importTable{Name: name} + if name == "mail_events" { + t.Columns = append(t.Columns, importColumn{"id", "BIGINT", true}) + } + cols, err := sqliteColumns(db, name) + if err != nil { + return nil, err + } + for _, col := range cols { + kind := "TEXT" + if strings.Contains(strings.ToUpper(col.Type), "INT") { + kind = "BIGINT" + } + t.Columns = append(t.Columns, importColumn{col.Name, kind, col.Name == "id" && identities[name]}) + } + out = append(out, t) + } + return out, nil +} + +func sqliteColumns(db *sql.DB, table string) ([]importColumn, error) { + // table_info omits generated and hidden columns, which must be refused even + // when their names match an ordinary column in a supported legacy layout. + rows, err := db.Query(`PRAGMA table_xinfo(` + quote(table) + `)`) + if err != nil { + return nil, err + } + defer rows.Close() + var out []importColumn + for rows.Next() { + var cid, notnull, pk, hidden int + var name, kind string + var def sql.NullString + if err := rows.Scan(&cid, &name, &kind, ¬null, &def, &pk, &hidden); err != nil { + return nil, err + } + if hidden != 0 { + return nil, fmt.Errorf("unsupported generated or hidden SQLite column %s.%s", table, name) + } + out = append(out, importColumn{Name: name, Type: kind}) + } + return out, rows.Err() +} + +func validateSource(db *sql.DB, kind string, tables []importTable) error { + derived := map[string]bool{} + if kind == "telemetry" { + objects, err := knownSQLiteHopIndex(db) + if err != nil { + return err + } + for _, object := range objects { + derived[object.Name] = true + } + } + var accountVersion int + if kind == "accounts" { + var n int + if err := db.QueryRow(`SELECT COUNT(*),COALESCE(MAX(version),0) FROM schema_version`).Scan(&n, &accountVersion); err != nil { + return errors.New("account source must contain its supported schema_version") + } + if n != 1 || accountVersion < 0 || accountVersion > users.SQLiteSchemaVersion { + return fmt.Errorf("unsupported account schema version %d (supported 0 through %d)", accountVersion, users.SQLiteSchemaVersion) + } + } + allowed := map[string]map[string]bool{} + for _, table := range tables { + cols := map[string]bool{} + for _, col := range table.Columns { + cols[col.Name] = true + } + allowed[table.Name] = cols + } + if kind == "accounts" { + allowed["schema_version"] = map[string]bool{"version": true} + } + rows, err := db.Query(`SELECT name,type,COALESCE(sql,'') FROM sqlite_master WHERE type IN ('table','view','trigger')`) + if err != nil { + return err + } + type object struct{ name, kind, sql string } + internalTables := map[string]bool{"sqlite_sequence": true, "sqlite_stat1": true, "sqlite_stat2": true, "sqlite_stat3": true, "sqlite_stat4": true} + var objects []object + for rows.Next() { + var o object + if err := rows.Scan(&o.name, &o.kind, &o.sql); err != nil { + rows.Close() + return err + } + ordinary := strings.HasPrefix(strings.ToUpper(strings.Join(strings.Fields(o.sql), " ")), "CREATE TABLE ") + if o.kind == "table" && internalTables[o.name] && ordinary { + continue + } + if derived[o.name] { + continue // Complete definitions were checked above, not names alone. + } + objects = append(objects, o) + if len(objects) > len(allowed)+1 { + rows.Close() + return errors.New("source has unexpected schema objects") + } + } + if err := rows.Err(); err != nil { + rows.Close() + return err + } + rows.Close() + for _, o := range objects { + if o.kind == "table" && !strings.HasPrefix(strings.ToUpper(strings.Join(strings.Fields(o.sql), " ")), "CREATE TABLE ") { + return fmt.Errorf("unsupported nonordinary source table %q", o.name) + } + if o.kind == "view" && kind == "telemetry" && o.name == "packets_v" { + continue + } + cols, known := allowed[o.name] + if o.kind != "table" || !known { + return fmt.Errorf("unsupported SQLite %s %q; no source data was discarded", o.kind, o.name) + } + have, err := sqliteColumns(db, o.name) + if err != nil { + return err + } + for _, col := range have { + if !cols[col.Name] { + if kind == "telemetry" && o.name == "observations" && col.Name == "observer_id" { + return errors.New("legacy v2 observer_id layout requires an offline upgrade with the last SQLite release before importing") + } + return fmt.Errorf("unsupported SQLite column %s.%s", o.name, col.Name) + } + } + } + if kind == "telemetry" { + return legacy.CheckLegacySource(db) + } + if err := users.ValidateSQLiteSourceLayout(db); err != nil { + return err + } + var integrity string + if err := db.QueryRow(`PRAGMA quick_check`).Scan(&integrity); err != nil { + return err + } + if integrity != "ok" { + return errors.New("account source failed SQLite integrity check") + } + return nil +} + +func normalizeSource(db *sql.DB, kind, rawPath string, tables []importTable) error { + if err := validateSource(db, kind, tables); err != nil { + return err + } + if kind == "accounts" { + // Only the private snapshot is normalized; exact known old/native v6 + // layouts upgrade through the same lossless writer migration as runtime. + return users.ApplySQLite(db) + } + + // This is the private normalized snapshot. Keep every source/recovery + // object intact, but remove the validated derived cache before canonical + // normalization so its SQLite JSON triggers cannot execute during import. + if _, err := dbschema.DropObservationHopsSchema(db); err != nil { + return err + } + if err := legacy.Normalize(db, nil); err != nil { + return err + } + // Mutation inventory of legacy.Normalize: only these columns can change + // existing rows. Duplicate/invalid-row deletion paths were refused above. + // Restore changed tuples by rowid, without rewriting the observation corpus. + recoveryURI, err := sqliteURL(rawPath, "ro") + if err != nil { + return err + } + if _, err := db.Exec(`ATTACH DATABASE ? AS recovery`, recoveryURI); err != nil { + return err + } + defer db.Exec(`DETACH DATABASE recovery`) + mutated := map[string]map[string]bool{ + "nodes": {"public_key": true, "advert_count": true}, + "observers": {"noise_floor": true, "last_packet_at": true}, + "transmissions": {"channel_hash": true}, + } + for _, table := range tables { + selected, changes := mutated[table.Name] + if !changes { + continue + } + var exists int + if err := db.QueryRow(`SELECT count(*) FROM recovery.sqlite_master WHERE type='table' AND name=?`, table.Name).Scan(&exists); err != nil { + return err + } + if exists == 0 { + continue + } + rows, err := db.Query(`PRAGMA recovery.table_info(` + quote(table.Name) + `)`) + if err != nil { + return err + } + var columns []string + for rows.Next() { + var cid, nn, pk int + var name, kind string + var def sql.NullString + if err := rows.Scan(&cid, &name, &kind, &nn, &def, &pk); err != nil { + rows.Close() + return err + } + if selected[name] { + columns = append(columns, name) + } + } + if err := rows.Err(); err != nil { + rows.Close() + return err + } + rows.Close() + var sourceCount, targetCount int64 + if err := db.QueryRow(`SELECT count(*) FROM recovery.` + quote(table.Name)).Scan(&sourceCount); err != nil { + return err + } + if err := db.QueryRow(`SELECT count(*) FROM main.` + quote(table.Name)).Scan(&targetCount); err != nil { + return err + } + if sourceCount != targetCount { + return fmt.Errorf("normalization changed row count in %s; source requires an explicit repair", table.Name) + } + if len(columns) == 0 { + continue + } + var targets, originals, differences []string + for _, name := range columns { + targets = append(targets, quote(name)) + originals = append(originals, "old."+quote(name)) + differences = append(differences, "current."+quote(name)+" IS NOT old."+quote(name)) + } + restore := `UPDATE main.` + quote(table.Name) + ` AS current SET (` + strings.Join(targets, ",") + `)=(SELECT ` + strings.Join(originals, ",") + ` FROM recovery.` + quote(table.Name) + ` old WHERE old.rowid=current.rowid) + WHERE EXISTS(SELECT 1 FROM recovery.` + quote(table.Name) + ` old WHERE old.rowid=current.rowid AND (` + strings.Join(differences, " OR ") + `))` + if _, err := db.Exec(restore); err != nil { + return fmt.Errorf("preserve original %s values: %w", table.Name, err) + } + } + return nil +} diff --git a/cmd/migrate/main.go b/cmd/migrate/main.go index d6f7eae09..a87b20f4a 100644 --- a/cmd/migrate/main.go +++ b/cmd/migrate/main.go @@ -1,58 +1,257 @@ -// Command migrate runs all dbschema migrations against a SQLite -// CoreScope database and exits. Used by CI / one-shot tooling to bring -// an unmigrated fixture (or a fresh DB) up to the schema shape the -// read-only server (cmd/server) requires via dbschema.AssertReady. -// -// In production the ingestor (cmd/ingestor) runs dbschema.Apply at -// startup before subscribing to MQTT — this binary exists so CI's E2E -// job can migrate the e2e-fixture.db without booting the full ingestor -// (which needs MQTT brokers). -// -// Usage: -// -// migrate -db path/to/file.db +// Command migrate bootstraps PostgreSQL or imports immutable offline SQLite +// recovery snapshots. Runtime services never apply DDL themselves. package main import ( - "database/sql" + "context" + "encoding/json" + "errors" "flag" - "log" + "fmt" + "io" + "os" + "os/signal" + "syscall" - _ "github.com/mattn/go-sqlite3" "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/pgutil" + "github.com/meshcore-analyzer/users" ) func main() { - dbPath := flag.String("db", "", "path to SQLite database to migrate (required)") - flag.Parse() - - if *dbPath == "" { - log.Fatalf("[migrate] -db is required") + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer cancel() + if err := runCommand(ctx, os.Args[1:], os.Stdout); err != nil { + fmt.Fprintln(os.Stderr, "[migrate]", err) + os.Exit(1) } +} - log.SetFlags(log.LstdFlags | log.Lmsgprefix) - log.SetPrefix("[migrate] ") +func runCommand(ctx context.Context, args []string, out io.Writer) error { + flags := flag.NewFlagSet("migrate", flag.ContinueOnError) + flags.SetOutput(io.Discard) + databaseURL := flags.String("database-url", "", "PostgreSQL telemetry owner URL (prefer CORESCOPE_DATABASE_URL)") + usersURL := flags.String("users-database-url", "", "PostgreSQL accounts owner URL (prefer CORESCOPE_USERS_DATABASE_URL)") + source := flags.String("from-sqlite", "", "offline telemetry SQLite source") + accountSource := flags.String("users-from-sqlite", "", "offline account SQLite source") + stateDir := flags.String("state-dir", "", "private migration snapshots and resume manifests (storage actions default beside selection; legacy imports use state/migration)") + offline := flags.Bool("offline", false, "confirm telemetry and account writers are stopped") + resume := flags.Bool("resume", false, "resume the same source/state/destination import") + checkReady := flags.Bool("check-ready", false, "check readiness without changing any schema or data") + checkImportKind := flags.String("check-import-kind", "", "read-only completed-import guard: telemetry or accounts, with its SQLite source path") + action := flags.String("storage-action", "", "installation status, setup, upgrade, adopt, init, switch, resume or abort") + selectionFile := flags.String("selection-file", "", "absolute stable storage-selection.json path") + backend := flags.String("backend", "", "bootstrap backend, or explicit switch destination: sqlite or postgres") + sqlitePath := flags.String("sqlite-path", "", "bootstrap or new destination telemetry SQLite path") + usersSQLitePath := flags.String("users-sqlite-path", "", "bootstrap or new destination account SQLite path") + jobID := flags.String("job-id", "", "exact pending storage switch job to resume or abort") + configFile := flags.String("config", "", "explicit configuration file for storage setup") + configDir := flags.String("config-dir", ".", "configuration discovery directory; relative data paths still use the working directory") + if err := flags.Parse(args); err != nil { + if errors.Is(err, flag.ErrHelp) { + flags.SetOutput(out) + flags.PrintDefaults() + return nil + } + return errors.New("invalid migration arguments; use -help") + } + if flags.NArg() != 0 { + return errors.New("unexpected positional migration argument; use -help") + } + if *action != "" { + if *source != "" || *accountSource != "" || *resume || *checkReady || *checkImportKind != "" { + return errors.New("storage actions cannot be combined with legacy import flags") + } + return runStorageAction(ctx, storageOptions{Action: *action, SelectionFile: *selectionFile, Backend: *backend, SQLitePath: *sqlitePath, UsersSQLitePath: *usersSQLitePath, OwnerURL: *databaseURL, UsersOwnerURL: *usersURL, StateDir: *stateDir, JobID: *jobID, Offline: *offline, ConfigFile: *configFile, ConfigDir: *configDir}, out) + } + if *stateDir == "" { + *stateDir = "state/migration" + } + if *databaseURL == "" { + *databaseURL = os.Getenv("CORESCOPE_DATABASE_URL") + } + if *usersURL == "" { + *usersURL = os.Getenv("CORESCOPE_USERS_DATABASE_URL") + } + if *databaseURL == "" && *usersURL == "" { + return errors.New("provide CORESCOPE_DATABASE_URL and/or CORESCOPE_USERS_DATABASE_URL") + } + if *databaseURL != "" && *usersURL != "" { + same, err := pgutil.SameDatabase(ctx, *databaseURL, *usersURL) + if err != nil { + return err + } + if same { + return errors.New("telemetry and accounts require separate PostgreSQL databases") + } + } + type store struct { + kind, dsn, source string + } + stores := []store{{kind: "telemetry", dsn: *databaseURL, source: *source}, {kind: "accounts", dsn: *usersURL, source: *accountSource}} + if *checkImportKind != "" { + if *checkReady || *resume { + return errors.New("-check-import-kind cannot be combined with -check-ready or -resume") + } + for _, s := range stores { + if s.kind == *checkImportKind { + if s.dsn == "" || s.source == "" { + return errors.New("completed-import check requires the matching PostgreSQL URL and SQLite source path") + } + if err := checkImportedSource(ctx, s.dsn, s.kind, s.source); err != nil { + return err + } + fmt.Fprintf(out, "%s verified import matches the retained SQLite source\n", s.kind) + return nil + } + } + return errors.New("-check-import-kind must be telemetry or accounts") + } + importing := *source != "" || *accountSource != "" + if importing && !*offline { + return errors.New("SQLite import requires -offline after stopping all telemetry and account writers") + } + if *checkReady && (importing || *resume) { + return errors.New("-check-ready cannot be combined with import or resume") + } + if *resume && !importing { + return errors.New("-resume requires a SQLite source") + } + var imported []store + for _, s := range stores { + if s.dsn == "" { + if s.source != "" { + return fmt.Errorf("%s source requires its PostgreSQL owner URL", s.kind) + } + continue + } + if *checkReady { + db, err := pgutil.Open(s.dsn, false) + if err != nil { + return err + } + if s.kind == "telemetry" { + err = dbschema.AssertReady(db) + } else { + err = users.AssertReady(db) + } + db.Close() + if err != nil { + return err + } + fmt.Fprintf(out, "%s PostgreSQL schema is ready\n", s.kind) + continue + } + if s.source != "" { + report, err := importSQLite(ctx, importOptions{Source: s.source, DatabaseURL: s.dsn, StateDir: *stateDir, Kind: s.kind, Resume: *resume}) + if err != nil { + return err + } + if !report.Verified { + return errors.New("import returned without complete verification") + } + if err := json.NewEncoder(out).Encode(report); err != nil { + return err + } + imported = append(imported, s) + } + } + if *checkReady { + return nil + } + // Empty/absent optional stores are initialized only after every requested + // SQLite import has verified, so an account failure cannot cut over telemetry. + for _, s := range stores { + if s.dsn == "" || s.source != "" { + continue + } + db, err := pgutil.Open(s.dsn, false) + if err != nil { + return err + } + if s.kind == "telemetry" { + err = dbschema.Apply(db, nil) + } else { + err = users.Apply(db) + } + db.Close() + if err != nil { + return err + } + fmt.Fprintf(out, "%s PostgreSQL schema initialized\n", s.kind) + } + var opened []store + for _, s := range imported { + didOpen, err := finalizeImport(ctx, s.dsn, s.kind) + if err != nil { + // There is no cross-database transaction. Writers stay offline throughout + // cutover; close only gates this invocation actually opened. Another + // finalizer can open a gate after our earlier import verification. + for _, prior := range opened { + db, openErr := pgutil.Open(prior.dsn, false) + if openErr == nil { + _, resetErr := db.Exec(`UPDATE corescope_schema SET ready=false WHERE kind=$1`, prior.kind) + db.Close() + if resetErr != nil { + return errors.Join(err, errors.New("an earlier readiness gate could not be reset; keep services stopped")) + } + } else { + return errors.Join(err, errors.New("an earlier readiness gate could not be reset; keep services stopped")) + } + } + return err + } + if didOpen { + opened = append(opened, s) + } + } + for _, s := range imported { + fmt.Fprintf(out, "%s import verified and finalized; keep the recovery snapshot for rollback\n", s.kind) + } + return nil +} - // Same DSN as the ingestor. A bare path here would mean synchronous=NORMAL - // under mattn, quietly weakening durability for a process that writes — and - // this one runs dbschema.Apply, which now also deletes duplicate rows. - db, err := sql.Open("sqlite3", dbschema.WriterDSN(*dbPath)) +func checkImportedSource(ctx context.Context, databaseURL, kind, source string) error { + db, err := pgutil.Open(databaseURL, true) if err != nil { - log.Fatalf("open %s: %v", *dbPath, err) + return err } defer db.Close() - - if err := db.Ping(); err != nil { - log.Fatalf("ping %s: %v", *dbPath, err) + if kind == "telemetry" { + err = dbschema.AssertReady(db) + } else { + err = users.AssertReady(db) } - - if err := dbschema.Apply(db, log.Printf); err != nil { - log.Fatalf("dbschema.Apply: %v", err) + if err != nil { + return err } - - if err := dbschema.AssertReady(db); err != nil { - log.Fatalf("dbschema.AssertReady after Apply: %v (this is a bug — Apply did not produce a ready schema)", err) + var runID, recordedKind, digest, normalized, reportText string + var verified bool + if err := db.QueryRowContext(ctx, `SELECT run_id,kind,source_digest,normalized_digest,verified,report FROM corescope_import WHERE id=1`).Scan(&runID, &recordedKind, &digest, &normalized, &verified, &reportText); err != nil { + return errors.New("PostgreSQL has no completed SQLite import; run the explicit offline upgrade before starting services") } - - log.Printf("OK: %s is migrated and ready", *dbPath) + var report importReport + if !verified || recordedKind != kind || runID == "" || normalized == "" || json.Unmarshal([]byte(reportText), &report) != nil || !report.Verified || report.Kind != kind { + return errors.New("PostgreSQL import verification marker is incomplete or belongs to another store") + } + tables, err := layouts(kind) + if err != nil { + return err + } + if len(report.Tables) != len(tables) { + return errors.New("PostgreSQL import report is incomplete") + } + for i, table := range tables { + if report.Tables[i].Table != table.Name || report.Tables[i].Rows < 0 || len(report.Tables[i].SHA256) != 64 { + return errors.New("PostgreSQL import report is invalid") + } + } + actual, err := fingerprint(ctx, source) + if err != nil { + return err + } + if actual != digest { + return errors.New("retained SQLite source does not match this PostgreSQL import; keep services stopped") + } + return nil } diff --git a/cmd/migrate/main_test.go b/cmd/migrate/main_test.go index b32b5c9b4..b08121737 100644 --- a/cmd/migrate/main_test.go +++ b/cmd/migrate/main_test.go @@ -6,14 +6,20 @@ package main import ( + "context" "database/sql" + "fmt" "io" "os" + "os/exec" "path/filepath" + "strings" "testing" _ "github.com/mattn/go-sqlite3" - "github.com/meshcore-analyzer/dbschema" + postgresSchemaAPI "github.com/meshcore-analyzer/dbschema" + dbschema "github.com/meshcore-analyzer/dbschema/legacy" + "github.com/meshcore-analyzer/packetpath" ) // fixtureCandidates lists possible locations of the committed e2e @@ -31,10 +37,152 @@ func locateFixture(t *testing.T) string { return abs } } - t.Skipf("e2e fixture not found (looked in: %v)", fixtureCandidates) + t.Fatalf("committed e2e fixture not found (looked in: %v)", fixtureCandidates) return "" } +func TestCommittedFixtureRequiresExplicitDuplicateRepair(t *testing.T) { + source := locateFixture(t) + _, err := importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: postgresSchema(t), StateDir: t.TempDir(), Kind: "telemetry"}) + if err == nil || !strings.Contains(err.Error(), "duplicate observation identity") { + t.Fatalf("raw legacy fixture must require explicit duplicate repair: %v", err) + } +} + +func TestPostgresImportCommittedFixture(t *testing.T) { + original := locateFixture(t) + before, err := sourceFingerprint(original) + if err != nil { + t.Fatal(err) + } + source := filepath.Join(t.TempDir(), "reviewed-fixture.sqlite") + if err := snapshotSource(context.Background(), original, source); err != nil { + t.Fatal(err) + } + legacyDB, err := openSQLite(source, "rw") + if err != nil { + t.Fatal(err) + } + // This historical fixture has one duplicate identity group. Its documented + // legacy repair is explicit and confined to this disposable copy; the runtime + // importer itself rejects duplicates and never silently collapses records. + if err := dbschema.Apply(legacyDB, t.Logf); err != nil { + legacyDB.Close() + t.Fatal(err) + } + var wantTransmissions, wantObservations int + if err := legacyDB.QueryRow(`SELECT count(*) FROM transmissions`).Scan(&wantTransmissions); err != nil { + t.Fatal(err) + } + if err := legacyDB.QueryRow(`SELECT count(*) FROM observations`).Scan(&wantObservations); err != nil { + t.Fatal(err) + } + legacyDB.Close() + dsn := postgresSchema(t) + report, err := importSQLite(context.Background(), importOptions{Source: source, DatabaseURL: dsn, StateDir: t.TempDir(), Kind: "telemetry", BatchSize: 64}) + if err != nil { + t.Fatal(err) + } + if !report.Verified { + t.Fatal("committed fixture import was not verified") + } + if _, err := finalizeImport(context.Background(), dsn, "telemetry"); err != nil { + t.Fatal(err) + } + db := openImportDB(t, dsn) + if err := postgresSchemaAPI.AssertReady(db); err != nil { + t.Fatal(err) + } + var transmissions, observations int + if err := db.QueryRow(`SELECT count(*) FROM transmissions`).Scan(&transmissions); err != nil { + t.Fatal(err) + } + if err := db.QueryRow(`SELECT count(*) FROM observations`).Scan(&observations); err != nil { + t.Fatal(err) + } + if transmissions != wantTransmissions || observations != wantObservations || transmissions == 0 { + t.Fatalf("reviewed fixture changed: %d/%d transmissions, %d/%d observations", transmissions, wantTransmissions, observations, wantObservations) + } + after, err := sourceFingerprint(original) + if err != nil { + t.Fatal(err) + } + if after != before { + t.Fatal("committed fixture changed during import") + } +} + +func TestFixturePreparationToolKeepsOriginalAndRefusesOverwrite(t *testing.T) { + source := locateFixture(t) + before, err := sourceFingerprint(source) + if err != nil { + t.Fatal(err) + } + destination := filepath.Join(t.TempDir(), "prepared.sqlite") + args := []string{"run", "../../scripts/prepare-postgres-fixture.go", "-source", source, "-destination", destination} + output, err := exec.Command("go", args...).CombinedOutput() + if err != nil { + t.Fatalf("fixture tool: %v: %s", err, output) + } + if !strings.Contains(string(output), "observations:") || !strings.Contains(string(output), "original fixture unchanged") { + t.Fatal("fixture tool did not report the repair counts") + } + if output, err := exec.Command("go", "run", "../../scripts/migrate-fixture-hashes.go", destination).CombinedOutput(); err != nil { + t.Fatalf("prepared fixture hash normalization: %v: %s", err, output) + } + db, err := openSQLite(destination, "ro") + if err != nil { + t.Fatal(err) + } + defer db.Close() + if err := dbschema.CheckLegacySource(db); err != nil { + t.Fatal(err) + } + if err := exec.Command("go", args...).Run(); err == nil { + t.Fatal("fixture tool overwrote an existing destination") + } + after, err := sourceFingerprint(source) + if err != nil || after != before { + t.Fatal("fixture tool changed original source", err) + } +} + +func TestFixtureHashToolUsesSharedRuntimeIdentity(t *testing.T) { + file := filepath.Join(t.TempDir(), "hashes.sqlite") + db, err := sql.Open("sqlite3", file) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`CREATE TABLE transmissions(id INTEGER PRIMARY KEY,raw_hex TEXT,hash TEXT UNIQUE)`); err != nil { + t.Fatal(err) + } + raws := []string{"14010203040001020304", "150001020305", "160001020306", "17010203040001020307", "250001020304", "1d0001020304"} + for i, raw := range raws { + if _, err := db.Exec(`INSERT INTO transmissions VALUES(?,?,?)`, i, raw, fmt.Sprint("old-", i)); err != nil { + t.Fatal(err) + } + } + db.Close() + output, err := exec.Command("go", "run", "../../scripts/migrate-fixture-hashes.go", file).CombinedOutput() + if err != nil { + t.Fatalf("hash fixture: %v: %s", err, output) + } + db, err = openSQLite(file, "ro") + if err != nil { + t.Fatal(err) + } + defer db.Close() + for i, raw := range raws { + var got string + if err := db.QueryRow(`SELECT hash FROM transmissions WHERE id=?`, i).Scan(&got); err != nil { + t.Fatal(err) + } + if want := packetpath.ContentHash(raw); got != want { + t.Fatalf("fixture route/type %s hash=%s, want runtime %s", raw, got, want) + } + } +} + func copyFile(t *testing.T, src, dst string) { t.Helper() in, err := os.Open(src) diff --git a/cmd/migrate/observation_hops_test.go b/cmd/migrate/observation_hops_test.go new file mode 100644 index 000000000..732ecdb54 --- /dev/null +++ b/cmd/migrate/observation_hops_test.go @@ -0,0 +1,318 @@ +package main + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "os" + "path/filepath" + "reflect" + "testing" + "time" + + "github.com/meshcore-analyzer/dbschema" +) + +// The accelerator can be absent, complete, or midway through its independently +// committed backfill. None of those states may change the canonical rows. +func hopIndexSource(t *testing.T, state string) string { + t.Helper() + path := telemetrySource(t) + db, err := openSQLite(path, "rw") + if err != nil { + t.Fatal(err) + } + defer db.Close() + for _, stmt := range []string{ + `UPDATE observations SET path_json=' {malformed',score=0.75 WHERE id=90`, + `UPDATE observations SET path_json=' ["aB","Cd","aB",17,null] ' WHERE id=91`, + `INSERT INTO observations(id,transmission_id,observer_idx,path_json,timestamp) VALUES(92,80,NULL,'["Ef"]',3)`, + } { + if _, err := db.Exec(stmt); err != nil { + t.Fatal(err) + } + } + if state == "absent" { + return path + } + if err := dbschema.EnsureObservationHopsSchema(db, nil); err != nil { + t.Fatal(err) + } + floor := int64(0) + if state == "partial" { + floor = 3 + } + if _, err := dbschema.IndexObservationHops(db, floor, 4); err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`UPDATE observation_hops_coverage SET since=? WHERE id=1`, floor); err != nil { + t.Fatal(err) + } + return path +} + +type observationTuple struct { + ID, Timestamp int64 + Path sql.NullString + Score sql.NullFloat64 +} + +func observationTuples(t *testing.T, db *sql.DB) []observationTuple { + t.Helper() + rows, err := db.Query(`SELECT id,timestamp,path_json,score FROM observations ORDER BY id`) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + var out []observationTuple + for rows.Next() { + var row observationTuple + if err := rows.Scan(&row.ID, &row.Timestamp, &row.Path, &row.Score); err != nil { + t.Fatal(err) + } + out = append(out, row) + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + return out +} + +func TestPrepareHopIndexSourcePreservesOriginalAndRecovery(t *testing.T) { + for _, state := range []string{"absent", "complete", "partial"} { + t.Run(state, func(t *testing.T) { + ctx := context.Background() + source := hopIndexSource(t, state) + before, err := fingerprint(ctx, source) + if err != nil { + t.Fatal(err) + } + o := importOptions{Source: source, StateDir: t.TempDir(), Kind: "telemetry", targetDigest: "fixture"} + tables, err := layouts(o.Kind) + if err != nil { + t.Fatal(err) + } + manifest, normalized, err := prepareSource(ctx, o, tables, true) + if err != nil { + t.Fatalf("known %s hop-index source rejected: %v", state, err) + } + if after, err := fingerprint(ctx, source); err != nil || after != before { + t.Fatal("preparation modified source", err) + } + var original []observationTuple + for i, path := range []string{source, filepath.Join(o.StateDir, o.Kind, "recovery.sqlite"), normalized} { + db, err := openSQLite(path, "ro") + if err != nil { + t.Fatal(err) + } + got := observationTuples(t, db) + if i == 0 { + original = got + } else if !reflect.DeepEqual(got, original) { + t.Fatal("canonical observations changed", got, original) + } + var objects int + err = db.QueryRow(`SELECT count(*) FROM sqlite_schema WHERE name LIKE 'observation_hops%' OR name LIKE 'trg_observation_hops%'`).Scan(&objects) + db.Close() + want := 5 + if state == "absent" || i == 2 { + want = 0 + } + if err != nil || objects != want { + t.Fatalf("snapshot %d has %d hop objects, want %d: %v", i, objects, want, err) + } + } + o.Resume = true + resumed, resumedPath, err := prepareSource(ctx, o, tables, false) + if err != nil || !reflect.DeepEqual(resumed, manifest) || resumedPath != normalized { + t.Fatal("prepared snapshots cannot resume unchanged", err) + } + }) + } +} + +func TestPrepareHopIndexRejectsUnknownOrChangedObjects(t *testing.T) { + for name, statements := range map[string][]string{ + "missing trigger": {`DROP TRIGGER trg_observation_hops_ad`}, + "changed trigger": {`DROP TRIGGER trg_observation_hops_ai`, `CREATE TRIGGER trg_observation_hops_ai AFTER INSERT ON observations BEGIN DELETE FROM observations; END`}, + "changed column": {`ALTER TABLE observation_hops ADD COLUMN private_payload TEXT`}, + "changed constraint": {`DROP TABLE observation_hops_coverage`, `CREATE TABLE observation_hops_coverage(id INTEGER PRIMARY KEY,since INTEGER NOT NULL)`}, + "changed primary key": {`DROP TABLE observation_hops`, `CREATE TABLE observation_hops(hop TEXT NOT NULL,timestamp INTEGER NOT NULL,observation_id INTEGER NOT NULL,PRIMARY KEY(observation_id,hop,timestamp)) WITHOUT ROWID`}, + "unknown index": {`CREATE INDEX surprise_hop_index ON observation_hops(observation_id)`}, + "unknown table": {`CREATE TABLE private_hops(payload TEXT)`}, + "unknown trigger": {`CREATE TRIGGER unexpected_hops AFTER INSERT ON observations BEGIN DELETE FROM observation_hops; END`}, + "unknown view": {`CREATE VIEW extra_hops AS SELECT * FROM observation_hops`}, + } { + t.Run(name, func(t *testing.T) { + source := hopIndexSource(t, "complete") + db, err := openSQLite(source, "rw") + if err != nil { + t.Fatal(err) + } + for _, stmt := range statements { + if _, err := db.Exec(stmt); err != nil { + t.Fatal(err) + } + } + db.Close() + ctx := context.Background() + before, err := fingerprint(ctx, source) + if err != nil { + t.Fatal(err) + } + o := importOptions{Source: source, StateDir: t.TempDir(), Kind: "telemetry", targetDigest: "fixture"} + tables, err := layouts(o.Kind) + if err != nil { + t.Fatal(err) + } + if _, _, err := prepareSource(ctx, o, tables, true); err == nil { + t.Fatal("unknown or altered source layout was accepted") + } + if after, err := fingerprint(ctx, source); err != nil || after != before { + t.Fatal("refusal modified source", err) + } + }) + } +} + +func TestHopIndexResumeRejectsChangedExclusionReport(t *testing.T) { + ctx := context.Background() + o := importOptions{Source: hopIndexSource(t, "partial"), StateDir: t.TempDir(), Kind: "telemetry", targetDigest: "fixture"} + tables, err := layouts(o.Kind) + if err != nil { + t.Fatal(err) + } + manifest, _, err := prepareSource(ctx, o, tables, true) + if err != nil { + t.Fatal(err) + } + manifest.ExcludedDerived = nil + data, err := json.Marshal(manifest) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(o.StateDir, o.Kind, "manifest.json"), data, 0600); err != nil { + t.Fatal(err) + } + o.Resume = true + if _, _, err := prepareSource(ctx, o, tables, false); err == nil { + t.Fatal("altered exclusion report was trusted on resume") + } +} + +func TestHopIndexNativeRoundTripAndInterruptedImport(t *testing.T) { + requirePostgres(t) + for _, state := range []string{"absent", "complete", "partial"} { + t.Run(state, func(t *testing.T) { + ctx := context.Background() + source := hopIndexSource(t, state) + before, err := fingerprint(ctx, source) + if err != nil { + t.Fatal(err) + } + o := importOptions{Source: source, DatabaseURL: postgresSchema(t), StateDir: t.TempDir(), Kind: "telemetry", BatchSize: 1, afterBatch: func() error { return context.Canceled }} + if _, err := importSQLite(ctx, o); !errors.Is(err, context.Canceled) { + t.Fatal("expected committed batch interruption", err) + } + o.Resume, o.afterBatch = true, nil + report, err := importSQLite(ctx, o) + if err != nil || !report.Verified { + t.Fatal("hop-index import did not resume", err) + } + for _, table := range report.Tables { + if table.Table == "observation_hops" || table.Table == "observation_hops_coverage" { + t.Fatal("derived rows included in canonical report") + } + } + var decoded struct { + ExcludedDerived []struct { + Table string `json:"table"` + SchemaVersion int `json:"schema_version"` + Reason string `json:"reason"` + } `json:"excluded_derived"` + } + data, err := json.Marshal(report) + if err != nil || json.Unmarshal(data, &decoded) != nil { + t.Fatal("invalid import report", err) + } + wantExclusions := 2 + if state == "absent" { + wantExclusions = 0 + } + if len(decoded.ExcludedDerived) != wantExclusions { + t.Fatal("report did not explicitly identify excluded derived rows", string(data)) + } + for i, table := range decoded.ExcludedDerived { + if table.SchemaVersion != 1 || table.Reason == "" || table.Table != []string{"observation_hops", "observation_hops_coverage"}[i] { + t.Fatal("exclusion report lacks its exact known layout", table) + } + } + target := openImportDB(t, o.DatabaseURL) + var count int + if err := target.QueryRow(`SELECT count(*) FROM pg_class WHERE relnamespace=current_schema()::regnamespace AND relname IN ('observation_hops','observation_hops_coverage')`).Scan(&count); err != nil || count != 0 { + t.Fatal("derived SQLite cache copied to PostgreSQL", count, err) + } + if _, err := finalizeImport(ctx, o.DatabaseURL, o.Kind); err != nil { + t.Fatal(err) + } + resumed, err := importSQLite(ctx, o) + if err != nil || !reflect.DeepEqual(resumed, report) { + t.Fatal("finalized resume changed report", err) + } + reverse := reverseOptions{DatabaseURL: o.DatabaseURL, Destination: filepath.Join(t.TempDir(), "reverse.sqlite"), StateDir: t.TempDir(), Kind: "telemetry", BatchSize: 1, afterBatch: func() error { return context.Canceled }} + if _, err := reversePostgres(ctx, reverse); !errors.Is(err, context.Canceled) { + t.Fatal("expected reverse interruption", err) + } + reverse.Resume, reverse.afterBatch = true, nil + back, err := reversePostgres(ctx, reverse) + if err != nil || !back.Verified { + t.Fatal("hop-index reverse did not resume", err) + } + if !reflect.DeepEqual(back.Tables, report.Tables) { + t.Fatal("canonical table counts or digests changed in the round trip") + } + recovered, err := openSQLite(reverse.Destination, "rw") + if err != nil { + t.Fatal(err) + } + defer recovered.Close() + if err := recovered.QueryRow(`SELECT count(*) FROM sqlite_schema WHERE name LIKE 'observation_hops%' OR name LIKE 'trg_observation_hops%'`).Scan(&count); err != nil || count != 0 { + t.Fatal("reverse created false hop coverage", count, err) + } + if got, want := observationTuples(t, recovered), observationTuples(t, target); !reflect.DeepEqual(got, want) { + t.Fatal("raw paths, IDs, or fractional scores changed in reverse", got, want) + } + // The owner-controlled SQLite ingestor installs this exact schema. + // Installing it must not inherit source coverage or imply old rows + // have been rebuilt; its existing bounded backfill does that later. + now := time.Now().Unix() + if err := dbschema.EnsureObservationHopsSchema(recovered, nil); err != nil { + t.Fatal(err) + } + var since int64 + if err := recovered.QueryRow(`SELECT since FROM observation_hops_coverage WHERE id=1`).Scan(&since); err != nil || since < now { + t.Fatal("new installation inherited old coverage", since, err) + } + if err := recovered.QueryRow(`SELECT count(*) FROM observation_hops`).Scan(&count); err != nil || count != 0 { + t.Fatal("unrebuilt cache unexpectedly populated", count, err) + } + if after, err := fingerprint(ctx, source); err != nil || after != before { + t.Fatal("round trip changed original source", err) + } + if state == "partial" { + report.ExcludedDerived = nil + data, err := json.Marshal(report) + if err != nil { + t.Fatal(err) + } + if _, err := target.Exec(`UPDATE corescope_import SET report=$1 WHERE id=1`, string(data)); err != nil { + t.Fatal(err) + } + if _, err := importSQLite(ctx, o); err == nil { + t.Fatal("finalized resume trusted changed exclusion report") + } + } + }) + } +} diff --git a/cmd/migrate/operator_paths_test.go b/cmd/migrate/operator_paths_test.go new file mode 100644 index 000000000..9649ab2c7 --- /dev/null +++ b/cmd/migrate/operator_paths_test.go @@ -0,0 +1,81 @@ +package main + +import ( + "bytes" + "context" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestOperatorCustomSourcePathsAndOptionalAccounts(t *testing.T) { + for _, existingAccounts := range []bool{false, true} { + t.Run(map[bool]string{false: "no previous accounts", true: "preserve existing accounts"}[existingAccounts], func(t *testing.T) { + t.Setenv("CORESCOPE_DATABASE_URL", postgresDatabase(t)) + accounts := postgresDatabase(t) + t.Setenv("CORESCOPE_USERS_DATABASE_URL", accounts) + dir := filepath.Join(t.TempDir(), "custom state and sources") + if err := os.Mkdir(dir, 0o700); err != nil { + t.Fatal(err) + } + source := filepath.Join(dir, "telemetry retained.db") + if err := os.Rename(telemetrySource(t), source); err != nil { + t.Fatal(err) + } + args := []string{"-offline", "-from-sqlite", source, "-state-dir", filepath.Join(dir, "migration snapshots")} + paths := []string{source} + if existingAccounts { + users := filepath.Join(dir, "existing accounts.db") + if err := os.Rename(accountSource(t, 5), users); err != nil { + t.Fatal(err) + } + args = append(args, "-users-from-sqlite", users) + paths = append(paths, users) + } + before := make([]string, len(paths)) + for i, path := range paths { + var err error + before[i], err = fingerprint(context.Background(), path) + if err != nil { + t.Fatal(err) + } + } + var output bytes.Buffer + if err := runCommand(context.Background(), args, &output); err != nil { + t.Fatal(err) + } + if err := runCommand(context.Background(), []string{"-check-ready"}, &output); err != nil { + t.Fatal(err) + } + for i, path := range paths { + kind, flag := "telemetry", "-from-sqlite" + if i == 1 { + kind, flag = "accounts", "-users-from-sqlite" + } + if err := runCommand(context.Background(), []string{"-check-import-kind=" + kind, flag, path}, &output); err != nil { + t.Fatal(err) + } + after, err := fingerprint(context.Background(), path) + if err != nil || before[i] != after { + t.Fatal("operator import/check changed the retained source", err) + } + } + var n int + if err := openImportDB(t, accounts).QueryRow(`SELECT count(*) FROM users`).Scan(&n); err != nil { + t.Fatal(err) + } + if (existingAccounts && n != 1) || (!existingAccounts && n != 0) { + t.Fatal("optional account initialization lost or invented accounts") + } + for _, kind := range []string{"telemetry", "accounts"} { + if !strings.Contains(output.String(), kind+" PostgreSQL schema is ready") { + t.Fatal("documented readiness message is missing") + } + } + if !existingAccounts && !strings.Contains(output.String(), "accounts PostgreSQL schema initialized") { + t.Fatal("documented absent-account initialization message is missing") + } + }) + } +} diff --git a/cmd/migrate/postgres_layout.go b/cmd/migrate/postgres_layout.go new file mode 100644 index 000000000..685da74a6 --- /dev/null +++ b/cmd/migrate/postgres_layout.go @@ -0,0 +1,128 @@ +package main + +import ( + "database/sql" + _ "embed" + "encoding/json" + "errors" + "slices" + "sort" + "strings" +) + +// Captured from the versioned native ApplyPostgres entrypoints. The regression +// recreates both canonical schemas and compares this manifest, so a physical +// schema change requires an explicit converter compatibility decision. +// +//go:embed postgres_layout.json +var postgresLayoutJSON []byte + +func postgresLayout(db *sql.DB, kind string, tables []importTable) ([]string, error) { + var schema string + if err := db.QueryRow(`SELECT current_schema()`).Scan(&schema); err != nil { + return nil, err + } + allowed := map[string]bool{"corescope_schema": true, "corescope_import": true, "corescope_import_progress": true} + if kind == "accounts" { + allowed["schema_version"] = true + } + var tableNames []string + for _, table := range tables { + tableNames = append(tableNames, `'`+strings.ReplaceAll(table.Name, `'`, `''`)+`'`) + allowed[table.Name] = true + } + views := []string{"packets_v"} + if kind == "accounts" { + views = []string{"approved_channels", "linked_companions"} + } + for _, view := range views { + allowed[view] = true + } + viewNames := "'" + strings.Join(views, "','") + "'" + relationRows, err := db.Query(`SELECT c.relname,c.relkind::text FROM pg_class c WHERE c.relnamespace=current_schema()::regnamespace AND c.relkind IN ('r','p','v','m','S','f') ORDER BY c.relname`) + if err != nil { + return nil, err + } + var relations [][2]string + for relationRows.Next() { + var pair [2]string + if err := relationRows.Scan(&pair[0], &pair[1]); err != nil { + relationRows.Close() + return nil, err + } + relations = append(relations, pair) + } + err = relationRows.Err() + relationRows.Close() + if err != nil { + return nil, err + } + for _, table := range tables { + for _, col := range table.Columns { + if !col.Identity { + continue + } + var name string + if err := db.QueryRow(`SELECT c.relname FROM pg_class c WHERE c.oid=pg_get_serial_sequence($1,$2)::regclass`, table.Name, col.Name).Scan(&name); err != nil { + return nil, err + } + allowed[name] = true + } + } + for _, pair := range relations { + if !allowed[pair[0]] { + return nil, errors.New("unsupported PostgreSQL source object; no source data was discarded") + } + } + names := strings.Join(tableNames, ",") + queries := []string{ + `SELECT json_build_array('column',table_name,ordinal_position,column_name,data_type,is_nullable,column_default,collation_name,is_identity,identity_generation,is_generated,generation_expression)::text FROM information_schema.columns WHERE table_schema=current_schema() AND table_name IN (` + names + `)`, + `SELECT json_build_array('table',c.relname,c.relkind,c.relrowsecurity,c.relforcerowsecurity)::text FROM pg_class c WHERE c.relnamespace=current_schema()::regnamespace AND c.relname IN (` + names + `)`, + `SELECT json_build_array('constraint',c.relname,k.conname,k.convalidated,pg_get_constraintdef(k.oid))::text FROM pg_constraint k JOIN pg_class c ON c.oid=k.conrelid WHERE c.relnamespace=current_schema()::regnamespace AND c.relname IN (` + names + `)`, + `SELECT json_build_array('index',c.relname,pg_get_indexdef(i.indexrelid))::text FROM pg_index i JOIN pg_class c ON c.oid=i.indrelid WHERE c.relnamespace=current_schema()::regnamespace AND c.relname IN (` + names + `)`, + `SELECT json_build_array('trigger',c.relname,pg_get_triggerdef(t.oid))::text FROM pg_trigger t JOIN pg_class c ON c.oid=t.tgrelid WHERE NOT t.tgisinternal AND c.relnamespace=current_schema()::regnamespace AND c.relname IN (` + names + `)`, + `SELECT json_build_array('policy',c.relname,p.polname,pg_get_expr(p.polqual,p.polrelid),pg_get_expr(p.polwithcheck,p.polrelid))::text FROM pg_policy p JOIN pg_class c ON c.oid=p.polrelid WHERE c.relnamespace=current_schema()::regnamespace AND c.relname IN (` + names + `)`, + `SELECT json_build_array('view',c.relname,pg_get_viewdef(c.oid))::text FROM pg_class c WHERE c.relnamespace=current_schema()::regnamespace AND c.relkind='v' AND c.relname IN (` + viewNames + `)`, + } + var out []string + for _, query := range queries { + rows, err := db.Query(query) + if err != nil { + return nil, err + } + for rows.Next() { + var value string + if err := rows.Scan(&value); err != nil { + rows.Close() + return nil, err + } + // Only catalog qualification varies between installations. User data is + // never normalized, decoded or rewritten by these schema comparisons. + value = strings.ReplaceAll(value, strings.ReplaceAll(quote(schema), `"`, `\"`)+".", "{schema}.") + value = strings.ReplaceAll(value, schema+".", "{schema}.") + out = append(out, value) + } + err = rows.Err() + rows.Close() + if err != nil { + return nil, err + } + } + sort.Strings(out) + return out, nil +} + +func validatePostgresLayout(db *sql.DB, kind string, tables []importTable) error { + var expected map[string][]string + if err := json.Unmarshal(postgresLayoutJSON, &expected); err != nil { + return errors.New("invalid compiled PostgreSQL conversion layout") + } + got, err := postgresLayout(db, kind, tables) + if err != nil { + return err + } + if !slices.Equal(got, expected[kind]) { + return errors.New("unsupported PostgreSQL columns, constraints, indexes, policies or views; conversion refused without changing the source") + } + return nil +} diff --git a/cmd/migrate/postgres_layout.json b/cmd/migrate/postgres_layout.json new file mode 100644 index 000000000..be39fdca9 --- /dev/null +++ b/cmd/migrate/postgres_layout.json @@ -0,0 +1,645 @@ +{ + "accounts": [ + "[\"column\", \"audit_log\", 1, \"id\", \"bigint\", \"NO\", null, null, \"YES\", \"BY DEFAULT\", \"NEVER\", null]", + "[\"column\", \"audit_log\", 2, \"at\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"audit_log\", 3, \"actor_user_id\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"audit_log\", 4, \"action\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"audit_log\", 5, \"target_user_id\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"audit_log\", 6, \"detail\", \"text\", \"NO\", \"'{}'::text\", \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"companion_links\", 1, \"pubkey\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"companion_links\", 2, \"user_id\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"companion_links\", 3, \"name\", \"text\", \"NO\", \"''::text\", \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"companion_links\", 4, \"linked_at\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"link_challenges\", 1, \"challenge_hash\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"link_challenges\", 2, \"user_id\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"link_challenges\", 3, \"pubkey\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"link_challenges\", 4, \"expires_at\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"mail_events\", 1, \"id\", \"bigint\", \"NO\", null, null, \"YES\", \"BY DEFAULT\", \"NEVER\", null]", + "[\"column\", \"mail_events\", 2, \"mail_id\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"mail_events\", 3, \"event\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"mail_events\", 4, \"at\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"mail_events\", 5, \"reason\", \"text\", \"NO\", \"''::text\", \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"mail_log\", 1, \"id\", \"bigint\", \"NO\", null, null, \"YES\", \"BY DEFAULT\", \"NEVER\", null]", + "[\"column\", \"mail_log\", 2, \"user_id\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"mail_log\", 3, \"to_email\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"mail_log\", 4, \"purpose\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"mail_log\", 5, \"provider_message_id\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"mail_log\", 6, \"sent_at\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"mail_log\", 7, \"last_event\", \"text\", \"NO\", \"'sent'::text\", \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"mail_log\", 8, \"last_event_at\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"mail_log\", 9, \"last_reason\", \"text\", \"NO\", \"''::text\", \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"notification_prefs\", 1, \"user_id\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"notification_prefs\", 2, \"enabled\", \"bigint\", \"NO\", \"1\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"notification_prefs\", 3, \"events\", \"text\", \"NO\", \"''::text\", \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"notification_prefs\", 4, \"unsub_token\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"notification_prefs\", 5, \"updated_at\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"notification_state\", 1, \"user_id\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"notification_state\", 2, \"event\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"notification_state\", 3, \"subject\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"notification_state\", 4, \"state\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"notification_state\", 5, \"changed_at\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"notification_watches\", 1, \"user_id\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"notification_watches\", 2, \"pubkey\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"notification_watches\", 3, \"created_at\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"proposals\", 1, \"id\", \"bigint\", \"NO\", null, null, \"YES\", \"BY DEFAULT\", \"NEVER\", null]", + "[\"column\", \"proposals\", 2, \"kind\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"proposals\", 3, \"subject\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"proposals\", 4, \"status\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"proposals\", 5, \"proposer_id\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"proposals\", 6, \"reviewer_id\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"proposals\", 7, \"note\", \"text\", \"NO\", \"''::text\", \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"proposals\", 8, \"created_at\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"proposals\", 9, \"decided_at\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"sessions\", 1, \"id\", \"bigint\", \"NO\", null, null, \"YES\", \"BY DEFAULT\", \"NEVER\", null]", + "[\"column\", \"sessions\", 10, \"label\", \"text\", \"NO\", \"''::text\", \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"sessions\", 11, \"scopes\", \"text\", \"NO\", \"''::text\", \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"sessions\", 2, \"token_hash\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"sessions\", 3, \"user_id\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"sessions\", 4, \"csrf_token\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"sessions\", 5, \"created_at\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"sessions\", 6, \"expires_at\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"sessions\", 7, \"last_seen_at\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"sessions\", 8, \"user_agent\", \"text\", \"NO\", \"''::text\", \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"sessions\", 9, \"kind\", \"text\", \"NO\", \"'web'::text\", \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"tokens\", 1, \"token_hash\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"tokens\", 2, \"user_id\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"tokens\", 3, \"purpose\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"tokens\", 4, \"new_email\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"tokens\", 5, \"expires_at\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"tokens\", 6, \"used_at\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"user_settings\", 1, \"user_id\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"user_settings\", 2, \"doc\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"user_settings\", 3, \"revision\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"user_settings\", 4, \"generation\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"user_settings\", 5, \"updated_at\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"users\", 1, \"id\", \"bigint\", \"NO\", null, null, \"YES\", \"BY DEFAULT\", \"NEVER\", null]", + "[\"column\", \"users\", 10, \"last_login_at\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"users\", 11, \"email_bouncing\", \"bigint\", \"NO\", \"0\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"users\", 2, \"email\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"users\", 3, \"display_name\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"users\", 4, \"password_hash\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"users\", 5, \"role\", \"text\", \"NO\", \"'user'::text\", \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"users\", 6, \"status\", \"text\", \"NO\", \"'pending'::text\", \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"users\", 7, \"created_at\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"users\", 8, \"activated_at\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"users\", 9, \"activated_by\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"constraint\", \"audit_log\", \"audit_log_action_not_null\", true, \"NOT NULL action\"]", + "[\"constraint\", \"audit_log\", \"audit_log_at_not_null\", true, \"NOT NULL at\"]", + "[\"constraint\", \"audit_log\", \"audit_log_detail_not_null\", true, \"NOT NULL detail\"]", + "[\"constraint\", \"audit_log\", \"audit_log_id_not_null\", true, \"NOT NULL id\"]", + "[\"constraint\", \"audit_log\", \"audit_log_pkey\", true, \"PRIMARY KEY (id)\"]", + "[\"constraint\", \"companion_links\", \"companion_links_linked_at_not_null\", true, \"NOT NULL linked_at\"]", + "[\"constraint\", \"companion_links\", \"companion_links_name_not_null\", true, \"NOT NULL name\"]", + "[\"constraint\", \"companion_links\", \"companion_links_pkey\", true, \"PRIMARY KEY (pubkey)\"]", + "[\"constraint\", \"companion_links\", \"companion_links_pubkey_not_null\", true, \"NOT NULL pubkey\"]", + "[\"constraint\", \"companion_links\", \"companion_links_user_id_fkey\", true, \"FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE\"]", + "[\"constraint\", \"companion_links\", \"companion_links_user_id_not_null\", true, \"NOT NULL user_id\"]", + "[\"constraint\", \"link_challenges\", \"link_challenges_challenge_hash_not_null\", true, \"NOT NULL challenge_hash\"]", + "[\"constraint\", \"link_challenges\", \"link_challenges_expires_at_not_null\", true, \"NOT NULL expires_at\"]", + "[\"constraint\", \"link_challenges\", \"link_challenges_pkey\", true, \"PRIMARY KEY (challenge_hash)\"]", + "[\"constraint\", \"link_challenges\", \"link_challenges_pubkey_not_null\", true, \"NOT NULL pubkey\"]", + "[\"constraint\", \"link_challenges\", \"link_challenges_user_id_fkey\", true, \"FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE\"]", + "[\"constraint\", \"link_challenges\", \"link_challenges_user_id_not_null\", true, \"NOT NULL user_id\"]", + "[\"constraint\", \"mail_events\", \"mail_events_at_not_null\", true, \"NOT NULL at\"]", + "[\"constraint\", \"mail_events\", \"mail_events_event_not_null\", true, \"NOT NULL event\"]", + "[\"constraint\", \"mail_events\", \"mail_events_id_not_null\", true, \"NOT NULL id\"]", + "[\"constraint\", \"mail_events\", \"mail_events_mail_id_event_at_key\", true, \"UNIQUE (mail_id, event, at)\"]", + "[\"constraint\", \"mail_events\", \"mail_events_mail_id_fkey\", true, \"FOREIGN KEY (mail_id) REFERENCES mail_log(id) ON DELETE CASCADE\"]", + "[\"constraint\", \"mail_events\", \"mail_events_mail_id_not_null\", true, \"NOT NULL mail_id\"]", + "[\"constraint\", \"mail_events\", \"mail_events_pkey\", true, \"PRIMARY KEY (id)\"]", + "[\"constraint\", \"mail_events\", \"mail_events_reason_not_null\", true, \"NOT NULL reason\"]", + "[\"constraint\", \"mail_log\", \"mail_log_id_not_null\", true, \"NOT NULL id\"]", + "[\"constraint\", \"mail_log\", \"mail_log_last_event_at_not_null\", true, \"NOT NULL last_event_at\"]", + "[\"constraint\", \"mail_log\", \"mail_log_last_event_not_null\", true, \"NOT NULL last_event\"]", + "[\"constraint\", \"mail_log\", \"mail_log_last_reason_not_null\", true, \"NOT NULL last_reason\"]", + "[\"constraint\", \"mail_log\", \"mail_log_pkey\", true, \"PRIMARY KEY (id)\"]", + "[\"constraint\", \"mail_log\", \"mail_log_purpose_not_null\", true, \"NOT NULL purpose\"]", + "[\"constraint\", \"mail_log\", \"mail_log_sent_at_not_null\", true, \"NOT NULL sent_at\"]", + "[\"constraint\", \"mail_log\", \"mail_log_to_email_not_null\", true, \"NOT NULL to_email\"]", + "[\"constraint\", \"mail_log\", \"mail_log_user_id_fkey\", true, \"FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE SET NULL\"]", + "[\"constraint\", \"notification_prefs\", \"notification_prefs_enabled_check\", true, \"CHECK ((enabled = ANY (ARRAY[(0)::bigint, (1)::bigint])))\"]", + "[\"constraint\", \"notification_prefs\", \"notification_prefs_enabled_not_null\", true, \"NOT NULL enabled\"]", + "[\"constraint\", \"notification_prefs\", \"notification_prefs_events_not_null\", true, \"NOT NULL events\"]", + "[\"constraint\", \"notification_prefs\", \"notification_prefs_pkey\", true, \"PRIMARY KEY (user_id)\"]", + "[\"constraint\", \"notification_prefs\", \"notification_prefs_unsub_token_key\", true, \"UNIQUE (unsub_token)\"]", + "[\"constraint\", \"notification_prefs\", \"notification_prefs_unsub_token_not_null\", true, \"NOT NULL unsub_token\"]", + "[\"constraint\", \"notification_prefs\", \"notification_prefs_updated_at_not_null\", true, \"NOT NULL updated_at\"]", + "[\"constraint\", \"notification_prefs\", \"notification_prefs_user_id_fkey\", true, \"FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE\"]", + "[\"constraint\", \"notification_prefs\", \"notification_prefs_user_id_not_null\", true, \"NOT NULL user_id\"]", + "[\"constraint\", \"notification_state\", \"notification_state_changed_at_not_null\", true, \"NOT NULL changed_at\"]", + "[\"constraint\", \"notification_state\", \"notification_state_event_not_null\", true, \"NOT NULL event\"]", + "[\"constraint\", \"notification_state\", \"notification_state_pkey\", true, \"PRIMARY KEY (user_id, event, subject)\"]", + "[\"constraint\", \"notification_state\", \"notification_state_state_not_null\", true, \"NOT NULL state\"]", + "[\"constraint\", \"notification_state\", \"notification_state_subject_not_null\", true, \"NOT NULL subject\"]", + "[\"constraint\", \"notification_state\", \"notification_state_user_id_fkey\", true, \"FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE\"]", + "[\"constraint\", \"notification_state\", \"notification_state_user_id_not_null\", true, \"NOT NULL user_id\"]", + "[\"constraint\", \"notification_watches\", \"notification_watches_created_at_not_null\", true, \"NOT NULL created_at\"]", + "[\"constraint\", \"notification_watches\", \"notification_watches_pkey\", true, \"PRIMARY KEY (user_id, pubkey)\"]", + "[\"constraint\", \"notification_watches\", \"notification_watches_pubkey_not_null\", true, \"NOT NULL pubkey\"]", + "[\"constraint\", \"notification_watches\", \"notification_watches_user_id_fkey\", true, \"FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE\"]", + "[\"constraint\", \"notification_watches\", \"notification_watches_user_id_not_null\", true, \"NOT NULL user_id\"]", + "[\"constraint\", \"proposals\", \"proposals_created_at_not_null\", true, \"NOT NULL created_at\"]", + "[\"constraint\", \"proposals\", \"proposals_id_not_null\", true, \"NOT NULL id\"]", + "[\"constraint\", \"proposals\", \"proposals_kind_not_null\", true, \"NOT NULL kind\"]", + "[\"constraint\", \"proposals\", \"proposals_note_not_null\", true, \"NOT NULL note\"]", + "[\"constraint\", \"proposals\", \"proposals_pkey\", true, \"PRIMARY KEY (id)\"]", + "[\"constraint\", \"proposals\", \"proposals_proposer_id_fkey\", true, \"FOREIGN KEY (proposer_id) REFERENCES users(id) ON DELETE SET NULL\"]", + "[\"constraint\", \"proposals\", \"proposals_reviewer_id_fkey\", true, \"FOREIGN KEY (reviewer_id) REFERENCES users(id) ON DELETE SET NULL\"]", + "[\"constraint\", \"proposals\", \"proposals_status_check\", true, \"CHECK ((status = ANY (ARRAY['pending'::text, 'approved'::text, 'rejected'::text, 'revoked'::text])))\"]", + "[\"constraint\", \"proposals\", \"proposals_status_not_null\", true, \"NOT NULL status\"]", + "[\"constraint\", \"proposals\", \"proposals_subject_not_null\", true, \"NOT NULL subject\"]", + "[\"constraint\", \"sessions\", \"sessions_created_at_not_null\", true, \"NOT NULL created_at\"]", + "[\"constraint\", \"sessions\", \"sessions_csrf_token_not_null\", true, \"NOT NULL csrf_token\"]", + "[\"constraint\", \"sessions\", \"sessions_expires_at_not_null\", true, \"NOT NULL expires_at\"]", + "[\"constraint\", \"sessions\", \"sessions_id_not_null\", true, \"NOT NULL id\"]", + "[\"constraint\", \"sessions\", \"sessions_kind_check\", true, \"CHECK ((kind = ANY (ARRAY['web'::text, 'device'::text])))\"]", + "[\"constraint\", \"sessions\", \"sessions_kind_not_null\", true, \"NOT NULL kind\"]", + "[\"constraint\", \"sessions\", \"sessions_label_not_null\", true, \"NOT NULL label\"]", + "[\"constraint\", \"sessions\", \"sessions_last_seen_at_not_null\", true, \"NOT NULL last_seen_at\"]", + "[\"constraint\", \"sessions\", \"sessions_pkey\", true, \"PRIMARY KEY (id)\"]", + "[\"constraint\", \"sessions\", \"sessions_scopes_not_null\", true, \"NOT NULL scopes\"]", + "[\"constraint\", \"sessions\", \"sessions_token_hash_key\", true, \"UNIQUE (token_hash)\"]", + "[\"constraint\", \"sessions\", \"sessions_token_hash_not_null\", true, \"NOT NULL token_hash\"]", + "[\"constraint\", \"sessions\", \"sessions_user_agent_not_null\", true, \"NOT NULL user_agent\"]", + "[\"constraint\", \"sessions\", \"sessions_user_id_fkey\", true, \"FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE\"]", + "[\"constraint\", \"sessions\", \"sessions_user_id_not_null\", true, \"NOT NULL user_id\"]", + "[\"constraint\", \"tokens\", \"tokens_expires_at_not_null\", true, \"NOT NULL expires_at\"]", + "[\"constraint\", \"tokens\", \"tokens_pkey\", true, \"PRIMARY KEY (token_hash)\"]", + "[\"constraint\", \"tokens\", \"tokens_purpose_check\", true, \"CHECK ((purpose = ANY (ARRAY['activate'::text, 'reset'::text, 'email_change'::text])))\"]", + "[\"constraint\", \"tokens\", \"tokens_purpose_not_null\", true, \"NOT NULL purpose\"]", + "[\"constraint\", \"tokens\", \"tokens_token_hash_not_null\", true, \"NOT NULL token_hash\"]", + "[\"constraint\", \"tokens\", \"tokens_user_id_fkey\", true, \"FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE\"]", + "[\"constraint\", \"tokens\", \"tokens_user_id_not_null\", true, \"NOT NULL user_id\"]", + "[\"constraint\", \"user_settings\", \"user_settings_doc_not_null\", true, \"NOT NULL doc\"]", + "[\"constraint\", \"user_settings\", \"user_settings_generation_not_null\", true, \"NOT NULL generation\"]", + "[\"constraint\", \"user_settings\", \"user_settings_pkey\", true, \"PRIMARY KEY (user_id)\"]", + "[\"constraint\", \"user_settings\", \"user_settings_revision_not_null\", true, \"NOT NULL revision\"]", + "[\"constraint\", \"user_settings\", \"user_settings_updated_at_not_null\", true, \"NOT NULL updated_at\"]", + "[\"constraint\", \"user_settings\", \"user_settings_user_id_fkey\", true, \"FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE\"]", + "[\"constraint\", \"user_settings\", \"user_settings_user_id_not_null\", true, \"NOT NULL user_id\"]", + "[\"constraint\", \"users\", \"users_created_at_not_null\", true, \"NOT NULL created_at\"]", + "[\"constraint\", \"users\", \"users_display_name_not_null\", true, \"NOT NULL display_name\"]", + "[\"constraint\", \"users\", \"users_email_bouncing_check\", true, \"CHECK ((email_bouncing = ANY (ARRAY[(0)::bigint, (1)::bigint])))\"]", + "[\"constraint\", \"users\", \"users_email_bouncing_not_null\", true, \"NOT NULL email_bouncing\"]", + "[\"constraint\", \"users\", \"users_email_key\", true, \"UNIQUE (email)\"]", + "[\"constraint\", \"users\", \"users_email_not_null\", true, \"NOT NULL email\"]", + "[\"constraint\", \"users\", \"users_id_not_null\", true, \"NOT NULL id\"]", + "[\"constraint\", \"users\", \"users_password_hash_not_null\", true, \"NOT NULL password_hash\"]", + "[\"constraint\", \"users\", \"users_pkey\", true, \"PRIMARY KEY (id)\"]", + "[\"constraint\", \"users\", \"users_role_check\", true, \"CHECK ((role = ANY (ARRAY['user'::text, 'admin'::text])))\"]", + "[\"constraint\", \"users\", \"users_role_not_null\", true, \"NOT NULL role\"]", + "[\"constraint\", \"users\", \"users_status_check\", true, \"CHECK ((status = ANY (ARRAY['pending'::text, 'active'::text, 'disabled'::text])))\"]", + "[\"constraint\", \"users\", \"users_status_not_null\", true, \"NOT NULL status\"]", + "[\"index\", \"audit_log\", \"CREATE INDEX audit_actor ON {schema}.audit_log USING btree (actor_user_id)\"]", + "[\"index\", \"audit_log\", \"CREATE INDEX audit_at ON {schema}.audit_log USING btree (at)\"]", + "[\"index\", \"audit_log\", \"CREATE INDEX audit_target ON {schema}.audit_log USING btree (target_user_id)\"]", + "[\"index\", \"audit_log\", \"CREATE UNIQUE INDEX audit_log_pkey ON {schema}.audit_log USING btree (id)\"]", + "[\"index\", \"companion_links\", \"CREATE INDEX companion_links_user ON {schema}.companion_links USING btree (user_id)\"]", + "[\"index\", \"companion_links\", \"CREATE UNIQUE INDEX companion_links_pkey ON {schema}.companion_links USING btree (pubkey)\"]", + "[\"index\", \"link_challenges\", \"CREATE UNIQUE INDEX link_challenges_pkey ON {schema}.link_challenges USING btree (challenge_hash)\"]", + "[\"index\", \"mail_events\", \"CREATE UNIQUE INDEX mail_events_mail_id_event_at_key ON {schema}.mail_events USING btree (mail_id, event, at)\"]", + "[\"index\", \"mail_events\", \"CREATE UNIQUE INDEX mail_events_pkey ON {schema}.mail_events USING btree (id)\"]", + "[\"index\", \"mail_log\", \"CREATE INDEX mail_user ON {schema}.mail_log USING btree (user_id)\"]", + "[\"index\", \"mail_log\", \"CREATE UNIQUE INDEX mail_log_pkey ON {schema}.mail_log USING btree (id)\"]", + "[\"index\", \"mail_log\", \"CREATE UNIQUE INDEX mail_msgid ON {schema}.mail_log USING btree (provider_message_id) WHERE (provider_message_id IS NOT NULL)\"]", + "[\"index\", \"notification_prefs\", \"CREATE UNIQUE INDEX notification_prefs_pkey ON {schema}.notification_prefs USING btree (user_id)\"]", + "[\"index\", \"notification_prefs\", \"CREATE UNIQUE INDEX notification_prefs_unsub_token_key ON {schema}.notification_prefs USING btree (unsub_token)\"]", + "[\"index\", \"notification_state\", \"CREATE UNIQUE INDEX notification_state_pkey ON {schema}.notification_state USING btree (user_id, event, subject)\"]", + "[\"index\", \"notification_watches\", \"CREATE UNIQUE INDEX notification_watches_pkey ON {schema}.notification_watches USING btree (user_id, pubkey)\"]", + "[\"index\", \"proposals\", \"CREATE INDEX proposals_status ON {schema}.proposals USING btree (status)\"]", + "[\"index\", \"proposals\", \"CREATE UNIQUE INDEX proposals_kind_subject ON {schema}.proposals USING btree (kind, subject)\"]", + "[\"index\", \"proposals\", \"CREATE UNIQUE INDEX proposals_pkey ON {schema}.proposals USING btree (id)\"]", + "[\"index\", \"sessions\", \"CREATE INDEX sessions_user ON {schema}.sessions USING btree (user_id)\"]", + "[\"index\", \"sessions\", \"CREATE UNIQUE INDEX sessions_pkey ON {schema}.sessions USING btree (id)\"]", + "[\"index\", \"sessions\", \"CREATE UNIQUE INDEX sessions_token_hash_key ON {schema}.sessions USING btree (token_hash)\"]", + "[\"index\", \"tokens\", \"CREATE INDEX tokens_user ON {schema}.tokens USING btree (user_id, purpose)\"]", + "[\"index\", \"tokens\", \"CREATE UNIQUE INDEX tokens_pkey ON {schema}.tokens USING btree (token_hash)\"]", + "[\"index\", \"user_settings\", \"CREATE UNIQUE INDEX user_settings_pkey ON {schema}.user_settings USING btree (user_id)\"]", + "[\"index\", \"users\", \"CREATE UNIQUE INDEX users_email_key ON {schema}.users USING btree (email)\"]", + "[\"index\", \"users\", \"CREATE UNIQUE INDEX users_pkey ON {schema}.users USING btree (id)\"]", + "[\"table\", \"audit_log\", \"r\", false, false]", + "[\"table\", \"companion_links\", \"r\", false, false]", + "[\"table\", \"link_challenges\", \"r\", false, false]", + "[\"table\", \"mail_events\", \"r\", false, false]", + "[\"table\", \"mail_log\", \"r\", false, false]", + "[\"table\", \"notification_prefs\", \"r\", false, false]", + "[\"table\", \"notification_state\", \"r\", false, false]", + "[\"table\", \"notification_watches\", \"r\", false, false]", + "[\"table\", \"proposals\", \"r\", false, false]", + "[\"table\", \"sessions\", \"r\", false, false]", + "[\"table\", \"tokens\", \"r\", false, false]", + "[\"table\", \"user_settings\", \"r\", false, false]", + "[\"table\", \"users\", \"r\", false, false]", + "[\"view\", \"approved_channels\", \" SELECT kind,\\n subject,\\n decided_at,\\n id\\n FROM proposals\\n WHERE (status = 'approved'::text);\"]", + "[\"view\", \"linked_companions\", \" SELECT pubkey\\n FROM companion_links;\"]" + ], + "telemetry": [ + "[\"column\", \"_async_migrations\", 1, \"name\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"_async_migrations\", 2, \"status\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"_async_migrations\", 3, \"started_at\", \"text\", \"NO\", \"to_char((CURRENT_TIMESTAMP AT TIME ZONE 'UTC'::text), 'YYYY-MM-DD HH24:MI:SS'::text)\", \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"_async_migrations\", 4, \"ended_at\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"_async_migrations\", 5, \"error\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"_migrations\", 1, \"name\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"advert_evidence_backfill\", 1, \"id\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"advert_evidence_backfill\", 2, \"tx_cursor\", \"bigint\", \"NO\", \"0\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"advert_evidence_backfill\", 3, \"obs_cursor\", \"bigint\", \"NO\", \"0\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"advert_route_evidence\", 1, \"id\", \"bigint\", \"NO\", null, null, \"YES\", \"BY DEFAULT\", \"NEVER\", null]", + "[\"column\", \"advert_route_evidence\", 2, \"tx_id\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"advert_route_evidence\", 3, \"bit\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_observers\", 1, \"pubkey\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_observers\", 2, \"name\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_observers\", 3, \"last_seen\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_receptions\", 1, \"id\", \"bigint\", \"NO\", null, null, \"YES\", \"BY DEFAULT\", \"NEVER\", null]", + "[\"column\", \"client_receptions\", 10, \"rx_at\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_receptions\", 11, \"ingested_at\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_receptions\", 12, \"src\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_receptions\", 2, \"rx_pubkey\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_receptions\", 3, \"heard_key\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_receptions\", 4, \"heard_keylen\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_receptions\", 5, \"rssi\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_receptions\", 6, \"snr\", \"double precision\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_receptions\", 7, \"lat\", \"double precision\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_receptions\", 8, \"lon\", \"double precision\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_receptions\", 9, \"pos_acc_m\", \"double precision\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 1, \"id\", \"bigint\", \"NO\", null, null, \"YES\", \"BY DEFAULT\", \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 10, \"battery_mv\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 11, \"queue_len\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 12, \"errors\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 13, \"noise_floor\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 14, \"last_rssi\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 15, \"last_snr\", \"double precision\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 16, \"tx_air_secs\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 17, \"rx_air_secs\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 18, \"recv\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 19, \"sent\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 2, \"rx_pubkey\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 20, \"flood_rx\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 21, \"direct_rx\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 22, \"flood_tx\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 23, \"direct_tx\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 24, \"recv_errors\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 3, \"sampled_at\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 4, \"ingested_at\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 5, \"lat\", \"double precision\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 6, \"lon\", \"double precision\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 7, \"pos_acc_m\", \"double precision\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 8, \"stationary\", \"bigint\", \"NO\", \"0\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rf_samples\", 9, \"uptime_secs\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rx_observations\", 1, \"id\", \"bigint\", \"NO\", null, null, \"YES\", \"BY DEFAULT\", \"NEVER\", null]", + "[\"column\", \"client_rx_observations\", 10, \"scope_name\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rx_observations\", 11, \"hash_size\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rx_observations\", 12, \"hop_count\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rx_observations\", 13, \"path_json\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rx_observations\", 14, \"forwarder\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rx_observations\", 15, \"snr\", \"double precision\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rx_observations\", 16, \"rssi\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rx_observations\", 17, \"lat\", \"double precision\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rx_observations\", 18, \"lon\", \"double precision\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rx_observations\", 19, \"pos_acc_m\", \"double precision\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rx_observations\", 2, \"rx_pubkey\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rx_observations\", 3, \"rx_at\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rx_observations\", 4, \"ingested_at\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rx_observations\", 5, \"pkt_hash\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rx_observations\", 6, \"route_type\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rx_observations\", 7, \"payload_type\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rx_observations\", 8, \"code1\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"client_rx_observations\", 9, \"code2\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"dropped_packets\", 1, \"id\", \"bigint\", \"NO\", null, null, \"YES\", \"BY DEFAULT\", \"NEVER\", null]", + "[\"column\", \"dropped_packets\", 2, \"hash\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"dropped_packets\", 3, \"raw_hex\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"dropped_packets\", 4, \"reason\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"dropped_packets\", 5, \"observer_id\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"dropped_packets\", 6, \"observer_name\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"dropped_packets\", 7, \"node_pubkey\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"dropped_packets\", 8, \"node_name\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"dropped_packets\", 9, \"dropped_at\", \"text\", \"YES\", \"to_char((CURRENT_TIMESTAMP AT TIME ZONE 'UTC'::text), 'YYYY-MM-DD HH24:MI:SS'::text)\", \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"inactive_nodes\", 1, \"public_key\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"inactive_nodes\", 10, \"temperature_c\", \"double precision\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"inactive_nodes\", 11, \"foreign_advert\", \"bigint\", \"YES\", \"0\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"inactive_nodes\", 12, \"default_scope\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"inactive_nodes\", 13, \"configured_scope\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"inactive_nodes\", 14, \"configured_scope_at\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"inactive_nodes\", 15, \"multibyte_sup\", \"bigint\", \"NO\", \"0\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"inactive_nodes\", 16, \"multibyte_evidence\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"inactive_nodes\", 2, \"name\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"inactive_nodes\", 3, \"role\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"inactive_nodes\", 4, \"lat\", \"double precision\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"inactive_nodes\", 5, \"lon\", \"double precision\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"inactive_nodes\", 6, \"last_seen\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"inactive_nodes\", 7, \"first_seen\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"inactive_nodes\", 8, \"advert_count\", \"bigint\", \"YES\", \"0\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"inactive_nodes\", 9, \"battery_mv\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"key_backfill_done\", 1, \"kind\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"key_backfill_done\", 2, \"key_fp\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"key_backfill_done\", 3, \"done_at\", \"bigint\", \"NO\", \"(floor(EXTRACT(epoch FROM CURRENT_TIMESTAMP)))::bigint\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"key_backfill_state\", 1, \"kind\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"key_backfill_state\", 2, \"run_fp\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"key_backfill_state\", 3, \"cursor\", \"bigint\", \"NO\", \"0\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"key_backfill_state\", 4, \"upper\", \"bigint\", \"NO\", \"0\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"key_backfill_state\", 5, \"rewritten\", \"bigint\", \"NO\", \"0\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"key_backfill_state\", 6, \"started_at\", \"bigint\", \"NO\", \"(floor(EXTRACT(epoch FROM CURRENT_TIMESTAMP)))::bigint\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"neighbor_edges\", 1, \"node_a\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"neighbor_edges\", 2, \"node_b\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"neighbor_edges\", 3, \"count\", \"bigint\", \"YES\", \"1\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"neighbor_edges\", 4, \"last_seen\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"node_declared_regions\", 1, \"id\", \"bigint\", \"NO\", null, null, \"YES\", \"BY DEFAULT\", \"NEVER\", null]", + "[\"column\", \"node_declared_regions\", 10, \"pos_acc_m\", \"double precision\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"node_declared_regions\", 11, \"repeater_clock\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"node_declared_regions\", 2, \"target\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"node_declared_regions\", 3, \"rx_pubkey\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"node_declared_regions\", 4, \"observed_at\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"node_declared_regions\", 5, \"ingested_at\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"node_declared_regions\", 6, \"regions_csv\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"node_declared_regions\", 7, \"truncated\", \"bigint\", \"NO\", \"0\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"node_declared_regions\", 8, \"lat\", \"double precision\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"node_declared_regions\", 9, \"lon\", \"double precision\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"nodes\", 1, \"public_key\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"nodes\", 10, \"temperature_c\", \"double precision\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"nodes\", 11, \"foreign_advert\", \"bigint\", \"YES\", \"0\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"nodes\", 12, \"default_scope\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"nodes\", 13, \"configured_scope\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"nodes\", 14, \"configured_scope_at\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"nodes\", 15, \"multibyte_sup\", \"bigint\", \"NO\", \"0\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"nodes\", 16, \"multibyte_evidence\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"nodes\", 2, \"name\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"nodes\", 3, \"role\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"nodes\", 4, \"lat\", \"double precision\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"nodes\", 5, \"lon\", \"double precision\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"nodes\", 6, \"last_seen\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"nodes\", 7, \"first_seen\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"nodes\", 8, \"advert_count\", \"bigint\", \"YES\", \"0\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"nodes\", 9, \"battery_mv\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observations\", 1, \"id\", \"bigint\", \"NO\", null, null, \"YES\", \"BY DEFAULT\", \"NEVER\", null]", + "[\"column\", \"observations\", 10, \"raw_hex\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observations\", 11, \"resolved_path\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observations\", 2, \"transmission_id\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observations\", 3, \"observer_idx\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observations\", 4, \"direction\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observations\", 5, \"snr\", \"double precision\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observations\", 6, \"rssi\", \"double precision\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observations\", 7, \"score\", \"double precision\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observations\", 8, \"path_json\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observations\", 9, \"timestamp\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observer_metrics\", 1, \"observer_id\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observer_metrics\", 2, \"timestamp\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observer_metrics\", 3, \"noise_floor\", \"double precision\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observer_metrics\", 4, \"tx_air_secs\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observer_metrics\", 5, \"rx_air_secs\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observer_metrics\", 6, \"recv_errors\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observer_metrics\", 7, \"battery_mv\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observer_metrics\", 8, \"packets_sent\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observer_metrics\", 9, \"packets_recv\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observers\", 1, \"rowid\", \"bigint\", \"NO\", null, null, \"YES\", \"BY DEFAULT\", \"NEVER\", null]", + "[\"column\", \"observers\", 10, \"client_version\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observers\", 11, \"radio\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observers\", 12, \"battery_mv\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observers\", 13, \"uptime_secs\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observers\", 14, \"noise_floor\", \"double precision\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observers\", 15, \"inactive\", \"bigint\", \"YES\", \"0\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observers\", 16, \"last_packet_at\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observers\", 17, \"clock_skew_seconds\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observers\", 18, \"clock_skew_count_24h\", \"bigint\", \"YES\", \"0\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observers\", 19, \"clock_last_naive_at\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observers\", 2, \"id\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observers\", 20, \"can_relay\", \"bigint\", \"YES\", \"1\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observers\", 21, \"can_relay_seen\", \"bigint\", \"YES\", \"0\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observers\", 3, \"name\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observers\", 4, \"iata\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observers\", 5, \"last_seen\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observers\", 6, \"first_seen\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observers\", 7, \"packet_count\", \"bigint\", \"YES\", \"0\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observers\", 8, \"model\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"observers\", 9, \"firmware\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"scope_match_totals\", 1, \"id\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"scope_match_totals\", 2, \"since_unix\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"scope_match_totals\", 3, \"unique_matches\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"scope_match_totals\", 4, \"explicit_over_derived\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"scope_match_totals\", 5, \"ambiguous\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"scope_match_totals\", 6, \"none_matches\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"scope_match_totals\", 7, \"updated_unix\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"transmissions\", 1, \"id\", \"bigint\", \"NO\", null, null, \"YES\", \"BY DEFAULT\", \"NEVER\", null]", + "[\"column\", \"transmissions\", 10, \"last_seen\", \"bigint\", \"NO\", \"0\", null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"transmissions\", 11, \"created_at\", \"text\", \"YES\", \"to_char((CURRENT_TIMESTAMP AT TIME ZONE 'UTC'::text), 'YYYY-MM-DD HH24:MI:SS'::text)\", \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"transmissions\", 12, \"code1\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"transmissions\", 13, \"code2\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"transmissions\", 14, \"channel_hash\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"transmissions\", 15, \"scope_name\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"transmissions\", 2, \"raw_hex\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"transmissions\", 3, \"hash\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"transmissions\", 4, \"first_seen\", \"text\", \"NO\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"transmissions\", 5, \"route_type\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"transmissions\", 6, \"payload_type\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"transmissions\", 7, \"payload_version\", \"bigint\", \"YES\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"transmissions\", 8, \"decoded_json\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"transmissions\", 9, \"from_pubkey\", \"text\", \"YES\", null, \"C\", \"NO\", null, \"NEVER\", null]", + "[\"column\", \"tx_rewrite_feed\", 1, \"id\", \"bigint\", \"NO\", null, null, \"YES\", \"BY DEFAULT\", \"NEVER\", null]", + "[\"column\", \"tx_rewrite_feed\", 2, \"tx_id\", \"bigint\", \"NO\", null, null, \"NO\", null, \"NEVER\", null]", + "[\"column\", \"tx_rewrite_feed\", 3, \"created_at\", \"bigint\", \"NO\", \"(floor(EXTRACT(epoch FROM CURRENT_TIMESTAMP)))::bigint\", null, \"NO\", null, \"NEVER\", null]", + "[\"constraint\", \"_async_migrations\", \"_async_migrations_name_not_null\", true, \"NOT NULL name\"]", + "[\"constraint\", \"_async_migrations\", \"_async_migrations_pkey\", true, \"PRIMARY KEY (name)\"]", + "[\"constraint\", \"_async_migrations\", \"_async_migrations_started_at_not_null\", true, \"NOT NULL started_at\"]", + "[\"constraint\", \"_async_migrations\", \"_async_migrations_status_not_null\", true, \"NOT NULL status\"]", + "[\"constraint\", \"_migrations\", \"_migrations_name_not_null\", true, \"NOT NULL name\"]", + "[\"constraint\", \"_migrations\", \"_migrations_pkey\", true, \"PRIMARY KEY (name)\"]", + "[\"constraint\", \"advert_evidence_backfill\", \"advert_evidence_backfill_id_check\", true, \"CHECK ((id = 1))\"]", + "[\"constraint\", \"advert_evidence_backfill\", \"advert_evidence_backfill_id_not_null\", true, \"NOT NULL id\"]", + "[\"constraint\", \"advert_evidence_backfill\", \"advert_evidence_backfill_obs_cursor_not_null\", true, \"NOT NULL obs_cursor\"]", + "[\"constraint\", \"advert_evidence_backfill\", \"advert_evidence_backfill_pkey\", true, \"PRIMARY KEY (id)\"]", + "[\"constraint\", \"advert_evidence_backfill\", \"advert_evidence_backfill_tx_cursor_not_null\", true, \"NOT NULL tx_cursor\"]", + "[\"constraint\", \"advert_route_evidence\", \"advert_route_evidence_bit_check\", true, \"CHECK ((\\\"bit\\\" = ANY (ARRAY[(1)::bigint, (2)::bigint])))\"]", + "[\"constraint\", \"advert_route_evidence\", \"advert_route_evidence_bit_not_null\", true, \"NOT NULL \\\"bit\\\"\"]", + "[\"constraint\", \"advert_route_evidence\", \"advert_route_evidence_id_not_null\", true, \"NOT NULL id\"]", + "[\"constraint\", \"advert_route_evidence\", \"advert_route_evidence_pkey\", true, \"PRIMARY KEY (id)\"]", + "[\"constraint\", \"advert_route_evidence\", \"advert_route_evidence_tx_id_bit_key\", true, \"UNIQUE (tx_id, \\\"bit\\\")\"]", + "[\"constraint\", \"advert_route_evidence\", \"advert_route_evidence_tx_id_fkey\", true, \"FOREIGN KEY (tx_id) REFERENCES transmissions(id) ON DELETE CASCADE\"]", + "[\"constraint\", \"advert_route_evidence\", \"advert_route_evidence_tx_id_not_null\", true, \"NOT NULL tx_id\"]", + "[\"constraint\", \"client_observers\", \"client_observers_pkey\", true, \"PRIMARY KEY (pubkey)\"]", + "[\"constraint\", \"client_observers\", \"client_observers_pubkey_not_null\", true, \"NOT NULL pubkey\"]", + "[\"constraint\", \"client_receptions\", \"client_receptions_heard_key_not_null\", true, \"NOT NULL heard_key\"]", + "[\"constraint\", \"client_receptions\", \"client_receptions_heard_keylen_not_null\", true, \"NOT NULL heard_keylen\"]", + "[\"constraint\", \"client_receptions\", \"client_receptions_id_not_null\", true, \"NOT NULL id\"]", + "[\"constraint\", \"client_receptions\", \"client_receptions_ingested_at_not_null\", true, \"NOT NULL ingested_at\"]", + "[\"constraint\", \"client_receptions\", \"client_receptions_lat_not_null\", true, \"NOT NULL lat\"]", + "[\"constraint\", \"client_receptions\", \"client_receptions_lon_not_null\", true, \"NOT NULL lon\"]", + "[\"constraint\", \"client_receptions\", \"client_receptions_pkey\", true, \"PRIMARY KEY (id)\"]", + "[\"constraint\", \"client_receptions\", \"client_receptions_rx_at_not_null\", true, \"NOT NULL rx_at\"]", + "[\"constraint\", \"client_receptions\", \"client_receptions_rx_pubkey_heard_key_rx_at_key\", true, \"UNIQUE (rx_pubkey, heard_key, rx_at)\"]", + "[\"constraint\", \"client_receptions\", \"client_receptions_rx_pubkey_not_null\", true, \"NOT NULL rx_pubkey\"]", + "[\"constraint\", \"client_receptions\", \"client_receptions_src_not_null\", true, \"NOT NULL src\"]", + "[\"constraint\", \"client_rf_samples\", \"client_rf_samples_id_not_null\", true, \"NOT NULL id\"]", + "[\"constraint\", \"client_rf_samples\", \"client_rf_samples_ingested_at_not_null\", true, \"NOT NULL ingested_at\"]", + "[\"constraint\", \"client_rf_samples\", \"client_rf_samples_lat_not_null\", true, \"NOT NULL lat\"]", + "[\"constraint\", \"client_rf_samples\", \"client_rf_samples_lon_not_null\", true, \"NOT NULL lon\"]", + "[\"constraint\", \"client_rf_samples\", \"client_rf_samples_pkey\", true, \"PRIMARY KEY (id)\"]", + "[\"constraint\", \"client_rf_samples\", \"client_rf_samples_rx_pubkey_not_null\", true, \"NOT NULL rx_pubkey\"]", + "[\"constraint\", \"client_rf_samples\", \"client_rf_samples_rx_pubkey_sampled_at_key\", true, \"UNIQUE (rx_pubkey, sampled_at)\"]", + "[\"constraint\", \"client_rf_samples\", \"client_rf_samples_sampled_at_not_null\", true, \"NOT NULL sampled_at\"]", + "[\"constraint\", \"client_rf_samples\", \"client_rf_samples_stationary_not_null\", true, \"NOT NULL stationary\"]", + "[\"constraint\", \"client_rf_samples\", \"client_rf_samples_uptime_secs_not_null\", true, \"NOT NULL uptime_secs\"]", + "[\"constraint\", \"client_rx_observations\", \"client_rx_observations_hash_size_not_null\", true, \"NOT NULL hash_size\"]", + "[\"constraint\", \"client_rx_observations\", \"client_rx_observations_hop_count_not_null\", true, \"NOT NULL hop_count\"]", + "[\"constraint\", \"client_rx_observations\", \"client_rx_observations_id_not_null\", true, \"NOT NULL id\"]", + "[\"constraint\", \"client_rx_observations\", \"client_rx_observations_ingested_at_not_null\", true, \"NOT NULL ingested_at\"]", + "[\"constraint\", \"client_rx_observations\", \"client_rx_observations_lat_not_null\", true, \"NOT NULL lat\"]", + "[\"constraint\", \"client_rx_observations\", \"client_rx_observations_lon_not_null\", true, \"NOT NULL lon\"]", + "[\"constraint\", \"client_rx_observations\", \"client_rx_observations_payload_type_not_null\", true, \"NOT NULL payload_type\"]", + "[\"constraint\", \"client_rx_observations\", \"client_rx_observations_pkey\", true, \"PRIMARY KEY (id)\"]", + "[\"constraint\", \"client_rx_observations\", \"client_rx_observations_pkt_hash_not_null\", true, \"NOT NULL pkt_hash\"]", + "[\"constraint\", \"client_rx_observations\", \"client_rx_observations_route_type_not_null\", true, \"NOT NULL route_type\"]", + "[\"constraint\", \"client_rx_observations\", \"client_rx_observations_rx_at_not_null\", true, \"NOT NULL rx_at\"]", + "[\"constraint\", \"client_rx_observations\", \"client_rx_observations_rx_pubkey_not_null\", true, \"NOT NULL rx_pubkey\"]", + "[\"constraint\", \"client_rx_observations\", \"client_rx_observations_rx_pubkey_pkt_hash_rx_at_key\", true, \"UNIQUE (rx_pubkey, pkt_hash, rx_at)\"]", + "[\"constraint\", \"dropped_packets\", \"dropped_packets_id_not_null\", true, \"NOT NULL id\"]", + "[\"constraint\", \"dropped_packets\", \"dropped_packets_pkey\", true, \"PRIMARY KEY (id)\"]", + "[\"constraint\", \"dropped_packets\", \"dropped_packets_reason_not_null\", true, \"NOT NULL reason\"]", + "[\"constraint\", \"inactive_nodes\", \"inactive_nodes_multibyte_sup_not_null\", true, \"NOT NULL multibyte_sup\"]", + "[\"constraint\", \"inactive_nodes\", \"inactive_nodes_pkey\", true, \"PRIMARY KEY (public_key)\"]", + "[\"constraint\", \"inactive_nodes\", \"inactive_nodes_public_key_not_null\", true, \"NOT NULL public_key\"]", + "[\"constraint\", \"key_backfill_done\", \"key_backfill_done_done_at_not_null\", true, \"NOT NULL done_at\"]", + "[\"constraint\", \"key_backfill_done\", \"key_backfill_done_key_fp_not_null\", true, \"NOT NULL key_fp\"]", + "[\"constraint\", \"key_backfill_done\", \"key_backfill_done_kind_not_null\", true, \"NOT NULL kind\"]", + "[\"constraint\", \"key_backfill_done\", \"key_backfill_done_pkey\", true, \"PRIMARY KEY (kind, key_fp)\"]", + "[\"constraint\", \"key_backfill_state\", \"key_backfill_state_cursor_not_null\", true, \"NOT NULL cursor\"]", + "[\"constraint\", \"key_backfill_state\", \"key_backfill_state_kind_check\", true, \"CHECK ((kind = ANY (ARRAY['channel'::text, 'region'::text])))\"]", + "[\"constraint\", \"key_backfill_state\", \"key_backfill_state_kind_not_null\", true, \"NOT NULL kind\"]", + "[\"constraint\", \"key_backfill_state\", \"key_backfill_state_pkey\", true, \"PRIMARY KEY (kind)\"]", + "[\"constraint\", \"key_backfill_state\", \"key_backfill_state_rewritten_not_null\", true, \"NOT NULL rewritten\"]", + "[\"constraint\", \"key_backfill_state\", \"key_backfill_state_run_fp_not_null\", true, \"NOT NULL run_fp\"]", + "[\"constraint\", \"key_backfill_state\", \"key_backfill_state_started_at_not_null\", true, \"NOT NULL started_at\"]", + "[\"constraint\", \"key_backfill_state\", \"key_backfill_state_upper_not_null\", true, \"NOT NULL upper\"]", + "[\"constraint\", \"neighbor_edges\", \"neighbor_edges_node_a_not_null\", true, \"NOT NULL node_a\"]", + "[\"constraint\", \"neighbor_edges\", \"neighbor_edges_node_b_not_null\", true, \"NOT NULL node_b\"]", + "[\"constraint\", \"neighbor_edges\", \"neighbor_edges_pkey\", true, \"PRIMARY KEY (node_a, node_b)\"]", + "[\"constraint\", \"node_declared_regions\", \"node_declared_regions_id_not_null\", true, \"NOT NULL id\"]", + "[\"constraint\", \"node_declared_regions\", \"node_declared_regions_ingested_at_not_null\", true, \"NOT NULL ingested_at\"]", + "[\"constraint\", \"node_declared_regions\", \"node_declared_regions_observed_at_not_null\", true, \"NOT NULL observed_at\"]", + "[\"constraint\", \"node_declared_regions\", \"node_declared_regions_pkey\", true, \"PRIMARY KEY (id)\"]", + "[\"constraint\", \"node_declared_regions\", \"node_declared_regions_regions_csv_not_null\", true, \"NOT NULL regions_csv\"]", + "[\"constraint\", \"node_declared_regions\", \"node_declared_regions_rx_pubkey_not_null\", true, \"NOT NULL rx_pubkey\"]", + "[\"constraint\", \"node_declared_regions\", \"node_declared_regions_target_not_null\", true, \"NOT NULL target\"]", + "[\"constraint\", \"node_declared_regions\", \"node_declared_regions_target_rx_pubkey_observed_at_key\", true, \"UNIQUE (target, rx_pubkey, observed_at)\"]", + "[\"constraint\", \"node_declared_regions\", \"node_declared_regions_truncated_not_null\", true, \"NOT NULL truncated\"]", + "[\"constraint\", \"nodes\", \"nodes_multibyte_sup_not_null\", true, \"NOT NULL multibyte_sup\"]", + "[\"constraint\", \"nodes\", \"nodes_pkey\", true, \"PRIMARY KEY (public_key)\"]", + "[\"constraint\", \"nodes\", \"nodes_public_key_not_null\", true, \"NOT NULL public_key\"]", + "[\"constraint\", \"observations\", \"observations_id_not_null\", true, \"NOT NULL id\"]", + "[\"constraint\", \"observations\", \"observations_pkey\", true, \"PRIMARY KEY (id)\"]", + "[\"constraint\", \"observations\", \"observations_timestamp_not_null\", true, \"NOT NULL \\\"timestamp\\\"\"]", + "[\"constraint\", \"observations\", \"observations_transmission_id_fkey\", true, \"FOREIGN KEY (transmission_id) REFERENCES transmissions(id)\"]", + "[\"constraint\", \"observations\", \"observations_transmission_id_not_null\", true, \"NOT NULL transmission_id\"]", + "[\"constraint\", \"observer_metrics\", \"observer_metrics_observer_id_not_null\", true, \"NOT NULL observer_id\"]", + "[\"constraint\", \"observer_metrics\", \"observer_metrics_pkey\", true, \"PRIMARY KEY (observer_id, \\\"timestamp\\\")\"]", + "[\"constraint\", \"observer_metrics\", \"observer_metrics_timestamp_not_null\", true, \"NOT NULL \\\"timestamp\\\"\"]", + "[\"constraint\", \"observers\", \"observers_id_not_null\", true, \"NOT NULL id\"]", + "[\"constraint\", \"observers\", \"observers_pkey\", true, \"PRIMARY KEY (id)\"]", + "[\"constraint\", \"observers\", \"observers_rowid_key\", true, \"UNIQUE (rowid)\"]", + "[\"constraint\", \"observers\", \"observers_rowid_not_null\", true, \"NOT NULL rowid\"]", + "[\"constraint\", \"scope_match_totals\", \"scope_match_totals_ambiguous_not_null\", true, \"NOT NULL ambiguous\"]", + "[\"constraint\", \"scope_match_totals\", \"scope_match_totals_explicit_over_derived_not_null\", true, \"NOT NULL explicit_over_derived\"]", + "[\"constraint\", \"scope_match_totals\", \"scope_match_totals_id_check\", true, \"CHECK ((id = 1))\"]", + "[\"constraint\", \"scope_match_totals\", \"scope_match_totals_id_not_null\", true, \"NOT NULL id\"]", + "[\"constraint\", \"scope_match_totals\", \"scope_match_totals_none_matches_not_null\", true, \"NOT NULL none_matches\"]", + "[\"constraint\", \"scope_match_totals\", \"scope_match_totals_pkey\", true, \"PRIMARY KEY (id)\"]", + "[\"constraint\", \"scope_match_totals\", \"scope_match_totals_since_unix_not_null\", true, \"NOT NULL since_unix\"]", + "[\"constraint\", \"scope_match_totals\", \"scope_match_totals_unique_matches_not_null\", true, \"NOT NULL unique_matches\"]", + "[\"constraint\", \"scope_match_totals\", \"scope_match_totals_updated_unix_not_null\", true, \"NOT NULL updated_unix\"]", + "[\"constraint\", \"transmissions\", \"transmissions_first_seen_not_null\", true, \"NOT NULL first_seen\"]", + "[\"constraint\", \"transmissions\", \"transmissions_hash_key\", true, \"UNIQUE (hash)\"]", + "[\"constraint\", \"transmissions\", \"transmissions_hash_not_null\", true, \"NOT NULL hash\"]", + "[\"constraint\", \"transmissions\", \"transmissions_id_not_null\", true, \"NOT NULL id\"]", + "[\"constraint\", \"transmissions\", \"transmissions_last_seen_not_null\", true, \"NOT NULL last_seen\"]", + "[\"constraint\", \"transmissions\", \"transmissions_pkey\", true, \"PRIMARY KEY (id)\"]", + "[\"constraint\", \"transmissions\", \"transmissions_raw_hex_not_null\", true, \"NOT NULL raw_hex\"]", + "[\"constraint\", \"tx_rewrite_feed\", \"tx_rewrite_feed_created_at_not_null\", true, \"NOT NULL created_at\"]", + "[\"constraint\", \"tx_rewrite_feed\", \"tx_rewrite_feed_id_not_null\", true, \"NOT NULL id\"]", + "[\"constraint\", \"tx_rewrite_feed\", \"tx_rewrite_feed_pkey\", true, \"PRIMARY KEY (id)\"]", + "[\"constraint\", \"tx_rewrite_feed\", \"tx_rewrite_feed_tx_id_fkey\", true, \"FOREIGN KEY (tx_id) REFERENCES transmissions(id) ON DELETE CASCADE\"]", + "[\"constraint\", \"tx_rewrite_feed\", \"tx_rewrite_feed_tx_id_not_null\", true, \"NOT NULL tx_id\"]", + "[\"index\", \"_async_migrations\", \"CREATE UNIQUE INDEX _async_migrations_pkey ON {schema}._async_migrations USING btree (name)\"]", + "[\"index\", \"_migrations\", \"CREATE UNIQUE INDEX _migrations_pkey ON {schema}._migrations USING btree (name)\"]", + "[\"index\", \"advert_evidence_backfill\", \"CREATE UNIQUE INDEX advert_evidence_backfill_pkey ON {schema}.advert_evidence_backfill USING btree (id)\"]", + "[\"index\", \"advert_route_evidence\", \"CREATE UNIQUE INDEX advert_route_evidence_pkey ON {schema}.advert_route_evidence USING btree (id)\"]", + "[\"index\", \"advert_route_evidence\", \"CREATE UNIQUE INDEX advert_route_evidence_tx_id_bit_key ON {schema}.advert_route_evidence USING btree (tx_id, \\\"bit\\\")\"]", + "[\"index\", \"client_observers\", \"CREATE UNIQUE INDEX client_observers_pkey ON {schema}.client_observers USING btree (pubkey)\"]", + "[\"index\", \"client_receptions\", \"CREATE INDEX idx_client_recept_heard_geo ON {schema}.client_receptions USING btree (heard_key, heard_keylen, lat, lon)\"]", + "[\"index\", \"client_receptions\", \"CREATE INDEX idx_client_recept_latlon ON {schema}.client_receptions USING btree (lat, lon)\"]", + "[\"index\", \"client_receptions\", \"CREATE INDEX idx_client_recept_rxat ON {schema}.client_receptions USING btree (rx_at)\"]", + "[\"index\", \"client_receptions\", \"CREATE UNIQUE INDEX client_receptions_pkey ON {schema}.client_receptions USING btree (id)\"]", + "[\"index\", \"client_receptions\", \"CREATE UNIQUE INDEX client_receptions_rx_pubkey_heard_key_rx_at_key ON {schema}.client_receptions USING btree (rx_pubkey, heard_key, rx_at)\"]", + "[\"index\", \"client_rf_samples\", \"CREATE INDEX idx_crf_prune ON {schema}.client_rf_samples USING btree (sampled_at)\"]", + "[\"index\", \"client_rf_samples\", \"CREATE INDEX idx_crf_track ON {schema}.client_rf_samples USING btree (rx_pubkey, sampled_at)\"]", + "[\"index\", \"client_rf_samples\", \"CREATE UNIQUE INDEX client_rf_samples_pkey ON {schema}.client_rf_samples USING btree (id)\"]", + "[\"index\", \"client_rf_samples\", \"CREATE UNIQUE INDEX client_rf_samples_rx_pubkey_sampled_at_key ON {schema}.client_rf_samples USING btree (rx_pubkey, sampled_at)\"]", + "[\"index\", \"client_rx_observations\", \"CREATE INDEX idx_cro_forwarder ON {schema}.client_rx_observations USING btree (forwarder, rx_at)\"]", + "[\"index\", \"client_rx_observations\", \"CREATE INDEX idx_cro_hash ON {schema}.client_rx_observations USING btree (pkt_hash, rx_at)\"]", + "[\"index\", \"client_rx_observations\", \"CREATE INDEX idx_cro_prune ON {schema}.client_rx_observations USING btree (rx_at)\"]", + "[\"index\", \"client_rx_observations\", \"CREATE INDEX idx_cro_scope ON {schema}.client_rx_observations USING btree (scope_name, rx_at)\"]", + "[\"index\", \"client_rx_observations\", \"CREATE UNIQUE INDEX client_rx_observations_pkey ON {schema}.client_rx_observations USING btree (id)\"]", + "[\"index\", \"client_rx_observations\", \"CREATE UNIQUE INDEX client_rx_observations_rx_pubkey_pkt_hash_rx_at_key ON {schema}.client_rx_observations USING btree (rx_pubkey, pkt_hash, rx_at)\"]", + "[\"index\", \"dropped_packets\", \"CREATE INDEX idx_dropped_node ON {schema}.dropped_packets USING btree (node_pubkey)\"]", + "[\"index\", \"dropped_packets\", \"CREATE INDEX idx_dropped_observer ON {schema}.dropped_packets USING btree (observer_id)\"]", + "[\"index\", \"dropped_packets\", \"CREATE UNIQUE INDEX dropped_packets_pkey ON {schema}.dropped_packets USING btree (id)\"]", + "[\"index\", \"inactive_nodes\", \"CREATE INDEX idx_inactive_nodes_last_seen ON {schema}.inactive_nodes USING btree (last_seen)\"]", + "[\"index\", \"inactive_nodes\", \"CREATE UNIQUE INDEX inactive_nodes_pkey ON {schema}.inactive_nodes USING btree (public_key)\"]", + "[\"index\", \"key_backfill_done\", \"CREATE UNIQUE INDEX key_backfill_done_pkey ON {schema}.key_backfill_done USING btree (kind, key_fp)\"]", + "[\"index\", \"key_backfill_state\", \"CREATE UNIQUE INDEX key_backfill_state_pkey ON {schema}.key_backfill_state USING btree (kind)\"]", + "[\"index\", \"neighbor_edges\", \"CREATE UNIQUE INDEX neighbor_edges_pkey ON {schema}.neighbor_edges USING btree (node_a, node_b)\"]", + "[\"index\", \"node_declared_regions\", \"CREATE INDEX idx_ndr_prune ON {schema}.node_declared_regions USING btree (observed_at)\"]", + "[\"index\", \"node_declared_regions\", \"CREATE INDEX idx_ndr_target ON {schema}.node_declared_regions USING btree (target, observed_at)\"]", + "[\"index\", \"node_declared_regions\", \"CREATE UNIQUE INDEX node_declared_regions_pkey ON {schema}.node_declared_regions USING btree (id)\"]", + "[\"index\", \"node_declared_regions\", \"CREATE UNIQUE INDEX node_declared_regions_target_rx_pubkey_observed_at_key ON {schema}.node_declared_regions USING btree (target, rx_pubkey, observed_at)\"]", + "[\"index\", \"nodes\", \"CREATE INDEX idx_nodes_foreign_advert ON {schema}.nodes USING btree (foreign_advert) WHERE (foreign_advert = 1)\"]", + "[\"index\", \"nodes\", \"CREATE INDEX idx_nodes_last_seen ON {schema}.nodes USING btree (last_seen)\"]", + "[\"index\", \"nodes\", \"CREATE UNIQUE INDEX nodes_pkey ON {schema}.nodes USING btree (public_key)\"]", + "[\"index\", \"observations\", \"CREATE INDEX idx_observations_observer_idx ON {schema}.observations USING btree (observer_idx)\"]", + "[\"index\", \"observations\", \"CREATE INDEX idx_observations_observer_idx_timestamp ON {schema}.observations USING btree (observer_idx, \\\"timestamp\\\")\"]", + "[\"index\", \"observations\", \"CREATE INDEX idx_observations_timestamp ON {schema}.observations USING btree (\\\"timestamp\\\")\"]", + "[\"index\", \"observations\", \"CREATE INDEX idx_observations_transmission_id ON {schema}.observations USING btree (transmission_id)\"]", + "[\"index\", \"observations\", \"CREATE INDEX idx_observations_tx_ts ON {schema}.observations USING btree (transmission_id, \\\"timestamp\\\")\"]", + "[\"index\", \"observations\", \"CREATE UNIQUE INDEX idx_observations_dedup ON {schema}.observations USING btree (transmission_id, observer_idx, COALESCE(path_json, ''::text))\"]", + "[\"index\", \"observations\", \"CREATE UNIQUE INDEX observations_pkey ON {schema}.observations USING btree (id)\"]", + "[\"index\", \"observer_metrics\", \"CREATE INDEX idx_observer_metrics_timestamp ON {schema}.observer_metrics USING btree (\\\"timestamp\\\")\"]", + "[\"index\", \"observer_metrics\", \"CREATE UNIQUE INDEX observer_metrics_pkey ON {schema}.observer_metrics USING btree (observer_id, \\\"timestamp\\\")\"]", + "[\"index\", \"observers\", \"CREATE INDEX idx_observers_iata_norm ON {schema}.observers USING btree (upper(TRIM(BOTH FROM iata)))\"]", + "[\"index\", \"observers\", \"CREATE INDEX idx_observers_last_seen ON {schema}.observers USING btree (last_seen)\"]", + "[\"index\", \"observers\", \"CREATE UNIQUE INDEX observers_pkey ON {schema}.observers USING btree (id)\"]", + "[\"index\", \"observers\", \"CREATE UNIQUE INDEX observers_rowid_key ON {schema}.observers USING btree (rowid)\"]", + "[\"index\", \"scope_match_totals\", \"CREATE UNIQUE INDEX scope_match_totals_pkey ON {schema}.scope_match_totals USING btree (id)\"]", + "[\"index\", \"transmissions\", \"CREATE INDEX idx_transmissions_channel_hash_payload ON {schema}.transmissions USING btree (channel_hash, payload_type, first_seen)\"]", + "[\"index\", \"transmissions\", \"CREATE INDEX idx_transmissions_first_seen ON {schema}.transmissions USING btree (first_seen, id)\"]", + "[\"index\", \"transmissions\", \"CREATE INDEX idx_transmissions_from_pubkey ON {schema}.transmissions USING btree (from_pubkey)\"]", + "[\"index\", \"transmissions\", \"CREATE INDEX idx_transmissions_hash ON {schema}.transmissions USING btree (hash)\"]", + "[\"index\", \"transmissions\", \"CREATE INDEX idx_transmissions_payload_type ON {schema}.transmissions USING btree (payload_type)\"]", + "[\"index\", \"transmissions\", \"CREATE INDEX idx_tx_channel_hash ON {schema}.transmissions USING btree (channel_hash) WHERE (payload_type = 5)\"]", + "[\"index\", \"transmissions\", \"CREATE INDEX idx_tx_code1 ON {schema}.transmissions USING btree (code1) WHERE (code1 IS NOT NULL)\"]", + "[\"index\", \"transmissions\", \"CREATE INDEX idx_tx_last_seen_zero ON {schema}.transmissions USING btree (id) WHERE (last_seen = 0)\"]", + "[\"index\", \"transmissions\", \"CREATE INDEX idx_tx_scope_name ON {schema}.transmissions USING btree (scope_name) WHERE (scope_name IS NOT NULL)\"]", + "[\"index\", \"transmissions\", \"CREATE UNIQUE INDEX transmissions_hash_key ON {schema}.transmissions USING btree (hash)\"]", + "[\"index\", \"transmissions\", \"CREATE UNIQUE INDEX transmissions_pkey ON {schema}.transmissions USING btree (id)\"]", + "[\"index\", \"tx_rewrite_feed\", \"CREATE INDEX idx_tx_rewrite_feed_created ON {schema}.tx_rewrite_feed USING btree (created_at)\"]", + "[\"index\", \"tx_rewrite_feed\", \"CREATE INDEX idx_tx_rewrite_feed_tx ON {schema}.tx_rewrite_feed USING btree (tx_id)\"]", + "[\"index\", \"tx_rewrite_feed\", \"CREATE UNIQUE INDEX tx_rewrite_feed_pkey ON {schema}.tx_rewrite_feed USING btree (id)\"]", + "[\"table\", \"_async_migrations\", \"r\", false, false]", + "[\"table\", \"_migrations\", \"r\", false, false]", + "[\"table\", \"advert_evidence_backfill\", \"r\", false, false]", + "[\"table\", \"advert_route_evidence\", \"r\", false, false]", + "[\"table\", \"client_observers\", \"r\", false, false]", + "[\"table\", \"client_receptions\", \"r\", false, false]", + "[\"table\", \"client_rf_samples\", \"r\", false, false]", + "[\"table\", \"client_rx_observations\", \"r\", false, false]", + "[\"table\", \"dropped_packets\", \"r\", false, false]", + "[\"table\", \"inactive_nodes\", \"r\", false, false]", + "[\"table\", \"key_backfill_done\", \"r\", false, false]", + "[\"table\", \"key_backfill_state\", \"r\", false, false]", + "[\"table\", \"neighbor_edges\", \"r\", false, false]", + "[\"table\", \"node_declared_regions\", \"r\", false, false]", + "[\"table\", \"nodes\", \"r\", false, false]", + "[\"table\", \"observations\", \"r\", false, false]", + "[\"table\", \"observer_metrics\", \"r\", false, false]", + "[\"table\", \"observers\", \"r\", false, false]", + "[\"table\", \"scope_match_totals\", \"r\", false, false]", + "[\"table\", \"transmissions\", \"r\", false, false]", + "[\"table\", \"tx_rewrite_feed\", \"r\", false, false]", + "[\"view\", \"packets_v\", \" SELECT o.id,\\n COALESCE(o.raw_hex, t.raw_hex) AS raw_hex,\\n to_char((to_timestamp((o.\\\"timestamp\\\")::double precision) AT TIME ZONE 'UTC'::text), 'YYYY-MM-DD HH24:MI:SS'::text) AS \\\"timestamp\\\",\\n obs.id AS observer_id,\\n obs.name AS observer_name,\\n o.direction,\\n o.snr,\\n o.rssi,\\n o.score,\\n t.hash,\\n t.route_type,\\n t.payload_type,\\n t.payload_version,\\n o.path_json,\\n t.decoded_json,\\n t.created_at\\n FROM ((observations o\\n JOIN transmissions t ON ((t.id = o.transmission_id)))\\n LEFT JOIN observers obs ON (((obs.rowid = o.observer_idx) AND ((obs.inactive IS NULL) OR (obs.inactive = 0)))));\"]" + ] +} diff --git a/cmd/migrate/postgres_layout_test.go b/cmd/migrate/postgres_layout_test.go new file mode 100644 index 000000000..ebb536765 --- /dev/null +++ b/cmd/migrate/postgres_layout_test.go @@ -0,0 +1,83 @@ +package main + +import ( + "encoding/json" + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/users" + "os" + "testing" +) + +func TestPostgresConversionLayoutMatchesNativeSchema(t *testing.T) { + for _, kind := range []string{"telemetry", "accounts"} { + t.Run(kind, func(t *testing.T) { + db := openImportDB(t, postgresSchema(t)) + var err error + if kind == "telemetry" { + err = dbschema.ApplyPostgres(db, nil) + } else { + err = users.ApplyPostgres(db) + } + if err != nil { + t.Fatal(err) + } + tables, err := layouts(kind) + if err != nil { + t.Fatal(err) + } + if err := validatePostgresLayout(db, kind, tables); err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`CREATE INDEX unexpected_extension ON ` + quote(tables[0].Name) + ` (` + quote(tables[0].Columns[0].Name) + `)`); err != nil { + t.Fatal(err) + } + if err := validatePostgresLayout(db, kind, tables); err == nil { + t.Fatal("custom source extension would be discarded") + } + }) + } +} + +// This explicit developer mode exports a prospective layout for review. It +// never changes the compiled manifest or skips its normal compatibility test. +func TestExportPostgresConversionLayout(t *testing.T) { + path := os.Getenv("CORESCOPE_EXPORT_POSTGRES_LAYOUT") + if path == "" { + t.Skip("explicit private layout export only") + } + output := map[string][]string{} + for _, kind := range []string{"telemetry", "accounts"} { + db := openImportDB(t, postgresSchema(t)) + var err error + if kind == "telemetry" { + err = dbschema.ApplyPostgres(db, nil) + } else { + err = users.ApplyPostgres(db) + } + if err != nil { + t.Fatal(err) + } + tables, err := layouts(kind) + if err != nil { + t.Fatal(err) + } + if output[kind], err = postgresLayout(db, kind, tables); err != nil { + t.Fatal(err) + } + } + data, err := json.MarshalIndent(output, "", " ") + if err != nil { + t.Fatal(err) + } + file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + if err != nil { + t.Fatal(err) + } + if _, err = file.Write(append(data, '\n')); err != nil { + file.Close() + t.Fatal(err) + } + if err := file.Close(); err != nil { + t.Fatal(err) + } +} diff --git a/cmd/migrate/recorded_upgrade.go b/cmd/migrate/recorded_upgrade.go new file mode 100644 index 000000000..dec7f9751 --- /dev/null +++ b/cmd/migrate/recorded_upgrade.go @@ -0,0 +1,191 @@ +package main + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "log" + "slices" + + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/pgutil" + "github.com/meshcore-analyzer/users" +) + +// A recorded installation may only upgrade its known existing schemas. Unlike +// bootstrap, this path never initializes an empty replacement or adds accounts. +// Each database upgrade is atomic; across databases an interrupted run is +// retried while services remain stopped, with the selection bytes unchanged. +func upgradeRecordedPostgres(ctx context.Context, o storageOptions, expected dbconfig.Selection, raw dbconfig.StorageInputs) error { + selected, lease, err := dbconfig.OpenSelectionForMaintenance(o.SelectionFile) + if err != nil { + return err + } + defer lease.Close() + if selected.Generation != expected.Generation || selected.Backend != dbconfig.Postgres { + return dbconfig.ErrSelectionChanged + } + if o.OwnerURL == "" || (selected.Accounts != nil && o.UsersOwnerURL == "") { + return errors.New("recorded PostgreSQL upgrade requires owner credentials for every initialized store, including disabled accounts") + } + // Check the supplied endpoints before ApplyTo pins selected schema defaults; + // a credential URL naming a different schema must not authorize its upgrade. + if err := validateInitialOwnerTargets(ctx, selected, dbconfig.Storage{WriterDatabaseURL: o.OwnerURL, UsersDatabaseURL: o.UsersOwnerURL}); err != nil { + return err + } + owners, err := bindOwnerURLs(selected, o.OwnerURL, o.UsersOwnerURL) + if err != nil { + return err + } + type storeUpgrade struct { + kind string + db *sql.DB + needed bool + } + var plans []storeUpgrade + defer func() { + for _, plan := range plans { + plan.db.Close() + } + }() + stores := []struct{ kind, url string }{{"telemetry", owners.WriterDatabaseURL}} + if selected.Accounts != nil { + stores = append(stores, struct{ kind, url string }{"accounts", owners.UsersDatabaseURL}) + } + // Finish all owner, readiness, version, and physical-layout preflights + // before mutating either store. Unknown extensions are never discarded. + for _, store := range stores { + db, err := pgutil.Open(store.url, false) + if err != nil { + return err + } + plan := storeUpgrade{kind: store.kind, db: db} + plans = append(plans, plan) + needed, err := validateRecordedPostgresUpgrade(ctx, db, store.kind) + if err != nil { + return err + } + plans[len(plans)-1].needed = needed + } + for _, plan := range plans { + if !plan.needed { + continue + } + previous, current := recordedPostgresVersions(plan.kind) + log.Printf("[storage] PostgreSQL %s schema upgrade required: %d -> %d", plan.kind, previous, current) + if plan.kind == "telemetry" { + err = dbschema.ApplyPostgres(plan.db, nil) + } else { + err = users.ApplyPostgres(plan.db) + } + if err != nil { + return err + } + log.Printf("[storage] upgraded recorded PostgreSQL %s schema %d -> %d", plan.kind, previous, current) + } + if err := grantRuntimeTargets(ctx, selected, owners, raw); err != nil { + return err + } + return validateRuntimeSelection(ctx, selected, raw) +} + +func validateRecordedPostgresUpgrade(ctx context.Context, db *sql.DB, kind string) (bool, error) { + var owner bool + if err := db.QueryRowContext(ctx, `SELECT pg_has_role(current_user,n.nspowner,'MEMBER') + AND NOT EXISTS(SELECT 1 FROM pg_class c WHERE c.relnamespace=n.oid AND c.relkind IN ('r','p','v','m','S','f') AND NOT pg_has_role(current_user,c.relowner,'MEMBER')) + FROM pg_namespace n WHERE n.nspname=current_schema()`).Scan(&owner); err != nil || !owner { + return false, errors.New("recorded PostgreSQL upgrade requires the selected schema and object owner credentials") + } + var count, version int + var recordedKind string + var ready bool + if err := db.QueryRowContext(ctx, `SELECT count(*),min(kind),min(version),bool_and(ready) FROM corescope_schema`).Scan(&count, &recordedKind, &version, &ready); err != nil || count != 1 || recordedKind != kind || !ready { + return false, errors.New("recorded PostgreSQL store is missing, unsupported or incomplete; it was not initialized") + } + previous, current := recordedPostgresVersions(kind) + if kind == "accounts" { + if version != users.PostgresSchemaVersion { + return false, errors.New("unsupported PostgreSQL account readiness version") + } + if err := db.QueryRowContext(ctx, `SELECT count(*),coalesce(max(version),0) FROM schema_version`).Scan(&count, &version); err != nil || count != 1 { + return false, errors.New("recorded PostgreSQL account schema is incomplete") + } + } + if version != previous && version != current { + return false, errors.New("unsupported PostgreSQL schema version; upgrade was not attempted") + } + var dropped bool + if err := db.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM pg_attribute a JOIN pg_class c ON c.oid=a.attrelid WHERE c.relnamespace=current_schema()::regnamespace AND a.attisdropped)`).Scan(&dropped); err != nil || dropped { + return false, errors.New("unsupported PostgreSQL dropped-column layout; upgrade was not attempted") + } + tables, err := layouts(kind) + if err != nil { + return false, err + } + if version == current { + return false, validatePostgresLayout(db, kind, tables) + } + // The historical layouts differ only by these exact additive objects. + // All other columns, constraints, indexes, views and policies must match + // the canonical manifest, including user extensions that would be unsafe. + oldTables := make([]importTable, 0, len(tables)) + for _, table := range tables { + if !upgradeAddedRelation(kind, table.Name) { + oldTables = append(oldTables, table) + } + } + got, err := postgresLayout(db, kind, oldTables) + if err != nil { + return false, err + } + var canonical map[string][]string + if err := json.Unmarshal(postgresLayoutJSON, &canonical); err != nil { + return false, err + } + var want []string + for _, entry := range canonical[kind] { + var parts []any + if err := json.Unmarshal([]byte(entry), &parts); err != nil { + return false, err + } + if len(parts) < 2 { + return false, errors.New("invalid canonical PostgreSQL layout") + } + relation, _ := parts[1].(string) + if upgradeAddedRelation(kind, relation) { + continue + } + if kind == "accounts" && relation == "sessions" { + if parts[0] == "column" && len(parts) > 3 && (parts[3] == "kind" || parts[3] == "label" || parts[3] == "scopes") { + continue + } + if parts[0] == "constraint" && len(parts) > 2 { + switch parts[2] { + case "sessions_kind_check", "sessions_kind_not_null", "sessions_label_not_null", "sessions_scopes_not_null": + continue + } + } + } + want = append(want, entry) + } + if !slices.Equal(got, want) { + return false, errors.New("unsupported PostgreSQL upgrade layout; source data and selection were not changed") + } + return true, nil +} + +func recordedPostgresVersions(kind string) (previous, current int) { + if kind == "accounts" { + return 5, users.CurrentSchemaVersion + } + return 1, dbschema.Version +} + +func upgradeAddedRelation(kind, name string) bool { + if kind == "telemetry" { + return name == "tx_rewrite_feed" || name == "key_backfill_state" || name == "key_backfill_done" + } + return name == "companion_links" || name == "link_challenges" || name == "linked_companions" +} diff --git a/cmd/migrate/recorded_upgrade_test.go b/cmd/migrate/recorded_upgrade_test.go new file mode 100644 index 000000000..cac9e04b0 --- /dev/null +++ b/cmd/migrate/recorded_upgrade_test.go @@ -0,0 +1,339 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "log" + "net/url" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/users" +) + +func TestRecordedPostgresUpgradeDiagnosticsKeepStatusJSON(t *testing.T) { + o, _ := recordedOldPostgres(t, true) + var diagnostic, status bytes.Buffer + old := log.Writer() + log.SetOutput(&diagnostic) + t.Cleanup(func() { log.SetOutput(old) }) + if err := runStorageAction(context.Background(), o, &status); err != nil { + t.Fatal(err) + } + if !json.Valid(status.Bytes()) { + t.Fatal("upgrade messages contaminated machine-readable status", status.String()) + } + for _, message := range []string{ + "PostgreSQL telemetry schema upgrade required: 1 -> 2", + "PostgreSQL accounts schema upgrade required: 5 -> 6", + "upgraded recorded PostgreSQL telemetry schema 1 -> 2", + "upgraded recorded PostgreSQL accounts schema 5 -> 6", + } { + if !strings.Contains(diagnostic.String(), message) { + t.Errorf("missing supported upgrade diagnostic %q", message) + } + } + diagnostic.Reset() + status.Reset() + if err := runStorageAction(context.Background(), o, &status); err != nil { + t.Fatal(err) + } + if strings.Contains(diagnostic.String(), "upgrade required") || strings.Contains(diagnostic.String(), "upgraded recorded") { + t.Fatal("unchanged current schemas reported a new upgrade") + } +} + +func TestRecordedPostgresUpgradeDoesNotAnnounceUnknownLayout(t *testing.T) { + o, _ := recordedOldPostgres(t, true) + if _, err := openImportDB(t, o.UsersOwnerURL).Exec(`ALTER TABLE sessions ADD COLUMN unknown_extension TEXT`); err != nil { + t.Fatal(err) + } + var diagnostic bytes.Buffer + old := log.Writer() + log.SetOutput(&diagnostic) + t.Cleanup(func() { log.SetOutput(old) }) + if err := runStorageAction(context.Background(), o, &bytes.Buffer{}); err == nil { + t.Fatal("unknown account layout accepted") + } + if strings.Contains(diagnostic.String(), "upgrade required") || strings.Contains(diagnostic.String(), "upgraded recorded") { + t.Fatal("unknown layout was announced as a supported upgrade") + } +} + +func recordedOldPostgres(t *testing.T, accounts bool) (storageOptions, dbconfig.Selection) { + t.Helper() + o, _ := switchFixture(t) + o.Action = "setup" + dir := t.TempDir() + o.SelectionFile = dbconfig.SelectionPath(dir) + telemetry := openImportDB(t, o.OwnerURL) + if err := dbschema.ApplyPostgres(telemetry, nil); err != nil { + t.Fatal(err) + } + for _, q := range []string{ + `DROP TABLE tx_rewrite_feed,key_backfill_state,key_backfill_done`, + `UPDATE corescope_schema SET version=1 WHERE kind='telemetry'`, + `INSERT INTO transmissions(id,raw_hex,hash,first_seen) VALUES(90,'00','preserved','raw timestamp')`, + `SELECT setval(pg_get_serial_sequence('transmissions','id'),10000,true)`, + } { + if _, err := telemetry.Exec(q); err != nil { + t.Fatal(err) + } + } + storage := dbconfig.Storage{Backend: dbconfig.Postgres, StateDir: dir, WriterDatabaseURL: o.OwnerURL} + if accounts { + storage.UsersDatabaseURL = o.UsersOwnerURL + account := openImportDB(t, o.UsersOwnerURL) + if err := users.ApplyPostgres(account); err != nil { + t.Fatal(err) + } + for _, q := range []string{ + `DROP VIEW linked_companions`, `DROP TABLE companion_links,link_challenges`, + `DROP TABLE sessions`, + `CREATE TABLE sessions ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + token_hash TEXT COLLATE "C" NOT NULL UNIQUE, + user_id BIGINT NOT NULL REFERENCES users(id) ON DELETE CASCADE, + csrf_token TEXT COLLATE "C" NOT NULL, + created_at BIGINT NOT NULL, expires_at BIGINT NOT NULL, + last_seen_at BIGINT NOT NULL, user_agent TEXT COLLATE "C" NOT NULL DEFAULT '')`, + `CREATE INDEX sessions_user ON sessions(user_id)`, + `UPDATE schema_version SET version=5`, + `INSERT INTO users(id,email,display_name,password_hash,created_at) VALUES(4000000000,'upgrade@example.invalid','Retained','opaque hash',1)`, + `INSERT INTO sessions(id,token_hash,user_id,csrf_token,created_at,expires_at,last_seen_at) VALUES(2,'opaque session',4000000000,'opaque csrf',1,9999999999,2)`, + `SELECT setval(pg_get_serial_sequence('users','id'),4000000100,true)`, + } { + if _, err := account.Exec(q); err != nil { + t.Fatal(err) + } + } + } + selection, err := dbconfig.NewSelection(storage) + if err != nil { + t.Fatal(err) + } + if _, err := dbconfig.AdoptSelection(o.SelectionFile, selection); err != nil { + t.Fatal(err) + } + return o, selection +} + +func TestRecordedPostgresSetupUpgradesExistingStores(t *testing.T) { + for _, enabled := range []bool{false, true} { + name := "disabled accounts" + if enabled { + name = "enabled accounts" + } + t.Run(name, func(t *testing.T) { + o, selected := recordedOldPostgres(t, true) + config := `{"userManagement":{"enabled":false}}` + if enabled { + config = `{"userManagement":{"enabled":true}}` + } + if err := os.WriteFile(filepath.Join(o.ConfigDir, "config.json"), []byte(config), 0600); err != nil { + t.Fatal(err) + } + before, err := os.ReadFile(o.SelectionFile) + if err != nil { + t.Fatal(err) + } + if err := runStorageAction(context.Background(), o, &bytes.Buffer{}); err != nil { + t.Fatal("recorded owner upgrade", err) + } + if err := runStorageAction(context.Background(), o, &bytes.Buffer{}); err != nil { + t.Fatal("repeated recorded owner upgrade", err) + } + after, err := os.ReadFile(o.SelectionFile) + if err != nil || !bytes.Equal(before, after) { + t.Fatal("upgrade changed selection", err) + } + telemetry := openImportDB(t, o.OwnerURL) + account := openImportDB(t, o.UsersOwnerURL) + if err := dbschema.AssertPostgresReady(telemetry); err != nil { + t.Fatal(err) + } + if err := users.AssertPostgresReady(account); err != nil { + t.Fatal(err) + } + var hash, stamp, token, kind string + if err := telemetry.QueryRow(`SELECT hash,first_seen FROM transmissions WHERE id=90`).Scan(&hash, &stamp); err != nil || hash != "preserved" || stamp != "raw timestamp" { + t.Fatal("telemetry changed", err) + } + if err := account.QueryRow(`SELECT token_hash,kind FROM sessions WHERE id=2 AND user_id=4000000000`).Scan(&token, &kind); err != nil || token != "opaque session" || kind != "web" { + t.Fatal("authentication changed", err) + } + for _, pair := range []struct { + kind, url, table string + high int64 + }{{"telemetry", o.OwnerURL, "transmissions", 10000}, {"accounts", o.UsersOwnerURL, "users", 4000000100}} { + db := openImportDB(t, pair.url) + var high int64 + if err := db.QueryRow(`SELECT last_value FROM ` + quote(pair.table+"_id_seq")).Scan(&high); err != nil || high != pair.high { + t.Fatal("identity changed", pair.kind, high, err) + } + } + raw, err := storageActionInputs(o, true) + if err != nil { + t.Fatal(err) + } + if err := validateRuntimeSelection(context.Background(), selected, raw); err != nil { + t.Fatal("runtime grants", err) + } + }) + } +} + +func TestRecordedPostgresUpgradePreservesAbsentAccounts(t *testing.T) { + o, _ := recordedOldPostgres(t, false) + o.Action = "upgrade" + before, err := os.ReadFile(o.SelectionFile) + if err != nil { + t.Fatal(err) + } + if err := runStorageAction(context.Background(), o, &bytes.Buffer{}); err != nil { + t.Fatal(err) + } + after, err := os.ReadFile(o.SelectionFile) + if err != nil || !bytes.Equal(before, after) { + t.Fatal("upgrade added an account target", err) + } + var tables int + if err := openImportDB(t, o.UsersOwnerURL).QueryRow(importDestinationObjects).Scan(&tables); err != nil || tables != 0 { + t.Fatal("upgrade initialized inactive account target", err) + } +} + +func TestRecordedPostgresUpgradeRejectsUnknownOrIncompleteBeforeAnyDDL(t *testing.T) { + for _, damage := range []string{ + `UPDATE schema_version SET version=999`, + `UPDATE corescope_schema SET ready=false`, + `ALTER TABLE sessions ADD COLUMN unexpected_extension TEXT`, + `ALTER TABLE sessions ADD COLUMN discarded TEXT; ALTER TABLE sessions DROP COLUMN discarded`, + `CREATE TABLE unexpected_extension(keep TEXT)`, + } { + t.Run(damage, func(t *testing.T) { + o, _ := recordedOldPostgres(t, true) + account := openImportDB(t, o.UsersOwnerURL) + if _, err := account.Exec(damage); err != nil { + t.Fatal(err) + } + before, err := os.ReadFile(o.SelectionFile) + if err != nil { + t.Fatal(err) + } + if err := runStorageAction(context.Background(), o, &bytes.Buffer{}); err == nil { + t.Fatal("unsupported source upgraded") + } + var version int + if err := openImportDB(t, o.OwnerURL).QueryRow(`SELECT version FROM corescope_schema WHERE kind='telemetry'`).Scan(&version); err != nil || version != 1 { + t.Fatal("telemetry upgraded before account preflight completed", err) + } + after, err := os.ReadFile(o.SelectionFile) + if err != nil || !bytes.Equal(before, after) { + t.Fatal("refused upgrade changed selection", err) + } + }) + } +} + +func TestRecordedPostgresUpgradeRejectsMissingStoresAndWrongOwners(t *testing.T) { + for _, boundary := range []string{"missing account store", "unknown telemetry version", "non-owner", "different database", "different schema", "different endpoint", "running service"} { + t.Run(boundary, func(t *testing.T) { + o, _ := recordedOldPostgres(t, true) + o.Action = "upgrade" + telemetryOwner := o.OwnerURL + var closeLease func() + switch boundary { + case "missing account store": + if _, err := openImportDB(t, o.UsersOwnerURL).Exec(`DROP SCHEMA public CASCADE; CREATE SCHEMA public`); err != nil { + t.Fatal(err) + } + case "unknown telemetry version": + if _, err := openImportDB(t, o.OwnerURL).Exec(`UPDATE corescope_schema SET version=999`); err != nil { + t.Fatal(err) + } + case "non-owner": + o.OwnerURL = os.Getenv("CORESCOPE_WRITER_DATABASE_URL") + u, err := url.Parse(o.OwnerURL) + if err != nil { + t.Fatal(err) + } + if _, err := openImportDB(t, telemetryOwner).Exec(`GRANT USAGE ON SCHEMA public TO ` + quote(u.User.Username())); err != nil { + t.Fatal(err) + } + case "different database": + o.OwnerURL = postgresDatabase(t) + case "different endpoint": + u, err := url.Parse(o.OwnerURL) + if err != nil { + t.Fatal(err) + } + host := "localhost" + if u.Hostname() == host { + host = "127.0.0.1" + } + u.Host = host + ":" + u.Port() + o.OwnerURL = u.String() + case "different schema": + if _, err := openImportDB(t, o.OwnerURL).Exec(`CREATE SCHEMA not_selected`); err != nil { + t.Fatal(err) + } + u, err := url.Parse(o.OwnerURL) + if err != nil { + t.Fatal(err) + } + q := u.Query() + q.Set("search_path", "not_selected") + u.RawQuery = q.Encode() + o.OwnerURL = u.String() + case "running service": + _, lease, err := dbconfig.OpenSelection(o.SelectionFile) + if err != nil { + t.Fatal(err) + } + closeLease = func() { lease.Close() } + defer closeLease() + } + before, err := os.ReadFile(o.SelectionFile) + if err != nil { + t.Fatal(err) + } + err = runStorageAction(context.Background(), o, &bytes.Buffer{}) + if err == nil { + t.Fatal("unsafe upgrade accepted") + } + if boundary == "running service" && !errors.Is(err, dbconfig.ErrSelectionBusy) { + t.Fatal("running service guard", err) + } + var version int + want := 1 + if boundary == "unknown telemetry version" { + want = 999 + } + if err := openImportDB(t, telemetryOwner).QueryRow(`SELECT version FROM corescope_schema WHERE kind='telemetry'`).Scan(&version); err != nil || version != want { + t.Fatal("refused upgrade changed telemetry", err) + } + after, err := os.ReadFile(o.SelectionFile) + if err != nil || !bytes.Equal(before, after) { + t.Fatal("refused upgrade changed selection", err) + } + }) + } +} + +func TestPostgresUpgradeNeverInitializesAnUnrecordedInstallation(t *testing.T) { + dir := t.TempDir() + o := storageOptions{Action: "upgrade", Offline: true, SelectionFile: dbconfig.SelectionPath(dir), ConfigDir: t.TempDir()} + if err := runStorageAction(context.Background(), o, &bytes.Buffer{}); err == nil { + t.Fatal("unrecorded upgrade accepted") + } + if _, err := os.Stat(o.SelectionFile); !os.IsNotExist(err) { + t.Fatal("upgrade created a selection", err) + } +} diff --git a/cmd/migrate/resume_finalized_test.go b/cmd/migrate/resume_finalized_test.go new file mode 100644 index 000000000..6bb8eaaae --- /dev/null +++ b/cmd/migrate/resume_finalized_test.go @@ -0,0 +1,358 @@ +package main + +import ( + "bytes" + "context" + "database/sql" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/users" +) + +// Include tuple versions and sequence state so a refusal or a verified ready +// store cannot conceal rewrites of identical values, progress, or readiness. +func importDatabaseState(t *testing.T, db *sql.DB, kind string) string { + t.Helper() + tables, err := layouts(kind) + if err != nil { + t.Fatal(err) + } + var state strings.Builder + for _, table := range append(tables, importTable{Name: "corescope_import"}, importTable{Name: "corescope_import_progress"}, importTable{Name: "corescope_schema"}) { + rows, err := db.Query(`SELECT row_to_json(t)::text,t.xmin::text,t.ctid::text FROM ` + quote(table.Name) + ` t ORDER BY row_to_json(t)::text COLLATE "C"`) + if err != nil { + t.Fatal(err) + } + for rows.Next() { + var values, version, position string + if err := rows.Scan(&values, &version, &position); err != nil { + t.Fatal(err) + } + fmt.Fprintln(&state, table.Name, values, version, position) + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + rows.Close() + for _, column := range table.Columns { + if !column.Identity { + continue + } + var sequence string + if err := db.QueryRow(`SELECT pg_get_serial_sequence($1,$2)`, table.Name, column.Name).Scan(&sequence); err != nil { + t.Fatal(err) + } + var value int64 + var called bool + if err := db.QueryRow(`SELECT last_value,is_called FROM `+sequence).Scan(&value, &called); err != nil { + t.Fatal(err) + } + fmt.Fprintln(&state, sequence, value, called) + } + } + return state.String() +} + +func mixedImport(t *testing.T, startAccounts, readyTelemetry bool) [2]importOptions { + t.Helper() + state := filepath.Join(t.TempDir(), "migration state") + options := [2]importOptions{ + {Source: telemetrySource(t), DatabaseURL: postgresDatabase(t), StateDir: state, Kind: "telemetry"}, + {Source: accountSource(t, 5), DatabaseURL: postgresDatabase(t), StateDir: state, Kind: "accounts"}, + } + for i, o := range options { + if i == 1 && !startAccounts { + continue + } + if _, err := importSQLite(context.Background(), o); err != nil { + t.Fatal(err) + } + } + if readyTelemetry { + if _, err := finalizeImport(context.Background(), options[0].DatabaseURL, "telemetry"); err != nil { + t.Fatal(err) + } + } + t.Setenv("CORESCOPE_DATABASE_URL", options[0].DatabaseURL) + t.Setenv("CORESCOPE_USERS_DATABASE_URL", options[1].DatabaseURL) + return options +} + +func resumeMixedCommand(options [2]importOptions, out io.Writer) error { + return runCommand(context.Background(), []string{"-offline", "-resume", "-from-sqlite", options[0].Source, "-users-from-sqlite", options[1].Source, "-state-dir", options[0].StateDir}, out) +} + +func TestCLIResumeAfterFirstStoreFinalized(t *testing.T) { + options := mixedImport(t, true, true) + telemetry := openImportDB(t, options[0].DatabaseURL) + accounts := openImportDB(t, options[1].DatabaseURL) + before := importDatabaseState(t, telemetry, "telemetry") + beforeSources := make(map[string]string) + for _, o := range options { + for _, path := range []string{o.Source, filepath.Join(o.StateDir, o.Kind, "recovery.sqlite"), filepath.Join(o.StateDir, o.Kind, "normalized.sqlite")} { + digest, err := fingerprint(context.Background(), path) + if err != nil { + t.Fatal(err) + } + beforeSources[path] = digest + } + } + if err := users.AssertReady(accounts); err == nil { + t.Fatal("fixture must reproduce the crash between the two readiness commits") + } + var output bytes.Buffer + if err := resumeMixedCommand(options, &output); err != nil { + t.Fatalf("resume after telemetry finalized but accounts unready: %v", err) + } + if err := dbschema.AssertReady(telemetry); err != nil { + t.Fatal(err) + } + if err := users.AssertReady(accounts); err != nil { + t.Fatal(err) + } + if got := importDatabaseState(t, telemetry, "telemetry"); got != before { + t.Fatal("resume wrote already-ready telemetry data, progress, sequence, or readiness") + } + for _, o := range options { + if !strings.Contains(output.String(), o.Kind+" import verified and finalized") || strings.Contains(output.String(), o.DatabaseURL) { + t.Fatal("missing success evidence or leaked database URL") + } + } + for path, before := range beforeSources { + after, err := fingerprint(context.Background(), path) + if err != nil || after != before { + t.Fatal("resume changed original source or immutable snapshot", err) + } + } +} + +type importOutputFunc func([]byte) (int, error) + +func (f importOutputFunc) Write(data []byte) (int, error) { return f(data) } + +func TestCLIResumeFailureDoesNotResetAlreadyReadyStore(t *testing.T) { + options := mixedImport(t, true, true) + telemetry := openImportDB(t, options[0].DatabaseURL) + accounts := openImportDB(t, options[1].DatabaseURL) + before := importDatabaseState(t, telemetry, "telemetry") + // Fail the second finalization after both imports have been verified, so + // the actual CLI rollback path must distinguish previously ready stores. + out := importOutputFunc(func(data []byte) (int, error) { + if bytes.Contains(data, []byte(`"kind":"accounts"`)) { + if _, err := accounts.Exec(`UPDATE users SET display_name='post-verification change'`); err != nil { + return 0, err + } + } + return len(data), nil + }) + if err := resumeMixedCommand(options, out); err == nil || !strings.Contains(err.Error(), "final verification failed") { + t.Fatalf("second finalization corruption should be refused: %v", err) + } + if got := importDatabaseState(t, telemetry, "telemetry"); got != before { + t.Fatal("failed second finalization reset or rewrote the previously ready store") + } + if err := users.AssertReady(accounts); err == nil { + t.Fatal("changed accounts became ready") + } +} + +func TestCLIRollbackDoesNotCloseGateOpenedByAnotherFinalizer(t *testing.T) { + options := mixedImport(t, true, false) + telemetry := openImportDB(t, options[0].DatabaseURL) + accounts := openImportDB(t, options[1].DatabaseURL) + var finalizedState string + out := importOutputFunc(func(data []byte) (int, error) { + if bytes.Contains(data, []byte(`"kind":"accounts"`)) { + // The importer observed telemetry unready. A separate allowed + // finalizer opens it before this CLI reaches its finalization loop. + if _, err := finalizeImport(context.Background(), options[0].DatabaseURL, "telemetry"); err != nil { + return 0, err + } + finalizedState = importDatabaseState(t, telemetry, "telemetry") + if _, err := accounts.Exec(`UPDATE users SET display_name='fail second finalization'`); err != nil { + return 0, err + } + } + return len(data), nil + }) + if err := resumeMixedCommand(options, out); err == nil || !strings.Contains(err.Error(), "final verification failed") { + t.Fatalf("second finalization should fail after the separate finalizer ran: %v", err) + } + if finalizedState == "" { + t.Fatal("fixture did not execute the intervening finalization") + } + if err := dbschema.AssertReady(telemetry); err != nil { + t.Error("CLI reset a gate it did not open", err) + } + if got := importDatabaseState(t, telemetry, "telemetry"); got != finalizedState { + t.Error("CLI changed tuples or readiness owned by the separate finalizer") + } + if err := users.AssertReady(accounts); err == nil { + t.Fatal("corrupted accounts became ready") + } +} + +func TestCLIRollbackClosesGateOpenedByThisInvocation(t *testing.T) { + options := mixedImport(t, true, false) + telemetry := openImportDB(t, options[0].DatabaseURL) + accounts := openImportDB(t, options[1].DatabaseURL) + out := importOutputFunc(func(data []byte) (int, error) { + if bytes.Contains(data, []byte(`"kind":"accounts"`)) { + if _, err := accounts.Exec(`UPDATE users SET display_name='fail second finalization'`); err != nil { + return 0, err + } + } + return len(data), nil + }) + if err := resumeMixedCommand(options, out); err == nil || !strings.Contains(err.Error(), "final verification failed") { + t.Fatalf("second finalization should fail: %v", err) + } + if err := dbschema.AssertReady(telemetry); err == nil { + t.Fatal("CLI did not close the telemetry gate it opened before account failure") + } + if err := users.AssertReady(accounts); err == nil { + t.Fatal("corrupted accounts became ready") + } +} + +func TestCLIResumeDoesNotStartOccupiedSecondStore(t *testing.T) { + options := mixedImport(t, false, true) + telemetry := openImportDB(t, options[0].DatabaseURL) + accounts := openImportDB(t, options[1].DatabaseURL) + if _, err := accounts.Exec(`CREATE SEQUENCE keep_me START WITH 77`); err != nil { + t.Fatal(err) + } + before := importDatabaseState(t, telemetry, "telemetry") + if err := resumeMixedCommand(options, io.Discard); err == nil || !strings.Contains(err.Error(), "not empty") { + t.Fatalf("resume accepted an occupied unmarked destination: %v", err) + } + if got := importDatabaseState(t, telemetry, "telemetry"); got != before { + t.Fatal("refusal changed already-ready telemetry") + } + var value int + var called bool + if err := accounts.QueryRow(`SELECT last_value,is_called FROM keep_me`).Scan(&value, &called); err != nil || value != 77 || called { + t.Fatal("refusal changed occupied account destination", err) + } + if _, err := os.Stat(filepath.Join(options[1].StateDir, "accounts")); !os.IsNotExist(err) { + t.Fatal("occupied destination should be refused before source preparation") + } +} + +func TestCLIResumeStartsUntouchedSecondStore(t *testing.T) { + options := mixedImport(t, false, false) + if err := resumeMixedCommand(options, io.Discard); err != nil { + t.Fatalf("resume must finish the verified first store and start untouched accounts: %v", err) + } + if err := dbschema.AssertReady(openImportDB(t, options[0].DatabaseURL)); err != nil { + t.Fatal(err) + } + if err := users.AssertReady(openImportDB(t, options[1].DatabaseURL)); err != nil { + t.Fatal(err) + } +} + +func TestResumePreparationArtifactsArePreserved(t *testing.T) { + o := importOptions{Source: telemetrySource(t), DatabaseURL: postgresSchema(t), StateDir: t.TempDir(), Kind: "telemetry", Resume: true} + recovery := filepath.Join(o.StateDir, o.Kind, "recovery.sqlite") + if err := os.MkdirAll(filepath.Dir(recovery), 0o700); err != nil { + t.Fatal(err) + } + if err := snapshotSource(context.Background(), o.Source, recovery); err != nil { + t.Fatal(err) + } + before, err := os.ReadFile(recovery) + if err != nil { + t.Fatal(err) + } + _, err = importSQLite(context.Background(), o) + if err == nil || !strings.Contains(err.Error(), "preserve") || !strings.Contains(err.Error(), "state directory") { + t.Fatalf("interrupted preparation needs preserving recovery guidance: %v", err) + } + after, err := os.ReadFile(recovery) + if err != nil || !bytes.Equal(before, after) { + t.Fatal("refusal changed the only recovery snapshot", err) + } + var n int + if err := openImportDB(t, o.DatabaseURL).QueryRow(`SELECT count(*) FROM information_schema.tables WHERE table_schema=current_schema()`).Scan(&n); err != nil || n != 0 { + t.Fatal("preparation refusal changed the empty destination", err) + } + // Exercise the documented recovery: archive the incomplete per-store state + // without deleting it, then let resume prepare the still-empty store anew. + archived := filepath.Join(t.TempDir(), "preserved incomplete preparation") + if err := os.Rename(filepath.Dir(recovery), archived); err != nil { + t.Fatal(err) + } + if report, err := importSQLite(context.Background(), o); err != nil || !report.Verified { + t.Fatal("resume after preserving incomplete preparation failed", err) + } + retained, err := os.ReadFile(filepath.Join(archived, "recovery.sqlite")) + if err != nil || !bytes.Equal(before, retained) { + t.Fatal("retry changed archived recovery evidence", err) + } +} + +func TestReadyResumeRefusesChangedEvidenceWithoutWrites(t *testing.T) { + changes := map[string]string{ + "new PostgreSQL row": `INSERT INTO observers(id) VALUES('post-cutover')`, + "changed PostgreSQL row": `UPDATE observers SET name='changed' WHERE rowid=0`, + "changed sequence only": `SELECT nextval(pg_get_serial_sequence('observers','rowid'))`, + "incomplete progress": `UPDATE corescope_import_progress SET complete=false WHERE table_name='observers'`, + "changed progress digest": `UPDATE corescope_import_progress SET sha256=repeat('0',64) WHERE table_name='observers'`, + "unverified marker": `UPDATE corescope_import SET verified=false`, + "changed report": `UPDATE corescope_import SET report=replace(report,'"rows":2','"rows":200')`, + "changed run identity": `UPDATE corescope_import SET run_id='different'`, + "source": "", + "recovery snapshot": "", + "normalized snapshot": "", + } + for name, statement := range changes { + t.Run(name, func(t *testing.T) { + o := importOptions{Source: telemetrySource(t), DatabaseURL: postgresSchema(t), StateDir: t.TempDir(), Kind: "telemetry"} + if _, err := importSQLite(context.Background(), o); err != nil { + t.Fatal(err) + } + if _, err := finalizeImport(context.Background(), o.DatabaseURL, o.Kind); err != nil { + t.Fatal(err) + } + db := openImportDB(t, o.DatabaseURL) + if statement != "" { + if _, err := db.Exec(statement); err != nil { + t.Fatal(err) + } + } else { + path := o.Source + if name != "source" { + file := "normalized.sqlite" + if name == "recovery snapshot" { + file = "recovery.sqlite" + } + path = filepath.Join(o.StateDir, o.Kind, file) + } + changed, err := sql.Open("sqlite3", path) + if err != nil { + t.Fatal(err) + } + _, err = changed.Exec(`UPDATE observers SET name='changed' WHERE rowid=0`) + changed.Close() + if err != nil { + t.Fatal(err) + } + } + before := importDatabaseState(t, db, o.Kind) + o.Resume = true + if _, err := importSQLite(context.Background(), o); err == nil { + t.Fatal("changed evidence accepted for an already-ready destination") + } + if after := importDatabaseState(t, db, o.Kind); after != before { + t.Fatal("refusal changed data, progress, sequence, or readiness") + } + }) + } +} diff --git a/cmd/migrate/reverse.go b/cmd/migrate/reverse.go new file mode 100644 index 000000000..c3c4330b0 --- /dev/null +++ b/cmd/migrate/reverse.go @@ -0,0 +1,817 @@ +package main + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "database/sql" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "math" + "net/url" + "os" + "path/filepath" + "slices" + "strings" + + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/dbschema/legacy" + "github.com/meshcore-analyzer/pgutil" + "github.com/meshcore-analyzer/users" +) + +type reverseOptions struct { + DatabaseURL, Destination, StateDir, Kind string + BatchSize int + Resume bool + keepPending bool + afterBatch func() error + source *postgresFence +} + +type identityState struct { + Table string `json:"table"` + Column string `json:"column"` + LastValue int64 `json:"last_value"` + IsCalled bool `json:"is_called"` + SQLiteFloor int64 `json:"sqlite_floor"` +} + +type reverseManifest struct { + Version int `json:"version"` + RunID string `json:"run_id"` + Kind string `json:"kind"` + Destination string `json:"destination"` + SourceIdentity string `json:"source_identity"` + RecoverySHA256 string `json:"recovery_sha256"` + Source []tableReport `json:"source"` + Identities []identityState `json:"identities"` + AddedSQLiteMigrations []string `json:"added_sqlite_migrations"` +} + +type postgresFence struct { + db *sql.DB + tx *sql.Tx + kind string + tables []importTable + identity string +} + +func (f *postgresFence) Close() { + if f.tx != nil { + f.tx.Rollback() + } + if f.db != nil { + f.db.Close() + } +} + +// Local service leases are only one fence. Native table locks and the ingestor's +// commit-order lock also exclude real database writers during the offline copy. +func fencePostgres(ctx context.Context, dsn, kind string) (_ *postgresFence, err error) { + f := &postgresFence{kind: kind} + if f.tables, err = layouts(kind); err != nil { + return nil, err + } + if f.db, err = pgutil.Open(dsn, false); err != nil { + return nil, err + } + defer func() { + if err != nil { + f.Close() + } + }() + if kind == "telemetry" { + err = dbschema.AssertPostgresReady(f.db) + } else { + err = users.AssertPostgresReady(f.db) + } + if err != nil { + return nil, err + } + f.tx, err = f.db.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelRepeatableRead}) + if err != nil { + return nil, err + } + var locked bool + if err = f.tx.QueryRowContext(ctx, `SELECT pg_try_advisory_xact_lock(hashtextextended(current_schema(),$1))`, dbschema.WriterLockKey).Scan(&locked); err != nil { + return nil, err + } + if !locked { + return nil, errors.New("PostgreSQL source writer is active; stop all writers before conversion") + } + var names []string + for _, table := range f.tables { + names = append(names, quote(table.Name)) + } + if _, err = f.tx.ExecContext(ctx, `LOCK TABLE `+strings.Join(names, ",")+` IN SHARE MODE NOWAIT`); err != nil { + return nil, errors.New("PostgreSQL source is in use; stop all writers before conversion") + } + if err = validatePostgresLayout(f.db, kind, f.tables); err != nil { + return nil, err + } + config, e := pgutil.ParseConfig(dsn) + if e != nil { + return nil, e + } + var database, schema string + var dbOID, schemaOID int64 + if err = f.tx.QueryRowContext(ctx, `SELECT current_database(),current_schema(),(SELECT oid::bigint FROM pg_database WHERE datname=current_database()),(SELECT oid::bigint FROM pg_namespace WHERE nspname=current_schema())`).Scan(&database, &schema, &dbOID, &schemaOID); err != nil { + return nil, err + } + data, _ := json.Marshal([]any{database, schema, dbOID, schemaOID, config.Host, config.Port}) + sum := sha256.Sum256(data) + f.identity = hex.EncodeToString(sum[:]) + return f, nil +} + +func identityReferences(kind string) map[string][][2]string { + if kind == "telemetry" { + return map[string][][2]string{ + "observers": {{"observations", "observer_idx"}}, + "transmissions": {{"observations", "transmission_id"}, {"advert_route_evidence", "tx_id"}, {"advert_evidence_backfill", "tx_cursor"}, {"tx_rewrite_feed", "tx_id"}, {"key_backfill_state", "cursor"}, {"key_backfill_state", "upper"}}, + "observations": {{"advert_evidence_backfill", "obs_cursor"}}, + } + } + return map[string][][2]string{ + "users": {{"audit_log", "actor_user_id"}, {"audit_log", "target_user_id"}, {"users", "activated_by"}, {"sessions", "user_id"}, {"tokens", "user_id"}, {"mail_log", "user_id"}, {"proposals", "proposer_id"}, {"proposals", "reviewer_id"}, {"user_settings", "user_id"}, {"notification_prefs", "user_id"}, {"notification_watches", "user_id"}, {"notification_state", "user_id"}, {"companion_links", "user_id"}, {"link_challenges", "user_id"}}, + "mail_log": {{"mail_events", "mail_id"}}, + } +} + +func postgresIdentities(ctx context.Context, f *postgresFence) ([]identityState, error) { + var out []identityState + refs := identityReferences(f.kind) + for _, table := range f.tables { + for _, col := range table.Columns { + if !col.Identity { + continue + } + var sequence string + if err := f.db.QueryRowContext(ctx, `SELECT pg_get_serial_sequence($1,$2)`, table.Name, col.Name).Scan(&sequence); err != nil { + return nil, err + } + var increment, min, max int64 + var cycle bool + if err := f.db.QueryRowContext(ctx, `SELECT seqincrement,seqmin,seqmax,seqcycle FROM pg_sequence WHERE seqrelid=$1::regclass`, sequence).Scan(&increment, &min, &max, &cycle); err != nil { + return nil, err + } + if increment != 1 || min != 1 || max != math.MaxInt64 || cycle { + return nil, errors.New("source identity sequence has unsupported custom allocation settings") + } + state := identityState{Table: table.Name, Column: col.Name} + if err := f.db.QueryRowContext(ctx, `SELECT last_value,is_called FROM `+sequence).Scan(&state.LastValue, &state.IsCalled); err != nil { + return nil, err + } + state.SQLiteFloor = state.LastValue + if !state.IsCalled { + state.SQLiteFloor-- + } + if state.SQLiteFloor < 0 { + return nil, errors.New("invalid source identity high water") + } + for _, ref := range append([][2]string{{table.Name, col.Name}}, refs[table.Name]...) { + var high int64 + if err := f.db.QueryRowContext(ctx, `SELECT COALESCE(MAX(`+quote(ref[1])+`),0) FROM `+quote(ref[0])).Scan(&high); err != nil { + return nil, err + } + if high > state.SQLiteFloor { + state.SQLiteFloor = high + } + } + out = append(out, state) + } + } + return out, nil +} + +func digestPostgresSource(ctx context.Context, f *postgresFence) ([]tableReport, error) { + var reports []tableReport + for _, table := range f.tables { + // Bound decompressed text before a large value reaches the Go driver. + sizes := []string{"0"} + for _, col := range table.Columns { + if col.Type == "TEXT" { + sizes = append(sizes, `COALESCE(octet_length(`+quote(col.Name)+`),0)::bigint`) + } else { + sizes = append(sizes, "8") + } + } + var oversized bool + if err := f.db.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM `+quote(table.Name)+` WHERE `+strings.Join(sizes, "+")+`>$1)`, maxImportRowBytes).Scan(&oversized); err != nil { + return nil, err + } + if oversized { + return nil, errors.New("a source row exceeds the 64 MiB migration safety limit") + } + report, err := digestTarget(ctx, f.db, table) + if err != nil { + return nil, err + } + reports = append(reports, report) + } + return reports, nil +} + +func writePrivateJSON(path string, value any) error { + data, err := json.MarshalIndent(value, "", " ") + if err != nil { + return err + } + file, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600) + if err != nil { + return err + } + _, writeErr := file.Write(append(data, '\n')) + syncErr := file.Sync() + closeErr := file.Close() + return errors.Join(writeErr, syncErr, closeErr) +} +func readPrivateJSON(path string, value any) error { + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + d := json.NewDecoder(io.LimitReader(file, 1024*1024)) + d.DisallowUnknownFields() + if err := d.Decode(value); err != nil { + return errors.New("invalid private conversion manifest") + } + if err := d.Decode(new(any)); err != io.EOF { + return errors.New("invalid trailing conversion manifest") + } + return nil +} +func fileDigest(ctx context.Context, path string) (string, error) { + file, err := os.Open(path) + if err != nil { + return "", err + } + defer file.Close() + h := sha256.New() + if _, err := io.CopyBuffer(h, contextReader{ctx, file}, make([]byte, 128*1024)); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} + +func prepareReverse(ctx context.Context, o reverseOptions, f *postgresFence) (reverseManifest, error) { + var manifest reverseManifest + dest, err := filepath.Abs(o.Destination) + if err != nil { + return manifest, err + } + if o.Destination == "" || o.StateDir == "" { + return manifest, errors.New("reverse conversion requires a fresh SQLite target and private recovery directory") + } + dir := filepath.Join(o.StateDir, o.Kind) + path := filepath.Join(dir, "reverse.json") + backup := filepath.Join(dir, "recovery.pg_dump") + source, err := digestPostgresSource(ctx, f) + if err != nil { + return manifest, err + } + identities, err := postgresIdentities(ctx, f) + if err != nil { + return manifest, err + } + err = readPrivateJSON(path, &manifest) + if err == nil { + if !o.Resume || manifest.Version != 1 || manifest.Kind != o.Kind || manifest.Destination != dest || manifest.SourceIdentity != f.identity || !slices.Equal(source, manifest.Source) || !slices.Equal(identities, manifest.Identities) { + return manifest, errors.New("reverse resume source, sequence state or destination differs from the immutable manifest") + } + digest, err := fileDigest(ctx, backup) + if err != nil { + return manifest, err + } + if digest != manifest.RecoverySHA256 { + return manifest, errors.New("native PostgreSQL recovery archive changed; resume refused") + } + return manifest, nil + } + if !errors.Is(err, os.ErrNotExist) { + return manifest, err + } + if o.Resume { + return manifest, errors.New("reverse preparation has no complete manifest; preserve recovery files and abort before choosing a new job") + } + if _, err := os.Lstat(dest); !errors.Is(err, os.ErrNotExist) { + return manifest, errors.New("reverse destination already exists or cannot be inspected; existing files were not changed") + } + entries, err := os.ReadDir(dir) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return manifest, err + } + if len(entries) != 0 { + return manifest, errors.New("reverse recovery directory is not empty") + } + if err := os.MkdirAll(dir, 0700); err != nil { + return manifest, err + } + // Recovery is native and contains the whole selected database. The source + // table fences remain held; removing search_path changes no database target. + u, err := url.Parse(o.DatabaseURL) + if err != nil { + return manifest, errors.New("invalid source database URL") + } + q := u.Query() + q.Del("search_path") + u.RawQuery = q.Encode() + if err := pgutil.Dump(ctx, u.String(), backup); err != nil { + return manifest, err + } + digest, err := fileDigest(ctx, backup) + if err != nil { + return manifest, err + } + var random [16]byte + if _, err := rand.Read(random[:]); err != nil { + return manifest, err + } + manifest = reverseManifest{Version: 1, RunID: hex.EncodeToString(random[:]), Kind: o.Kind, Destination: dest, SourceIdentity: f.identity, RecoverySHA256: digest, Source: source, Identities: identities} + if o.Kind == "telemetry" { + template, err := sql.Open(importSQLiteDriver, ":memory:") + if err != nil { + return manifest, err + } + defer template.Close() + template.SetMaxOpenConns(1) + if err := dbschema.ApplySQLite(template, func(string, ...interface{}) {}); err != nil { + return manifest, err + } + rows, err := template.Query(`SELECT name FROM _migrations ORDER BY name`) + if err != nil { + return manifest, err + } + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + rows.Close() + return manifest, err + } + var n int + if err := f.db.QueryRowContext(ctx, `SELECT count(*) FROM _migrations WHERE name=$1`, name).Scan(&n); err != nil { + rows.Close() + return manifest, err + } + if n == 0 { + manifest.AddedSQLiteMigrations = append(manifest.AddedSQLiteMigrations, name) + } + } + err = rows.Err() + rows.Close() + if err != nil { + return manifest, err + } + } + after, err := postgresIdentities(ctx, f) + if err != nil { + return manifest, err + } + if !slices.Equal(identities, after) { + return manifest, errors.New("source sequence allocation changed during backup; stop every writer") + } + return manifest, writePrivateJSON(path, manifest) +} + +func reversePostgres(ctx context.Context, o reverseOptions) (importReport, error) { + report := importReport{Kind: o.Kind} + if o.BatchSize == 0 { + o.BatchSize = 1000 + } + if o.BatchSize < 1 || o.BatchSize > 10000 { + return report, errors.New("copy batch size must be between 1 and 10000") + } + f := o.source + if f == nil { + var err error + f, err = fencePostgres(ctx, o.DatabaseURL, o.Kind) + if err != nil { + return report, err + } + defer f.Close() + } + manifest, err := prepareReverse(ctx, o, f) + if err != nil { + return report, err + } + destination, verifyOnly, err := openReverseTarget(ctx, o, manifest) + if err != nil { + return report, err + } + defer destination.Close() + for i, table := range f.tables { + got, err := copyReverseTable(ctx, f.db, destination, table, o, manifest, verifyOnly) + if err != nil { + return report, fmt.Errorf("reverse %s: %w", table.Name, err) + } + if got != manifest.Source[i] { + return report, errors.New("reverse source count or logical digest changed") + } + report.Tables = append(report.Tables, got) + } + for _, identity := range manifest.Identities { + var count int + var high int64 + if err := destination.QueryRowContext(ctx, `SELECT count(*),COALESCE(MAX(seq),0) FROM sqlite_sequence WHERE name=?`, identity.Table).Scan(&count, &high); err != nil { + return report, err + } + if verifyOnly { + if count != 1 || high != identity.SQLiteFloor { + return report, errors.New("completed SQLite identity allocation changed; resume refused") + } + continue + } + if _, err := destination.ExecContext(ctx, `DELETE FROM sqlite_sequence WHERE name=?`, identity.Table); err != nil { + return report, err + } + if _, err := destination.ExecContext(ctx, `INSERT INTO sqlite_sequence(name,seq) VALUES(?,?)`, identity.Table, identity.SQLiteFloor); err != nil { + return report, err + } + } + if !verifyOnly { + for _, name := range manifest.AddedSQLiteMigrations { + if _, err := destination.ExecContext(ctx, `INSERT OR IGNORE INTO _migrations(name) VALUES(?)`, name); err != nil { + return report, err + } + } + } + for _, name := range manifest.AddedSQLiteMigrations { + var n int + if err := destination.QueryRowContext(ctx, `SELECT count(*) FROM _migrations WHERE name=?`, name).Scan(&n); err != nil || n != 1 { + return report, errors.New("SQLite physical migration marker is missing") + } + } + if o.Kind == "telemetry" { + // A copied ledger can already mark this async task done. Build the real + // index offline; never rewrite its retained status to hide absent work. + if err := dbschema.EnsureSQLiteObserverTimeIndex(ctx, destination); err != nil { + return report, err + } + } + if err := assertSQLiteIntegrity(ctx, destination); err != nil { + return report, err + } + sourceAfter, err := digestPostgresSource(ctx, f) + if err != nil { + return report, err + } + identitiesAfter, err := postgresIdentities(ctx, f) + if err != nil { + return report, err + } + if !slices.Equal(sourceAfter, manifest.Source) || !slices.Equal(identitiesAfter, manifest.Identities) { + return report, errors.New("PostgreSQL source or sequence changed during conversion; target remains unselected") + } + if err := validatePostgresLayout(f.db, o.Kind, f.tables); err != nil { + return report, err + } + if !o.keepPending { + if !verifyOnly { + if _, err := destination.ExecContext(ctx, `DROP TABLE corescope_reverse_progress`); err != nil { + return report, err + } + } + if err := assertSQLiteData(ctx, destination, o.Kind); err != nil { + return report, err + } + } + report.Verified = true + return report, nil +} + +func createReverseTarget(ctx context.Context, destination, kind, runID string) (err error) { + if err := os.MkdirAll(filepath.Dir(destination), 0700); err != nil { + return err + } + file, err := os.CreateTemp(filepath.Dir(destination), ".corescope-reverse-*.sqlite") + if err != nil { + return err + } + temp := file.Name() + if err := file.Close(); err != nil { + return err + } + defer func() { + for _, suffix := range []string{"", "-wal", "-shm", "-journal"} { + _ = os.Remove(temp + suffix) + } + }() + uri, err := legacy.WriterDSN(temp) + if err != nil { + return err + } + db, err := sql.Open(importSQLiteDriver, uri) + if err != nil { + return err + } + defer db.Close() + db.SetMaxOpenConns(1) + if kind == "telemetry" { + err = dbschema.ApplySQLite(db, func(string, ...interface{}) {}) + } else { + err = users.ApplySQLite(db) + } + if err != nil { + return err + } + if _, err := db.ExecContext(ctx, `PRAGMA foreign_keys=OFF`); err != nil { + return err + } + tables, err := layouts(kind) + if err != nil { + return err + } + tx, err := db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + for i := len(tables) - 1; i >= 0; i-- { + if _, err := tx.ExecContext(ctx, `DELETE FROM `+quote(tables[i].Name)); err != nil { + return err + } + } + if _, err := tx.ExecContext(ctx, `CREATE TABLE corescope_reverse_progress(table_name TEXT PRIMARY KEY,rows_copied INTEGER NOT NULL,sha256 TEXT NOT NULL,complete INTEGER NOT NULL); INSERT INTO corescope_reverse_progress VALUES('',0,?,0)`, runID); err != nil { + return err + } + if err := tx.Commit(); err != nil { + return err + } + var busy, log, checkpointed int + if err := db.QueryRowContext(ctx, `PRAGMA wal_checkpoint(TRUNCATE)`).Scan(&busy, &log, &checkpointed); err != nil { + return err + } + if busy != 0 { + return errors.New("private SQLite staging checkpoint is busy") + } + if err := db.Close(); err != nil { + return err + } + // Publish the complete native pending marker with the file. Link never + // overwrites an existing path, unlike rename on Unix. Both paths share a + // directory/filesystem and the closed database has no outstanding WAL. + return os.Link(temp, destination) +} + +func openReverseTarget(ctx context.Context, o reverseOptions, m reverseManifest) (*sql.DB, bool, error) { + if _, err := os.Lstat(m.Destination); errors.Is(err, os.ErrNotExist) { + if err := createReverseTarget(ctx, m.Destination, o.Kind, m.RunID); err != nil { + return nil, false, err + } + } else if err != nil { + return nil, false, err + } else if !o.Resume { + return nil, false, errors.New("SQLite destination exists; use the exact verified job to resume") + } + uri, err := legacy.WriterDSN(m.Destination) + if err != nil { + return nil, false, err + } + db, err := sql.Open(importSQLiteDriver, uri) + if err != nil { + return nil, false, err + } + db.SetMaxOpenConns(1) + fail := func(err error) (*sql.DB, bool, error) { db.Close(); return nil, false, err } + var progress int + if err := db.QueryRowContext(ctx, `SELECT count(*) FROM sqlite_schema WHERE name='corescope_reverse_progress' AND type='table'`).Scan(&progress); err != nil { + return fail(err) + } + if progress == 0 { + return db, true, nil + } + var id string + if err := db.QueryRowContext(ctx, `SELECT sha256 FROM corescope_reverse_progress WHERE table_name=''`).Scan(&id); err != nil || id != m.RunID { + return fail(errors.New("SQLite target belongs to another or incomplete preparation job")) + } + if _, err := db.ExecContext(ctx, `PRAGMA foreign_keys=OFF`); err != nil { + return fail(err) + } + return db, false, nil +} +func digestSQLiteTable(ctx context.Context, source, target *sql.DB, table importTable, m reverseManifest) (tableReport, error) { + report := tableReport{Table: table.Name} + _, names := columnNames(table) + for i := range names { + names[i] = "+" + names[i] + } + order, err := orderBy(source, table, false, false) + if err != nil { + return report, err + } + where := "" + var args []any + if table.Name == "_migrations" && len(m.AddedSQLiteMigrations) > 0 { + var params []string + for _, name := range m.AddedSQLiteMigrations { + params = append(params, "?") + args = append(args, name) + } + where = ` WHERE name NOT IN (` + strings.Join(params, ",") + `)` + } + rows, err := target.QueryContext(ctx, `SELECT `+strings.Join(names, ",")+` FROM `+quote(table.Name)+where+` ORDER BY `+order, args...) + if err != nil { + return report, err + } + defer rows.Close() + h := sha256.New() + for rows.Next() { + values, err := scanCanonicalRow(rows, table) + if err != nil { + return report, err + } + hashRow(h, values) + report.Rows++ + } + report.SHA256 = hex.EncodeToString(h.Sum(nil)) + return report, rows.Err() +} + +func scanCanonicalRow(rows *sql.Rows, table importTable) ([]any, error) { + values := make([]any, len(table.Columns)) + dest := make([]any, len(values)) + for i := range dest { + dest[i] = &values[i] + } + if err := rows.Scan(dest...); err != nil { + return nil, err + } + for i, col := range table.Columns { + value, err := canonical(values[i], col.Type) + if err != nil { + return nil, fmt.Errorf("column %s: %w", col.Name, err) + } + if f, ok := value.(float64); ok && math.IsNaN(f) { + return nil, errors.New("SQLite cannot preserve a PostgreSQL NaN; source was not changed") + } + values[i] = value + } + return values, nil +} + +func copyReverseTable(ctx context.Context, source, target *sql.DB, table importTable, o reverseOptions, m reverseManifest, verifyOnly bool) (tableReport, error) { + report := tableReport{Table: table.Name} + existing, err := digestSQLiteTable(ctx, source, target, table, m) + if err != nil { + return report, err + } + var copied int64 + var stored string + var complete bool + if verifyOnly { + copied, stored, complete = existing.Rows, existing.SHA256, true + } else { + err := target.QueryRowContext(ctx, `SELECT rows_copied,sha256,complete FROM corescope_reverse_progress WHERE table_name=?`, table.Name).Scan(&copied, &stored, &complete) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return report, err + } + if stored == "" { + sum := sha256.Sum256(nil) + stored = hex.EncodeToString(sum[:]) + } + if existing.Rows != copied || existing.SHA256 != stored { + return report, errors.New("SQLite destination differs from committed copy progress") + } + } + _, names := columnNames(table) + order, err := orderBy(source, table, true, false) + if err != nil { + return report, err + } + rows, err := source.QueryContext(ctx, `SELECT `+strings.Join(names, ",")+` FROM `+quote(table.Name)+` ORDER BY `+order) + if err != nil { + return report, err + } + defer rows.Close() + placeholders := make([]string, len(names)) + for i := range placeholders { + placeholders[i] = fmt.Sprintf("?%d", i+1) + } + insert := `INSERT INTO ` + quote(table.Name) + ` (` + strings.Join(names, ",") + `) VALUES(` + strings.Join(placeholders, ",") + `)` + h := sha256.New() + var batch [][]any + bytes := 0 + flush := func(done bool) error { + if verifyOnly { + return nil + } + if len(batch) == 0 && !done { + return nil + } + tx, err := target.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + stmt, err := tx.PrepareContext(ctx, insert) + if err != nil { + return err + } + defer stmt.Close() + for _, values := range batch { + if _, err := stmt.ExecContext(ctx, values...); err != nil { + return err + } + } + if _, err := tx.ExecContext(ctx, `INSERT INTO corescope_reverse_progress VALUES(?,?,?,?) ON CONFLICT(table_name) DO UPDATE SET rows_copied=excluded.rows_copied,sha256=excluded.sha256,complete=excluded.complete`, table.Name, report.Rows, hex.EncodeToString(h.Sum(nil)), done); err != nil { + return err + } + if err := tx.Commit(); err != nil { + return err + } + wrote := len(batch) > 0 + batch = nil + bytes = 0 + if wrote && o.afterBatch != nil { + return o.afterBatch() + } + return nil + } + for rows.Next() { + values, err := scanCanonicalRow(rows, table) + if err != nil { + return report, err + } + hashRow(h, values) + report.Rows++ + if report.Rows <= copied { + if report.Rows == copied && hex.EncodeToString(h.Sum(nil)) != stored { + return report, errors.New("source prefix differs from reverse copy progress") + } + continue + } + if complete { + return report, errors.New("completed SQLite table has missing source rows") + } + batch = append(batch, values) + for _, value := range values { + if s, ok := value.(string); ok { + bytes += len(s) + } else { + bytes += 8 + } + } + if len(batch) >= o.BatchSize || bytes >= 8*1024*1024 { + if err := flush(false); err != nil { + return report, err + } + } + } + if err := rows.Err(); err != nil { + return report, err + } + if report.Rows < copied { + return report, errors.New("SQLite target contains extra rows") + } + if err := flush(true); err != nil { + return report, err + } + report.SHA256 = hex.EncodeToString(h.Sum(nil)) + got, err := digestSQLiteTable(ctx, source, target, table, m) + if err != nil { + return report, err + } + if got != report { + return report, errors.New("SQLite count or logical digest verification failed") + } + return report, nil +} + +func assertSQLiteData(ctx context.Context, db *sql.DB, kind string) error { + if err := assertSQLiteIntegrity(ctx, db); err != nil { + return err + } + if kind == "telemetry" { + return dbschema.AssertSQLiteReady(db) + } + return users.AssertSQLiteReady(db) +} + +func assertSQLiteIntegrity(ctx context.Context, db *sql.DB) error { + var integrity string + if err := db.QueryRowContext(ctx, `PRAGMA quick_check`).Scan(&integrity); err != nil { + return err + } + if integrity != "ok" { + return errors.New("SQLite target failed integrity check") + } + rows, err := db.QueryContext(ctx, `PRAGMA foreign_key_check`) + if err != nil { + return err + } + invalid := rows.Next() + rowErr := rows.Err() + rows.Close() + if rowErr != nil { + return rowErr + } + if invalid { + return errors.New("SQLite target relationships failed verification") + } + return nil +} diff --git a/cmd/migrate/reverse_test.go b/cmd/migrate/reverse_test.go new file mode 100644 index 000000000..d0727211c --- /dev/null +++ b/cmd/migrate/reverse_test.go @@ -0,0 +1,190 @@ +package main + +import ( + "context" + "errors" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/pgutil" + "github.com/meshcore-analyzer/users" +) + +func TestReverseTelemetryPreservesRowsAndFutureIDs(t *testing.T) { + dsn := postgresDatabase(t) + ctx := context.Background() + _, err := importSQLite(ctx, importOptions{Source: telemetrySource(t), DatabaseURL: dsn, StateDir: t.TempDir(), Kind: "telemetry"}) + if err != nil { + t.Fatal(err) + } + if _, err := finalizeImport(ctx, dsn, "telemetry"); err != nil { + t.Fatal(err) + } + pg := openImportDB(t, dsn) + if _, err := pg.Exec(`INSERT INTO _async_migrations(name,status,started_at,ended_at,error) VALUES('obs_observer_ts_idx_v1','done','raw started','raw ended',NULL)`); err != nil { + t.Fatal(err) + } + for _, q := range []string{`SELECT setval(pg_get_serial_sequence('transmissions','id'),100500,false)`, `UPDATE observations SET score=3.5,resolved_path=' {"raw": true} ' WHERE id=90`, `UPDATE transmissions SET first_seen='malformed date kept as text' WHERE id=80`} { + if _, err := pg.Exec(q); err != nil { + t.Fatal(err) + } + } + dest := filepath.Join(t.TempDir(), "reverse # % é.sqlite") + options := reverseOptions{DatabaseURL: dsn, Destination: dest, StateDir: t.TempDir(), Kind: "telemetry", BatchSize: 2} + injected := false + options.afterBatch = func() error { + if !injected { + injected = true + return errors.New("interrupt after committed batch") + } + return nil + } + if _, err := reversePostgres(ctx, options); err == nil || !injected { + t.Fatal("expected interruption after committed batch", err) + } + options.Resume = true + options.afterBatch = nil + report, err := reversePostgres(ctx, options) + if err != nil || !report.Verified { + t.Fatal("resume reverse", err) + } + db, err := openSQLite(dest, "rw") + if err != nil { + t.Fatal(err) + } + defer db.Close() + if err := dbschema.AssertSQLiteReady(db); err != nil { + t.Fatal(err) + } + var indexCount int + if err := db.QueryRow(`SELECT count(*) FROM sqlite_master WHERE type='index' AND name='idx_observations_observer_idx_timestamp'`).Scan(&indexCount); err != nil || indexCount != 1 { + t.Fatal("copied done ledger lacks its native index", err) + } + var status, started, ended string + if err := db.QueryRow(`SELECT status,started_at,ended_at FROM _async_migrations WHERE name='obs_observer_ts_idx_v1'`).Scan(&status, &started, &ended); err != nil || status != "done" || started != "raw started" || ended != "raw ended" { + t.Fatal("source migration ledger changed", err) + } + var node, parent, unused int + var plan string + if err := db.QueryRow(`EXPLAIN QUERY PLAN SELECT timestamp FROM observations WHERE observer_idx=999 ORDER BY timestamp`).Scan(&node, &parent, &unused, &plan); err != nil || !strings.Contains(plan, "idx_observations_observer_idx_timestamp") { + t.Fatal("observer/time query cannot use native index", plan, err) + } + var n int + if err := db.QueryRow(`SELECT count(*) FROM observations WHERE (id=0 AND observer_idx=0 AND path_json IS NULL) OR (id=90 AND observer_idx=999 AND path_json='' AND score=3.5 AND resolved_path=' {"raw": true} ') OR (id=91 AND observer_idx IS NULL)`).Scan(&n); err != nil || n != 3 { + t.Fatal("row parity", n, err) + } + var id int64 + if err := db.QueryRow(`INSERT INTO transmissions(raw_hex,hash,first_seen) VALUES('00','new','2026-01-03') RETURNING id`).Scan(&id); err != nil || id != 100500 { + t.Fatal("is_called=false future ID", id, err) + } + if err := db.QueryRow(`INSERT INTO observers(id) VALUES('future') RETURNING rowid`).Scan(&id); err != nil || id <= 999 { + t.Fatal("observer orphan reuse", id, err) + } + if _, err := os.Stat(filepath.Join(options.StateDir, "telemetry", "recovery.pg_dump")); err != nil { + t.Fatal("native recovery missing", err) + } + // Verify the actual native recovery archive, not only its filename/hash. + restored := postgresDatabase(t) + config, err := pgutil.ParseConfig(restored) + if err != nil { + t.Fatal(err) + } + command := exec.Command("pg_restore", "--no-password", "--no-owner", "--no-privileges", "--exit-on-error", "--dbname", config.Database, filepath.Join(options.StateDir, "telemetry", "recovery.pg_dump")) + command.Env, err = pgutil.CommandEnv(restored) + if err != nil { + t.Fatal(err) + } + if err := command.Run(); err != nil { + t.Fatal("native PostgreSQL recovery restore failed") + } + var manifest reverseManifest + if err := readPrivateJSON(filepath.Join(options.StateDir, "telemetry", "reverse.json"), &manifest); err != nil { + t.Fatal(err) + } + recovery := openImportDB(t, restored) + tables, err := layouts("telemetry") + if err != nil { + t.Fatal(err) + } + for i, table := range tables { + got, err := digestTarget(ctx, recovery, table) + if err != nil || got != manifest.Source[i] { + t.Fatal("native recovery data mismatch", table.Name, err) + } + } +} + +func TestReverseSequenceMutationLeavesNativePendingMarker(t *testing.T) { + owner := postgresDatabase(t) + ctx := context.Background() + db := openImportDB(t, owner) + if err := dbschema.ApplyPostgres(db, nil); err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`INSERT INTO observers(id) VALUES('only-observer')`); err != nil { + t.Fatal(err) + } + dest := filepath.Join(t.TempDir(), "pending.sqlite") + options := reverseOptions{DatabaseURL: owner, Destination: dest, StateDir: t.TempDir(), Kind: "telemetry", BatchSize: 1, afterBatch: func() error { + _, err := db.Exec(`SELECT nextval(pg_get_serial_sequence('observers','rowid'))`) + return err + }} + if _, err := reversePostgres(ctx, options); err == nil { + t.Fatal("changed source sequence accepted") + } + target, err := openSQLite(dest, "ro") + if err != nil { + t.Fatal(err) + } + defer target.Close() + if err := dbconfig.AssertSQLiteImportComplete(target); !errors.Is(err, dbconfig.ErrSQLiteImportIncomplete) { + t.Fatal("partial reverse target can be adopted elsewhere", err) + } +} + +func TestReverseAccountsPreservesAuthenticationAndIdentity(t *testing.T) { + dsn := postgresDatabase(t) + ctx := context.Background() + _, err := importSQLite(ctx, importOptions{Source: accountSource(t, 5), DatabaseURL: dsn, StateDir: t.TempDir(), Kind: "accounts"}) + if err != nil { + t.Fatal(err) + } + if _, err := finalizeImport(ctx, dsn, "accounts"); err != nil { + t.Fatal(err) + } + pg := openImportDB(t, dsn) + for _, q := range []string{`SELECT setval(pg_get_serial_sequence('mail_events','id'),9001,false)`, `INSERT INTO audit_log(at,actor_user_id,action,target_user_id) VALUES(2,888,'deleted',999)`} { + if _, err := pg.Exec(q); err != nil { + t.Fatal(err) + } + } + dest := filepath.Join(t.TempDir(), "accounts.sqlite") + options := reverseOptions{DatabaseURL: dsn, Destination: dest, StateDir: t.TempDir(), Kind: "accounts", BatchSize: 2} + report, err := reversePostgres(ctx, options) + if err != nil || !report.Verified { + t.Fatal("reverse accounts", err) + } + db, err := openSQLite(dest, "rw") + if err != nil { + t.Fatal(err) + } + defer db.Close() + if err := users.AssertSQLiteReady(db); err != nil { + t.Fatal(err) + } + var id int64 + if err := db.QueryRow(`SELECT id FROM mail_events WHERE mail_id=4`).Scan(&id); err != nil || id != 900 { + t.Fatal("event id", id, err) + } + if err := db.QueryRow(`INSERT INTO mail_events(mail_id,event,at) VALUES(4,'new',99) RETURNING id`).Scan(&id); err != nil || id != 9001 { + t.Fatal("event sequence", id, err) + } + if err := db.QueryRow(`INSERT INTO users(email,display_name,password_hash,created_at) VALUES('new@example.invalid','','hash',3) RETURNING id`).Scan(&id); err != nil || id != 1000 { + t.Fatal("historical user references", id, err) + } +} diff --git a/cmd/migrate/source.go b/cmd/migrate/source.go new file mode 100644 index 000000000..2c723628b --- /dev/null +++ b/cmd/migrate/source.go @@ -0,0 +1,119 @@ +package main + +import ( + "context" + "crypto/sha256" + "database/sql" + "errors" + "io" + "net/url" + "os" + + "github.com/mattn/go-sqlite3" + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/sqliteutil" +) + +const importSQLiteDriver = "corescope_sqlite_import" +const maxImportRowBytes = 64 * 1024 * 1024 + +func init() { + // Apply the bound before any SQLite row reaches the Go driver. A check + // after Scan would already have allocated an arbitrarily large value. + sql.Register(importSQLiteDriver, &sqlite3.SQLiteDriver{ConnectHook: func(conn *sqlite3.SQLiteConn) error { + conn.SetLimit(sqlite3.SQLITE_LIMIT_LENGTH, maxImportRowBytes) + return nil + }}) +} + +// snapshotSource adds offline source immutability checks to the shared native +// backup. Snapshot itself is also used by live backups that permit writers. +func snapshotSource(ctx context.Context, source, destination string) error { + before, err := sourceFingerprintContext(ctx, source) + if err != nil { + return err + } + if err := sqliteutil.Snapshot(ctx, source, destination); err != nil { + return err + } + after, err := sourceFingerprintContext(ctx, source) + if err != nil || before != after { + for _, suffix := range []string{"", "-wal", "-shm", "-journal"} { + _ = os.Remove(destination + suffix) + } + if err != nil { + return err + } + return errors.New("SQLite source changed during snapshot; stop all instance writers before retrying") + } + return nil +} + +func sqliteURL(path, mode string) (string, error) { + return dbconfig.SQLiteURI(path, url.Values{"mode": {mode}, "_busy_timeout": {"5000"}}) +} + +func openSQLite(path, mode string) (*sql.DB, error) { + uri, err := sqliteURL(path, mode) + if err != nil { + return nil, err + } + return sql.Open(importSQLiteDriver, uri) +} + +// Hash the main file and committed-write log without loading either into RAM. +// Shared-memory coordination files are not application data. +func sourceFingerprint(path string) ([2][32]byte, error) { + return sourceFingerprintContext(context.Background(), path) +} + +type contextReader struct { + ctx context.Context + r io.Reader +} + +func (r contextReader) Read(p []byte) (int, error) { + if err := r.ctx.Err(); err != nil { + return 0, err + } + return r.r.Read(p) +} + +func sourceFingerprintContext(ctx context.Context, path string) ([2][32]byte, error) { + var result [2][32]byte + if err := ctx.Err(); err != nil { + return result, err + } + for i, suffix := range []string{"", "-wal"} { + f, err := os.Open(path + suffix) + if i == 1 && errors.Is(err, os.ErrNotExist) { + continue + } + if err != nil { + return result, err + } + if i == 1 { + info, err := f.Stat() + if err != nil { + f.Close() + return result, err + } + // Read-only WAL-mode opens can create/remove a zero-byte sidecar. + // Every nonempty byte, including headers, still affects identity. + if info.Size() == 0 { + if err := f.Close(); err != nil { + return result, err + } + continue + } + } + h := sha256.New() + _, copyErr := io.CopyBuffer(h, contextReader{ctx, f}, make([]byte, 128*1024)) + closeErr := f.Close() + if err := errors.Join(copyErr, closeErr); err != nil { + return result, err + } + copy(result[i][:], h.Sum(nil)) + } + return result, nil +} diff --git a/cmd/migrate/source_test.go b/cmd/migrate/source_test.go new file mode 100644 index 000000000..32f790946 --- /dev/null +++ b/cmd/migrate/source_test.go @@ -0,0 +1,188 @@ +package main + +import ( + "context" + "crypto/sha256" + "database/sql" + "os" + "path/filepath" + "testing" + + "github.com/mattn/go-sqlite3" +) + +func TestSnapshotSourceIncludesWALWithoutChangingSource(t *testing.T) { + dir := t.TempDir() + source := filepath.Join(dir, "source.db") + destination := filepath.Join(dir, "recovery.db") + db, err := sql.Open("sqlite3", source+"?_journal_mode=WAL") + if err != nil { + t.Fatal(err) + } + defer db.Close() + db.SetMaxOpenConns(1) + for _, statement := range []string{ + `PRAGMA wal_autocheckpoint=0`, + `CREATE TABLE records (id INTEGER PRIMARY KEY, value TEXT)`, + `INSERT INTO records VALUES (0, NULL), (90, '')`, + `CREATE TABLE identities (key TEXT PRIMARY KEY)`, + `INSERT INTO identities(rowid,key) VALUES (0,'zero'),(900,'gap')`, + } { + if _, err := db.Exec(statement); err != nil { + t.Fatal(err) + } + } + before := make(map[string][32]byte) + for _, path := range []string{source, source + "-wal"} { + content, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + before[path] = sha256.Sum256(content) + } + if err := snapshotSource(context.Background(), source, destination); err != nil { + t.Fatalf("WAL-aware recovery snapshot: %v", err) + } + for path, expected := range before { + content, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if sha256.Sum256(content) != expected { + t.Fatalf("source was modified: %s", filepath.Base(path)) + } + } + copyDB, err := sql.Open("sqlite3", destination) + if err != nil { + t.Fatal(err) + } + defer copyDB.Close() + var count int + if err := copyDB.QueryRow(`SELECT COUNT(*) FROM records WHERE (id=0 AND value IS NULL) OR (id=90 AND value='')`).Scan(&count); err != nil { + t.Fatal(err) + } + if count != 2 { + t.Fatalf("snapshot lost committed WAL rows or null/empty values: got %d", count) + } + if err := copyDB.QueryRow(`SELECT COUNT(*) FROM identities WHERE (rowid=0 AND key='zero') OR (rowid=900 AND key='gap')`).Scan(&count); err != nil { + t.Fatal(err) + } + if count != 2 { + t.Fatal("snapshot changed hidden rowids") + } +} + +func TestSnapshotSourceDoesNotOverwriteRecoveryOrOriginal(t *testing.T) { + dir := t.TempDir() + source := filepath.Join(dir, "source.db") + db, err := sql.Open("sqlite3", source) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`CREATE TABLE records(id INTEGER)`); err != nil { + t.Fatal(err) + } + if err := db.Close(); err != nil { + t.Fatal(err) + } + for _, destination := range []string{source, filepath.Join(dir, "existing.db")} { + if destination != source { + if err := os.WriteFile(destination, []byte("existing recovery"), 0o600); err != nil { + t.Fatal(err) + } + } + before, err := os.ReadFile(destination) + if err != nil { + t.Fatal(err) + } + if err := snapshotSource(context.Background(), source, destination); err == nil { + t.Fatal("overwriting an existing file was accepted") + } + after, err := os.ReadFile(destination) + if err != nil { + t.Fatal(err) + } + if sha256.Sum256(before) != sha256.Sum256(after) { + t.Fatal("existing file was changed") + } + } +} + +func TestSnapshotSourceFailureLeavesNoPartialCopy(t *testing.T) { + for _, cancelled := range []bool{false, true} { + dir := t.TempDir() + source, destination := filepath.Join(dir, "source.db"), filepath.Join(dir, "recovery.db") + if err := os.WriteFile(source, []byte("not a SQLite database"), 0o600); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + if cancelled { + cancel() + } + if err := snapshotSource(ctx, source, destination); err == nil { + t.Fatal("invalid/cancelled snapshot accepted") + } + cancel() + if _, err := os.Stat(destination); !os.IsNotExist(err) { + t.Fatalf("partial output retained: %v", err) + } + } +} + +func TestSourceFingerprintEmptyWALMatchesAbsentWAL(t *testing.T) { + source := filepath.Join(t.TempDir(), "source.db") + if err := os.WriteFile(source, []byte("stable database bytes"), 0o600); err != nil { + t.Fatal(err) + } + before, err := sourceFingerprint(source) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(source+"-wal", nil, 0o600); err != nil { + t.Fatal(err) + } + empty, err := sourceFingerprint(source) + if err != nil { + t.Fatal(err) + } + if empty != before { + t.Fatal("an empty WAL changed the source identity") + } + if err := os.WriteFile(source+"-wal", []byte("nonempty WAL bytes must be observed"), 0o600); err != nil { + t.Fatal(err) + } + nonempty, err := sourceFingerprint(source) + if err != nil { + t.Fatal(err) + } + if nonempty == before { + t.Fatal("nonempty WAL bytes were ignored") + } +} + +func TestSQLiteSourceLengthLimitPrecedesScan(t *testing.T) { + db, err := sql.Open(importSQLiteDriver, ":memory:") + if err != nil { + t.Fatal(err) + } + defer db.Close() + conn, err := db.Conn(context.Background()) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + if err := conn.Raw(func(driver any) error { + sqlite := driver.(*sqlite3.SQLiteConn) + if got := sqlite.GetLimit(sqlite3.SQLITE_LIMIT_LENGTH); got != maxImportRowBytes { + t.Fatalf("native SQLite length limit=%d", got) + } + sqlite.SetLimit(sqlite3.SQLITE_LIMIT_LENGTH, 1024) + return nil + }); err != nil { + t.Fatal(err) + } + var data []byte + if err := conn.QueryRowContext(context.Background(), `SELECT zeroblob(2048)`).Scan(&data); err == nil { + t.Fatal("native length limit did not reject a large result before Scan") + } +} diff --git a/cmd/migrate/sqlite_derived.go b/cmd/migrate/sqlite_derived.go new file mode 100644 index 000000000..b03fdbb64 --- /dev/null +++ b/cmd/migrate/sqlite_derived.go @@ -0,0 +1,108 @@ +package main + +import ( + "database/sql" + _ "embed" + "encoding/json" + "errors" + "fmt" + "slices" +) + +// Frozen SQLite layout installed by #2214. Do not regenerate this allowlist +// when runtime DDL changes: a new supported layout needs a separate version +// and migration tests. Exact SQL preserves literals, column types, NOT NULL, +// CHECK, primary-key order, WITHOUT ROWID, and trigger bodies. +// +//go:embed sqlite_hop_index_v1.json +var sqliteHopIndexV1JSON []byte + +type sqliteDerivedObject struct { + Type string `json:"type"` + Name string `json:"name"` + Table string `json:"table"` + SQL string `json:"sql"` +} + +type derivedTableReport struct { + Table string `json:"table"` + SchemaVersion int `json:"schema_version"` + Reason string `json:"reason"` +} + +// knownSQLiteHopIndex recognizes the complete schema even when its data +// backfill is partial. No derived content is trusted or promoted to canonical +// data. Unknown indexes on either table also fail this exact-object comparison; +// v1 has only the implicit WITHOUT ROWID primary key, no standalone indexes. +func knownSQLiteHopIndex(db *sql.DB) ([]sqliteDerivedObject, error) { + var known struct { + Version int `json:"version"` + Objects []sqliteDerivedObject `json:"objects"` + } + if err := json.Unmarshal(sqliteHopIndexV1JSON, &known); err != nil || known.Version != 1 || len(known.Objects) != 5 { + return nil, errors.New("invalid built-in SQLite hop-index layout") + } + rows, err := db.Query(`SELECT type,name,tbl_name,COALESCE(sql,'') FROM sqlite_schema + WHERE name LIKE 'observation_hops%' OR name LIKE 'trg_observation_hops%' + OR tbl_name IN ('observation_hops','observation_hops_coverage') + ORDER BY name`) + if err != nil { + return nil, err + } + defer rows.Close() + var got []sqliteDerivedObject + for rows.Next() { + var o sqliteDerivedObject + if err := rows.Scan(&o.Type, &o.Name, &o.Table, &o.SQL); err != nil { + return nil, err + } + got = append(got, o) + if len(got) > len(known.Objects) { + return nil, errors.New("unsupported SQLite hop-index objects; source data was not discarded") + } + } + if err := rows.Err(); err != nil { + return nil, err + } + if len(got) > 0 && !slices.Equal(got, known.Objects) { + return nil, errors.New("unsupported SQLite hop-index layout (expected complete v1 tables, constraints, indexes and triggers); source data was not discarded") + } + return got, nil +} + +func sqliteDerivedExclusions(db *sql.DB, kind string) ([]derivedTableReport, error) { + if kind != "telemetry" { + return nil, nil + } + objects, err := knownSQLiteHopIndex(db) + if err != nil { + return nil, err + } + var reports []derivedTableReport + for _, o := range objects { + if o.Type == "table" { + reports = append(reports, derivedTableReport{ + Table: o.Name, SchemaVersion: 1, + Reason: "derived SQLite hop-index rows and coverage excluded; rebuild through the SQLite ingestor", + }) + } + } + return reports, nil +} + +func recoveryDerivedExclusions(rawPath, kind string) ([]derivedTableReport, error) { + db, err := openSQLite(rawPath, "ro") + if err != nil { + return nil, err + } + defer db.Close() + db.SetMaxOpenConns(1) + if _, err := db.Exec(`PRAGMA trusted_schema=OFF`); err != nil { + return nil, err + } + reports, err := sqliteDerivedExclusions(db, kind) + if err != nil { + return nil, fmt.Errorf("validate recovery derived schema: %w", err) + } + return reports, nil +} diff --git a/cmd/migrate/sqlite_hop_index_v1.json b/cmd/migrate/sqlite_hop_index_v1.json new file mode 100644 index 000000000..2a572bb14 --- /dev/null +++ b/cmd/migrate/sqlite_hop_index_v1.json @@ -0,0 +1,35 @@ +{ + "version": 1, + "objects": [ + { + "type": "table", + "name": "observation_hops", + "table": "observation_hops", + "sql": "CREATE TABLE observation_hops (\n\t\t\thop TEXT NOT NULL,\n\t\t\ttimestamp INTEGER NOT NULL,\n\t\t\tobservation_id INTEGER NOT NULL,\n\t\t\tPRIMARY KEY (hop, timestamp, observation_id)\n\t\t) WITHOUT ROWID" + }, + { + "type": "table", + "name": "observation_hops_coverage", + "table": "observation_hops_coverage", + "sql": "CREATE TABLE observation_hops_coverage (\n\t\t\tid INTEGER PRIMARY KEY CHECK (id = 1),\n\t\t\tsince INTEGER NOT NULL\n\t\t)" + }, + { + "type": "trigger", + "name": "trg_observation_hops_ad", + "table": "observations", + "sql": "CREATE TRIGGER trg_observation_hops_ad\n\t\tAFTER DELETE ON observations\n\t\tWHEN OLD.path_json IS NOT NULL AND json_valid(OLD.path_json)\n\t\t AND OLD.timestamp \u003e= (SELECT since FROM observation_hops_coverage WHERE id = 1)\n\t\tBEGIN\n\t\t\tDELETE FROM observation_hops\n\t\t\tWHERE observation_id = OLD.id AND timestamp = OLD.timestamp\n\t\t\t AND hop IN (SELECT UPPER(value) FROM json_each(OLD.path_json) WHERE type = 'text');\n\t\tEND" + }, + { + "type": "trigger", + "name": "trg_observation_hops_ai", + "table": "observations", + "sql": "CREATE TRIGGER trg_observation_hops_ai\n\t\tAFTER INSERT ON observations\n\t\tWHEN NEW.path_json IS NOT NULL AND json_valid(NEW.path_json)\n\t\t AND NEW.timestamp \u003e= (SELECT since FROM observation_hops_coverage WHERE id = 1)\n\t\tBEGIN\n\t\t\tINSERT OR IGNORE INTO observation_hops (hop, timestamp, observation_id)\n\t\t\tSELECT DISTINCT UPPER(value), NEW.timestamp, NEW.id FROM json_each(NEW.path_json) WHERE type = 'text';\n\t\tEND" + }, + { + "type": "trigger", + "name": "trg_observation_hops_au", + "table": "observations", + "sql": "CREATE TRIGGER trg_observation_hops_au\n\t\tAFTER UPDATE OF path_json ON observations\n\t\tWHEN IFNULL(OLD.path_json, '') != IFNULL(NEW.path_json, '')\n\t\tBEGIN\n\t\t\tDELETE FROM observation_hops\n\t\t\tWHERE observation_id = OLD.id AND timestamp = OLD.timestamp\n\t\t\t AND json_valid(OLD.path_json)\n\t\t\t AND OLD.timestamp \u003e= (SELECT since FROM observation_hops_coverage WHERE id = 1)\n\t\t\t AND hop IN (SELECT UPPER(value) FROM json_each(OLD.path_json) WHERE type = 'text');\n\t\t\tINSERT OR IGNORE INTO observation_hops (hop, timestamp, observation_id)\n\t\t\tSELECT DISTINCT UPPER(value), NEW.timestamp, NEW.id FROM json_each(NEW.path_json)\n\t\t\tWHERE json_valid(NEW.path_json) AND type = 'text'\n\t\t\t AND NEW.timestamp \u003e= (SELECT since FROM observation_hops_coverage WHERE id = 1);\n\t\tEND" + } + ] +} diff --git a/cmd/migrate/storage_command.go b/cmd/migrate/storage_command.go new file mode 100644 index 000000000..5e28a2176 --- /dev/null +++ b/cmd/migrate/storage_command.go @@ -0,0 +1,492 @@ +package main + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/jackc/pgx/v5/pgconn" + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/dbschema/legacy" + "github.com/meshcore-analyzer/pgutil" + "github.com/meshcore-analyzer/users" +) + +type storageOptions struct { + Action, SelectionFile, Backend, SQLitePath, UsersSQLitePath string + OwnerURL, UsersOwnerURL, StateDir, JobID, ConfigFile, ConfigDir string + Offline bool + afterStore func(string) error + beforeCommit func() error +} + +func firstSetting(values ...string) string { + for _, value := range values { + if value != "" { + return value + } + } + return "" +} + +func storageActionInputs(o storageOptions, recorded bool) (dbconfig.StorageInputs, error) { + var raw dbconfig.StorageInputs + cwd, err := os.Getwd() + if err != nil { + return raw, err + } + configDir, err := filepath.Abs(firstSetting(o.ConfigDir, ".")) + if err != nil { + return raw, err + } + if o.ConfigFile != "" { + raw, err = dbconfig.ReadStorageConfig(o.ConfigFile) + if err != nil { + return raw, err + } + } else { + for _, path := range []string{filepath.Join(configDir, "config.json"), filepath.Join(configDir, "data", "config.json")} { + config, e := dbconfig.ReadStorageConfig(path) + if e == nil { + raw = config + err = nil + break + } + if !errors.Is(e, os.ErrNotExist) { + return raw, e + } + } + if err != nil { + return raw, err + } + } + envPath := os.Getenv("DB_PATH") + if !recorded && o.SQLitePath == "" && envPath != "" && raw.DBPath != "" { + a, e := filepath.Abs(envPath) + if e != nil { + return raw, e + } + b, e := filepath.Abs(raw.DBPath) + if e != nil { + return raw, e + } + if a != b { + return raw, errors.New("legacy config dbPath and DB_PATH differ between services; supply one explicit -sqlite-path before adoption") + } + } + raw.BaseDir = cwd + raw.Backend = dbconfig.Backend(firstSetting(o.Backend, string(raw.Backend))) + raw.EnvBackend = dbconfig.Backend(os.Getenv("CORESCOPE_DB_BACKEND")) + if o.Backend != "" { + raw.EnvBackend = raw.Backend + } + raw.DBPath = firstSetting(o.SQLitePath, envPath, raw.DBPath) + raw.UsersDBPath = firstSetting(o.UsersSQLitePath, raw.UsersDBPath) + raw.DatabaseURL = firstSetting(os.Getenv("CORESCOPE_DATABASE_URL"), raw.DatabaseURL) + raw.ReaderDatabaseURL = os.Getenv("CORESCOPE_READER_DATABASE_URL") + raw.WriterDatabaseURL = os.Getenv("CORESCOPE_WRITER_DATABASE_URL") + raw.UsersDatabaseURL = firstSetting(os.Getenv("CORESCOPE_USERS_DATABASE_URL"), raw.UsersDatabaseURL) + raw.ApprovedChannelsDatabaseURL = firstSetting(os.Getenv("CORESCOPE_APPROVED_CHANNELS_DATABASE_URL"), raw.ApprovedChannelsDatabaseURL) + raw.StateDir = filepath.Dir(o.SelectionFile) // stable discovery, not the recovery directory + raw.FreshInstall = true // only permits resolution; native facts below authorize initialization + sqliteDefault := raw.Backend == dbconfig.SQLite || raw.EnvBackend == dbconfig.SQLite || (raw.Backend == "" && raw.EnvBackend == "" && raw.DatabaseURL == "" && raw.ReaderDatabaseURL == "" && raw.WriterDatabaseURL == "" && raw.UsersDatabaseURL == "" && raw.ApprovedChannelsDatabaseURL == "" && o.OwnerURL == "" && o.UsersOwnerURL == "") + if raw.DBPath == "" && sqliteDefault { + raw.DBPath = filepath.Join(configDir, "data", "meshcore.db") + } + return raw, nil +} + +func runStorageAction(ctx context.Context, o storageOptions, out io.Writer) (err error) { + defer func() { + var pgError *pgconn.PgError + if errors.As(err, &pgError) { + err = fmt.Errorf("storage operation failed (SQLSTATE %s); keep writers stopped and inspect the private recovery job", pgError.Code) + } + }() + if !filepath.IsAbs(o.SelectionFile) || filepath.Clean(o.SelectionFile) != o.SelectionFile || filepath.Base(o.SelectionFile) != "storage-selection.json" { + return errors.New("provide an absolute normalized -selection-file ending in storage-selection.json") + } + status, inspectErr := dbconfig.InspectSelection(o.SelectionFile) + if o.Action == "status" { + if err := json.NewEncoder(out).Encode(status); err != nil { + return err + } + return inspectErr + } + if !o.Offline { + return errors.New("storage setup and switching require -offline after stopping all telemetry and account writers") + } + if o.StateDir == "" { + o.StateDir = filepath.Join(filepath.Dir(o.SelectionFile), "migration") + } + switch o.Action { + case "setup", "upgrade", "adopt", "init", "switch", "resume", "abort": + default: + return errors.New("unsupported storage action; use -help") + } + if inspectErr != nil { + return inspectErr + } + o.OwnerURL = firstSetting(o.OwnerURL, os.Getenv("CORESCOPE_DATABASE_URL")) + o.UsersOwnerURL = firstSetting(o.UsersOwnerURL, os.Getenv("CORESCOPE_USERS_OWNER_DATABASE_URL")) + if o.Action == "resume" || o.Action == "abort" || o.Action == "switch" { + if err := runStorageSwitch(ctx, o); err != nil { + return err + } + } else { + if status.State == "pending" { + return dbconfig.ErrSelectionInProgress + } + if err := setupStorage(ctx, o, status.State == "ready"); err != nil { + return err + } + } + status, err = dbconfig.InspectSelection(o.SelectionFile) + if err != nil { + return err + } + return json.NewEncoder(out).Encode(status) +} + +func setupStorage(ctx context.Context, o storageOptions, recorded bool) error { + if o.Action == "upgrade" && !recorded { + return errors.New("schema upgrade requires an existing ready PostgreSQL selection") + } + if recorded && o.Action == "init" { + return errors.New("installation already has a recorded backend; use a verified switch") + } + raw, err := storageActionInputs(o, recorded) + if err != nil { + return err + } + if recorded { + selection, lease, err := dbconfig.OpenSelection(o.SelectionFile) + if err != nil { + return err + } + defer lease.Close() + if o.Action == "upgrade" && (selection.Backend != dbconfig.Postgres || o.OwnerURL == "") { + return errors.New("schema upgrade requires an existing PostgreSQL selection and its owner credentials") + } + upgraded := false + if selection.Backend == dbconfig.Postgres && o.OwnerURL != "" && (o.Action == "setup" || o.Action == "upgrade") { + lease.Close() + if err := upgradeRecordedPostgres(ctx, o, selection, raw); err != nil { + return err + } + upgraded = true + } + if selection.Accounts == nil && o.Action == "setup" && (selection.Backend == dbconfig.SQLite || o.UsersOwnerURL != "" || raw.UsersDatabaseURL != "" || raw.ApprovedChannelsDatabaseURL != "") { + lease.Close() + return initializeAccounts(ctx, o, selection, raw) + } + if upgraded { + return nil + } + if selection.Backend == dbconfig.Postgres && o.OwnerURL != "" { + owners, err := bindOwnerURLs(selection, o.OwnerURL, o.UsersOwnerURL) + if err != nil { + return err + } + if err := grantRuntimeTargets(ctx, selection, owners, raw); err != nil { + return err + } + } + return validateRuntimeSelection(ctx, selection, raw) + } + ownerInputs := raw + // Owner endpoints are used only for schema operations. Runtime credentials + // stay separate and are validated against the resulting selected targets. + if o.OwnerURL != "" { + ownerInputs.DatabaseURL = o.OwnerURL + ownerInputs.ReaderDatabaseURL = "" + ownerInputs.WriterDatabaseURL = "" + } + if o.UsersOwnerURL != "" { + ownerInputs.UsersDatabaseURL = o.UsersOwnerURL + ownerInputs.ApprovedChannelsDatabaseURL = "" + } + storage, err := dbconfig.ResolveStorage(ownerInputs) + if err != nil { + return err + } + selected, err := dbconfig.NewSelection(storage) + if err != nil { + return err + } + if storage.Backend == dbconfig.SQLite { + legacyTelemetry, err := sqliteOccupied(ctx, storage.DBPath) + if err != nil { + return err + } + for _, store := range []struct{ kind, path string }{{"telemetry", storage.DBPath}, {"accounts", storage.UsersDBPath}} { + occupied, err := sqliteOccupied(ctx, store.path) + if err != nil { + return err + } + if store.kind == "accounts" && legacyTelemetry && !occupied { + selected.Accounts = nil + storage.UsersDBPath = "" + continue + } + if o.Action == "init" && occupied { + return errors.New("initialization target already contains SQLite schema; use setup/adopt for an existing installation") + } + if o.Action == "adopt" && store.kind == "telemetry" && !occupied { + return errors.New("adoption requires an existing telemetry database") + } + if err := applySQLiteTarget(ctx, store.path, store.kind); err != nil { + return err + } + } + } else { + if o.OwnerURL == "" { + return errors.New("unrecorded PostgreSQL setup requires its telemetry owner URL through CORESCOPE_DATABASE_URL") + } + if (raw.UsersDatabaseURL != "" || raw.ApprovedChannelsDatabaseURL != "") && o.UsersOwnerURL == "" { + return errors.New("PostgreSQL account setup requires CORESCOPE_USERS_OWNER_DATABASE_URL; runtime credentials are never owner credentials") + } + if err := validateInitialOwnerTargets(ctx, selected, storage); err != nil { + return err + } + storage, err = bindOwnerURLs(selected, o.OwnerURL, o.UsersOwnerURL) + if err != nil { + return err + } + if storage.UsersDatabaseURL != "" { + same, err := pgutil.SameDatabase(ctx, storage.WriterDatabaseURL, storage.UsersDatabaseURL) + if err != nil { + return err + } + if same { + return errors.New("telemetry and accounts require separate PostgreSQL databases") + } + } + knownSQLite := firstSetting(raw.DBPath, filepath.Join(filepath.Dir(o.SelectionFile), "meshcore.db")) + knownUsers := firstSetting(raw.UsersDBPath, filepath.Join(filepath.Dir(knownSQLite), "users.db")) + for _, store := range []struct{ kind, url, oldPath string }{{"telemetry", storage.WriterDatabaseURL, knownSQLite}, {"accounts", storage.UsersDatabaseURL, knownUsers}} { + occupied, err := sqliteOccupied(ctx, store.oldPath) + if err != nil { + return err + } + if store.url == "" { + if occupied { + return errors.New("an existing SQLite account store must be preserved; supply its PostgreSQL owner target and use a verified switch") + } + continue + } + db, err := pgutil.Open(store.url, false) + if err != nil { + return err + } + readyErr := assertPostgresReady(db, store.kind) + if readyErr == nil { + db.Close() + if o.Action == "init" { + return errors.New("PostgreSQL target is already initialized; use setup/adopt") + } + if occupied && o.Action != "adopt" { + if err := checkImportedSource(ctx, store.url, store.kind, store.oldPath); err != nil { + return errors.New("retained SQLite data needs its verified import or explicit adoption of the existing ready PostgreSQL store") + } + } + continue + } + var objects int + err = db.QueryRowContext(ctx, importDestinationObjects).Scan(&objects) + if err != nil { + db.Close() + return err + } + if objects != 0 || o.Action == "adopt" { + db.Close() + return errors.New("PostgreSQL target is not a ready supported store or a fresh empty schema") + } + if occupied { + db.Close() + return errors.New("refusing to initialize empty PostgreSQL while primary SQLite data exists; use a verified offline switch") + } + if raw.Backend != dbconfig.Postgres && raw.EnvBackend != dbconfig.Postgres { + db.Close() + return errors.New("fresh PostgreSQL initialization requires an explicit backend choice") + } + if store.kind == "telemetry" { + err = dbschema.ApplyPostgres(db, nil) + } else { + err = users.ApplyPostgres(db) + } + db.Close() + if err != nil { + return err + } + } + } + if selected.Backend == dbconfig.Postgres { + if err := grantRuntimeTargets(ctx, selected, storage, raw); err != nil { + return err + } + } + if err := validateRuntimeSelection(ctx, selected, raw); err != nil { + return err + } + _, err = dbconfig.AdoptSelection(o.SelectionFile, selected) + return err +} + +func sqliteOccupied(ctx context.Context, path string) (bool, error) { + if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) { + return false, nil + } else if err != nil { + return false, err + } + db, err := openSQLite(path, "ro") + if err != nil { + return false, err + } + defer db.Close() + var count int + if err := db.QueryRowContext(ctx, `SELECT count(*) FROM sqlite_schema WHERE name NOT LIKE 'sqlite_%'`).Scan(&count); err != nil { + return false, errors.New("existing SQLite target cannot be validated; preserve it instead of selecting an empty database") + } + return count != 0, nil +} + +func applySQLiteTarget(ctx context.Context, path, kind string) error { + if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { + return err + } + if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) { + f, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600) + if err != nil { + return err + } + if err := f.Close(); err != nil { + return err + } + } + uri, err := legacy.WriterDSN(path) + if err != nil { + return err + } + db, err := sql.Open(importSQLiteDriver, uri) + if err != nil { + return err + } + defer db.Close() + db.SetMaxOpenConns(1) + if err := dbconfig.AssertSQLiteImportComplete(db); err != nil { + return err + } + if kind == "telemetry" { + return dbschema.ApplySQLite(db, func(string, ...interface{}) {}) + } + return users.ApplySQLite(db) +} + +func assertPostgresReady(db *sql.DB, kind string) error { + if kind == "telemetry" { + return dbschema.AssertPostgresReady(db) + } + return users.AssertPostgresReady(db) +} + +func validateRuntimeSelection(ctx context.Context, selection dbconfig.Selection, raw dbconfig.StorageInputs) error { + bound, err := selection.ApplyTo(raw) + if err != nil { + return err + } + storage, err := dbconfig.ResolveStorage(bound) + if err != nil { + return err + } + if storage.Backend == dbconfig.SQLite { + a, err := os.Stat(storage.DBPath) + if err != nil { + return err + } + stores := []struct{ kind, path string }{{"telemetry", storage.DBPath}} + if selection.Accounts != nil { + b, err := os.Stat(storage.UsersDBPath) + if err != nil { + return err + } + if os.SameFile(a, b) { + return errors.New("telemetry and accounts cannot share the same SQLite file") + } + stores = append(stores, struct{ kind, path string }{"accounts", storage.UsersDBPath}) + } + for _, store := range stores { + db, err := openSQLite(store.path, "ro") + if err != nil { + return err + } + if err := dbconfig.AssertSQLiteImportComplete(db); err != nil { + db.Close() + return err + } + if store.kind == "telemetry" { + err = dbschema.AssertSQLiteReady(db) + } else { + err = users.AssertSQLiteReady(db) + } + db.Close() + if err != nil { + return err + } + } + return nil + } + if storage.ReaderDatabaseURL == "" || storage.WriterDatabaseURL == "" { + return errors.New("prepare both PostgreSQL reader and writer role URLs before selecting this backend") + } + reader, err := pgutil.Open(storage.ReaderDatabaseURL, true) + if err != nil { + return err + } + if err = dbschema.AssertPostgresReady(reader); err == nil { + err = pgutil.AssertReadOnly(reader) + } + reader.Close() + if err != nil { + return err + } + writer, err := pgutil.Open(storage.WriterDatabaseURL, false) + if err != nil { + return err + } + if err = dbschema.AssertPostgresReady(writer); err == nil { + err = dbschema.AssertWriter(writer) + } + writer.Close() + if err != nil { + return err + } + if selection.Accounts != nil { + if storage.UsersDatabaseURL == "" { + return errors.New("prepare the selected PostgreSQL account runtime URL before cutover") + } + store, err := users.OpenStorage(storage) + if err != nil { + return err + } + store.Close() + } + if storage.ApprovedChannelsDatabaseURL != "" { + db, err := pgutil.Open(storage.ApprovedChannelsDatabaseURL, true) + if err != nil { + return err + } + err = pgutil.AssertReadOnly(db) + db.Close() + if err != nil { + return err + } + } + return nil +} diff --git a/cmd/migrate/storage_command_test.go b/cmd/migrate/storage_command_test.go new file mode 100644 index 000000000..6f45e3074 --- /dev/null +++ b/cmd/migrate/storage_command_test.go @@ -0,0 +1,149 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/pgutil" +) + +func TestStorageStatusNeedsNoCredentialsOrDatabase(t *testing.T) { + t.Setenv("CORESCOPE_DATABASE_URL", "unreachable-private-secret") + t.Setenv("CORESCOPE_USERS_DATABASE_URL", "unreachable-private-secret") + selection := filepath.Join(t.TempDir(), "storage-selection.json") + var out bytes.Buffer + if err := runCommand(context.Background(), []string{"-storage-action=status", "-selection-file", selection}, &out); err != nil { + t.Fatal(err) + } + var status dbconfig.SelectionStatus + if err := json.Unmarshal(out.Bytes(), &status); err != nil { + t.Fatal(err) + } + if status.State != "unrecorded" || status.Backend != "" { + t.Fatalf("status=%+v", status) + } + if bytes.Contains(out.Bytes(), []byte("secret")) { + t.Fatal("status exposed connection input") + } +} + +func TestRecordedPostgresPortCannotDriftThroughEnvironment(t *testing.T) { + t.Setenv("PGPORT", "") + t.Setenv("PGSERVICE", "") + t.Setenv("PGOPTIONS", "") + base := t.TempDir() + selected, err := dbconfig.NewSelection(dbconfig.Storage{Backend: dbconfig.Postgres, StateDir: base, WriterDatabaseURL: "postgres://owner:secret@localhost/data"}) + if err != nil { + t.Fatal(err) + } + t.Setenv("PGPORT", "6543") + raw, err := selected.ApplyTo(dbconfig.StorageInputs{BaseDir: base, DatabaseURL: "postgres://rotated:secret@localhost/data"}) + if err != nil { + t.Fatal(err) + } + config, err := pgutil.ParseConfig(raw.WriterDatabaseURL) + if err != nil || config.Port != 5432 || config.Database != "data" || config.RuntimeParams["search_path"] != `"public"` { + t.Fatal("native parser drifted from recorded target", err) + } +} + +func TestSetupRefusesMalformedPrimaryBeforeValidFallback(t *testing.T) { + dir := t.TempDir() + if err := os.MkdirAll(filepath.Join(dir, "data"), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "config.json"), []byte(`{broken`), 0600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "data", "config.json"), []byte(`{"db":{"backend":"sqlite"}}`), 0600); err != nil { + t.Fatal(err) + } + selection := filepath.Join(dir, "data", "storage-selection.json") + if err := runCommand(context.Background(), []string{"-storage-action=setup", "-selection-file", selection, "-config-dir", dir, "-offline"}, &bytes.Buffer{}); err == nil { + t.Fatal("bad primary config silently chose a fallback") + } + if _, err := os.Stat(selection); !os.IsNotExist(err) { + t.Fatal("bad primary initialized an installation", err) + } +} + +func TestStorageSetupDefaultsSQLiteAndPreservesRecordedTargets(t *testing.T) { + for _, name := range []string{"CORESCOPE_DB_BACKEND", "DB_PATH", "CORESCOPE_DATABASE_URL", "CORESCOPE_READER_DATABASE_URL", "CORESCOPE_WRITER_DATABASE_URL", "CORESCOPE_USERS_DATABASE_URL", "CORESCOPE_USERS_OWNER_DATABASE_URL", "CORESCOPE_APPROVED_CHANNELS_DATABASE_URL"} { + t.Setenv(name, "") + } + root := t.TempDir() + selection := filepath.Join(root, "data", "storage-selection.json") + var out bytes.Buffer + args := []string{"-storage-action=setup", "-selection-file", selection, "-config-dir", root, "-offline"} + if err := runCommand(context.Background(), args, &out); err != nil { + t.Fatal(err) + } + got, err := dbconfig.ReadSelection(selection) + if err != nil { + t.Fatal(err) + } + if got.Backend != dbconfig.SQLite || got.Telemetry.SQLitePath != filepath.Join(root, "data", "meshcore.db") || got.Accounts == nil { + t.Fatal("fresh default or account target wrong") + } + db, err := openSQLite(got.Telemetry.SQLitePath, "rw") + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`INSERT INTO nodes(public_key,name) VALUES('retained','still here')`); err != nil { + t.Fatal(err) + } + db.Close() + before, err := sourceFingerprint(got.Telemetry.SQLitePath) + if err != nil { + t.Fatal(err) + } + t.Setenv("CORESCOPE_DB_BACKEND", "postgres") + t.Setenv("DB_PATH", filepath.Join(root, "obsolete.sqlite")) + out.Reset() + if err := runCommand(context.Background(), args, &out); err != nil { + t.Fatal(err) + } + after, err := dbconfig.ReadSelection(selection) + if err != nil || after.Generation != got.Generation || after.Backend != dbconfig.SQLite { + t.Fatal("update changed selection", err) + } + if current, err := sourceFingerprint(got.Telemetry.SQLitePath); err != nil || current != before { + t.Fatal("ready setup changed working data", err) + } + out.Reset() + args[0] = "-storage-action=init" + if err := runCommand(context.Background(), args, &out); err == nil { + t.Fatal("init replaced existing installation") + } +} + +func TestUnrecordedConfigPathConflictRequiresOverride(t *testing.T) { + t.Setenv("CORESCOPE_DB_BACKEND", "") + t.Setenv("CORESCOPE_DATABASE_URL", "") + root := t.TempDir() + config := filepath.Join(root, "custom.json") + if err := os.WriteFile(config, []byte(`{"dbPath":"one.sqlite","userManagement":{"enabled":false,"dbPath":"saved-users.sqlite"}}`), 0600); err != nil { + t.Fatal(err) + } + t.Setenv("DB_PATH", "two.sqlite") + args := []string{"-storage-action=setup", "-selection-file", filepath.Join(root, "storage-selection.json"), "-config", config, "-offline"} + var out bytes.Buffer + if err := runCommand(context.Background(), args, &out); err == nil { + t.Fatal("conflicting server/ingestor paths silently adopted") + } + chosen := filepath.Join(root, "chosen.sqlite") + accounts := filepath.Join(root, "chosen-users.sqlite") + args = append(args, "-sqlite-path", chosen, "-users-sqlite-path", accounts) + if err := runCommand(context.Background(), args, &out); err != nil { + t.Fatal(err) + } + got, err := dbconfig.ReadSelection(filepath.Join(root, "storage-selection.json")) + if err != nil || got.Telemetry.SQLitePath != chosen || got.Accounts.SQLitePath != accounts { + t.Fatal("explicit target override failed", err) + } +} diff --git a/cmd/migrate/switch.go b/cmd/migrate/switch.go new file mode 100644 index 000000000..d698df217 --- /dev/null +++ b/cmd/migrate/switch.go @@ -0,0 +1,425 @@ +package main + +import ( + "context" + "database/sql" + "errors" + "fmt" + "os" + "path/filepath" + "reflect" + + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/pgutil" + "github.com/meshcore-analyzer/users" +) + +type switchJob struct { + Version int `json:"version"` + ID string `json:"id"` + Source dbconfig.Selection `json:"source"` + Next dbconfig.Selection `json:"next"` +} + +type sqliteFence struct { + read, writer *sql.DB + conn *sql.Conn + path, digest string +} + +func (f *sqliteFence) Close() { + if f.conn != nil { + f.conn.ExecContext(context.Background(), `ROLLBACK`) + f.conn.Close() + } + if f.writer != nil { + f.writer.Close() + } + if f.read != nil { + f.read.Close() + } +} +func (f *sqliteFence) unchanged(ctx context.Context) error { + got, err := fingerprint(ctx, f.path) + if err != nil { + return err + } + if got != f.digest { + return errors.New("original SQLite source changed; keep writers stopped and preserve recovery files") + } + return nil +} + +func fenceSQLite(ctx context.Context, path, kind string) (_ *sqliteFence, err error) { + f := &sqliteFence{path: path} + defer func() { + if err != nil { + f.Close() + } + }() + // The read-only keeper outlives the writer handle, preventing a last-writer + // close from checkpointing the source's pre-existing WAL into its main file. + if f.read, err = openSQLite(path, "ro"); err != nil { + return nil, err + } + f.read.SetMaxOpenConns(1) + if err = dbconfig.AssertSQLiteImportComplete(f.read); err != nil { + return nil, err + } + if kind == "telemetry" { + err = dbschema.AssertSQLiteReady(f.read) + } else { + err = users.AssertSQLiteReady(f.read) + } + if err != nil { + return nil, err + } + if f.digest, err = fingerprint(ctx, path); err != nil { + return nil, err + } + if f.writer, err = openSQLite(path, "rw"); err != nil { + return nil, err + } + f.writer.SetMaxOpenConns(1) + if f.conn, err = f.writer.Conn(ctx); err != nil { + return nil, err + } + if _, err = f.conn.ExecContext(ctx, `PRAGMA busy_timeout=0`); err != nil { + return nil, err + } + if _, err = f.conn.ExecContext(ctx, `BEGIN IMMEDIATE`); err != nil { + return nil, errors.New("SQLite source writer is active; stop all writers before conversion") + } + if err = f.unchanged(ctx); err != nil { + return nil, err + } + return f, nil +} + +func bindOwnerURLs(selected dbconfig.Selection, owner, accounts string) (dbconfig.Storage, error) { + raw := dbconfig.StorageInputs{BaseDir: selected.StateDir, DatabaseURL: owner, UsersDatabaseURL: accounts} + bound, err := selected.ApplyTo(raw) + if err != nil { + return dbconfig.Storage{}, err + } + storage, err := dbconfig.ResolveStorage(bound) + if err != nil { + return storage, err + } + if selected.Backend == dbconfig.Postgres && selected.Accounts != nil && storage.UsersDatabaseURL == "" { + return storage, errors.New("the existing account store requires its PostgreSQL owner URL even when accounts are disabled") + } + return storage, nil +} + +func validateInitialOwnerTargets(ctx context.Context, selected dbconfig.Selection, storage dbconfig.Storage) error { + checks := []struct { + url string + target *dbconfig.PostgresTarget + }{{storage.WriterDatabaseURL, selected.Telemetry.Postgres}} + if selected.Accounts != nil { + checks = append(checks, struct { + url string + target *dbconfig.PostgresTarget + }{storage.UsersDatabaseURL, selected.Accounts.Postgres}) + } + for _, check := range checks { + if check.target == nil || check.url == "" { + return errors.New("initial PostgreSQL owner targets are incomplete") + } + db, err := pgutil.Open(check.url, true) + if err != nil { + return err + } + var database, schema string + err = db.QueryRowContext(ctx, `SELECT current_database(),current_schema()`).Scan(&database, &schema) + db.Close() + if err != nil || database != check.target.Database || schema != check.target.Schema { + return errors.New("initial PostgreSQL database/schema differs from its proposed selection; specify the intended database and search_path explicitly") + } + } + return nil +} + +func runStorageSwitch(ctx context.Context, o storageOptions) error { + state, err := filepath.Abs(o.StateDir) + if err != nil { + return err + } + if o.Action == "abort" { + if o.JobID == "" { + return errors.New("abort requires the exact -job-id") + } + update, err := dbconfig.ResumeSelectionUpdate(o.SelectionFile, o.JobID) + if err != nil { + return err + } + defer update.Close() + return update.Abort() // Staged targets and every recovery copy are preserved. + } + var job switchJob + var update *dbconfig.SelectionUpdate + raw, err := storageActionInputs(o, true) + if err != nil { + return err + } + if o.Action == "switch" { + if o.Backend != "sqlite" && o.Backend != "postgres" { + return errors.New("switch requires an explicit -backend=sqlite or postgres") + } + current, err := dbconfig.ReadSelection(o.SelectionFile) + if err != nil { + return err + } + if string(current.Backend) == o.Backend { + return errors.New("the requested backend is already selected") + } + targetInputs := raw + targetInputs.Backend = dbconfig.Backend(o.Backend) + targetInputs.EnvBackend = targetInputs.Backend + targetInputs.ExistingBackend = "" + targetInputs.StateDir = current.StateDir + if targetInputs.Backend == dbconfig.SQLite { + if o.SQLitePath == "" { + return errors.New("reverse switch requires a new explicit -sqlite-path; retained original files are never overwritten") + } + targetInputs.DBPath = o.SQLitePath + targetInputs.UsersDBPath = o.UsersSQLitePath + } else { + if o.OwnerURL == "" { + return errors.New("PostgreSQL destination requires its owner URL through the private environment") + } + if current.Accounts != nil && o.UsersOwnerURL == "" { + return errors.New("the existing account store must also be converted; configure CORESCOPE_USERS_OWNER_DATABASE_URL") + } + targetInputs.DatabaseURL = o.OwnerURL + targetInputs.ReaderDatabaseURL = "" + targetInputs.WriterDatabaseURL = "" + targetInputs.UsersDatabaseURL = o.UsersOwnerURL + targetInputs.ApprovedChannelsDatabaseURL = "" + } + target, err := dbconfig.ResolveStorage(targetInputs) + if err != nil { + return err + } + if current.Accounts == nil { + target.UsersDBPath = "" + target.UsersDatabaseURL = "" + target.ApprovedChannelsDatabaseURL = "" + } + next, err := dbconfig.NewSelection(target) + if err != nil { + return err + } + if next.Backend == dbconfig.Postgres { + if err := validateInitialOwnerTargets(ctx, next, target); err != nil { + return err + } + } + if next.Backend == dbconfig.SQLite { + paths := []string{next.Telemetry.SQLitePath} + if next.Accounts != nil { + paths = append(paths, next.Accounts.SQLitePath) + } + for _, path := range paths { + if _, err := os.Lstat(path); !errors.Is(err, os.ErrNotExist) { + return errors.New("switch SQLite targets must be fresh absent paths") + } + } + } + update, err = dbconfig.BeginSelectionUpdate(o.SelectionFile, current.Generation) + if err != nil { + return err + } + defer update.Close() + job = switchJob{Version: 1, ID: update.ID(), Source: current, Next: next} + if err := update.Stage(next); err != nil { + return err + } + dir := filepath.Join(state, job.ID) + if err := os.MkdirAll(dir, 0700); err != nil { + return err + } + if err := writePrivateJSON(filepath.Join(dir, "job.json"), job); err != nil { + return err + } + } else { + if o.JobID == "" { + return errors.New("resume requires the exact -job-id") + } + update, err = dbconfig.ResumeSelectionUpdate(o.SelectionFile, o.JobID) + if err != nil { + return err + } + defer update.Close() + if update.Target().Generation == "" { + return errors.New("pending job has no staged destination; abort this metadata-only update and start a new job") + } + if accountInitialization(update.Current(), update.Target()) { + return finishAccountSetup(ctx, o, update, raw, true) + } + if err := readPrivateJSON(filepath.Join(state, o.JobID, "job.json"), &job); err != nil { + return errors.New("pending switch has no complete readable job manifest; preserve any recovery files, abort the uncommitted update and start a new job") + } + if job.Version != 1 || job.ID != o.JobID || !reflect.DeepEqual(job.Source, update.Current()) || !reflect.DeepEqual(job.Next, update.Target()) || job.Next.StateDir != job.Source.StateDir || job.Next.Backend == job.Source.Backend { + return errors.New("switch recovery manifest does not match the pending installation generation") + } + } + dir := filepath.Join(state, job.ID) + ownerSelection := job.Next + if job.Source.Backend == dbconfig.Postgres { + ownerSelection = job.Source + } + owners, err := bindOwnerURLs(ownerSelection, o.OwnerURL, o.UsersOwnerURL) + if err != nil { + return err + } + if ownerSelection.Accounts != nil { + same, err := pgutil.SameDatabase(ctx, owners.WriterDatabaseURL, owners.UsersDatabaseURL) + if err != nil { + return err + } + if same { + return errors.New("telemetry and account databases must remain separate") + } + } + var sqliteSources []*sqliteFence + var postgresSources []*postgresFence + stores := []struct { + kind string + source dbconfig.Target + target dbconfig.Target + url string + }{{"telemetry", job.Source.Telemetry, job.Next.Telemetry, owners.WriterDatabaseURL}} + if job.Source.Accounts != nil { + if job.Next.Accounts == nil { + return errors.New("switch manifest would omit an existing account store") + } + stores = append(stores, struct { + kind string + source, target dbconfig.Target + url string + }{"accounts", *job.Source.Accounts, *job.Next.Accounts, owners.UsersDatabaseURL}) + } + // Fence both sources before copying either one. Every defer runs on all + // failure paths; the durable selection journal remains to block startup. + for _, store := range stores { + if job.Source.Backend == dbconfig.SQLite { + f, err := fenceSQLite(ctx, store.source.SQLitePath, store.kind) + if err != nil { + return err + } + sqliteSources = append(sqliteSources, f) + defer f.Close() + } else { + f, err := fencePostgres(ctx, store.url, store.kind) + if err != nil { + return err + } + postgresSources = append(postgresSources, f) + defer f.Close() + } + } + var reports []importReport + for i, store := range stores { + var report importReport + if job.Source.Backend == dbconfig.SQLite { + resume := o.Action == "resume" + if resume { + if _, err := os.Stat(filepath.Join(dir, store.kind, "manifest.json")); errors.Is(err, os.ErrNotExist) { + resume = false + } + } + report, err = importSQLite(ctx, importOptions{Source: store.source.SQLitePath, DatabaseURL: store.url, StateDir: dir, Kind: store.kind, Resume: resume}) + } else { + resume := o.Action == "resume" + if resume { + if _, err := os.Stat(filepath.Join(dir, store.kind, "reverse.json")); errors.Is(err, os.ErrNotExist) { + resume = false + } + } + report, err = reversePostgres(ctx, reverseOptions{DatabaseURL: store.url, Destination: store.target.SQLitePath, StateDir: dir, Kind: store.kind, Resume: resume, source: postgresSources[i], keepPending: true}) + } + if err != nil { + return err + } + if !report.Verified { + return errors.New("store copy returned without complete verification") + } + reports = append(reports, report) + if o.afterStore != nil { + if err := o.afterStore(store.kind); err != nil { + return err + } + } + } + for _, source := range sqliteSources { + if err := source.unchanged(ctx); err != nil { + return err + } + } + verification := struct { + JobID string `json:"job_id"` + Reports []importReport `json:"reports"` + }{job.ID, reports} + if err := writePrivateJSON(filepath.Join(dir, "verified.json"), verification); err != nil { + if !errors.Is(err, os.ErrExist) { + return err + } + previous := verification + previous.Reports = nil + if err := readPrivateJSON(filepath.Join(dir, "verified.json"), &previous); err != nil { + return err + } + if !reflect.DeepEqual(previous, verification) { + return errors.New("saved verification receipt differs from the reverified stores") + } + } + if job.Next.Backend == dbconfig.Postgres { + for _, store := range stores { + if _, err := finalizeImport(ctx, store.url, store.kind); err != nil { + return err + } + } + if err := grantRuntimeTargets(ctx, job.Next, owners, raw); err != nil { + return err + } + } else { + targets := []struct{ kind, path string }{{"telemetry", job.Next.Telemetry.SQLitePath}} + if job.Next.Accounts != nil { + targets = append(targets, struct{ kind, path string }{"accounts", job.Next.Accounts.SQLitePath}) + } + for _, target := range targets { + db, err := openSQLite(target.path, "rw") + if err != nil { + return err + } + _, err = db.ExecContext(ctx, `DROP TABLE IF EXISTS corescope_reverse_progress`) + if err == nil { + err = assertSQLiteData(ctx, db, target.kind) + } + db.Close() + if err != nil { + return err + } + } + } + if err := validateRuntimeSelection(ctx, job.Next, raw); err != nil { + return err + } + if o.beforeCommit != nil { + if err := o.beforeCommit(); err != nil { + return err + } + } + for _, source := range sqliteSources { + if err := source.unchanged(ctx); err != nil { + return err + } + } + if _, err := update.Commit(job.Next); err != nil { + return fmt.Errorf("selection commit needs recovery for job %s: %w", job.ID, err) + } + return nil +} diff --git a/cmd/migrate/switch_test.go b/cmd/migrate/switch_test.go new file mode 100644 index 000000000..51f5fde70 --- /dev/null +++ b/cmd/migrate/switch_test.go @@ -0,0 +1,461 @@ +package main + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "net/url" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/pgutil" +) + +func restrictedRole(t *testing.T, owner string) string { + t.Helper() + db := openImportDB(t, owner) + var random [12]byte + if _, err := rand.Read(random[:]); err != nil { + t.Fatal(err) + } + role := "corescope_convert_" + hex.EncodeToString(random[:]) + password := "test_" + hex.EncodeToString(random[:]) + if _, err := db.Exec(`CREATE ROLE ` + quote(role) + ` LOGIN PASSWORD '` + password + `'`); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if _, err := db.Exec(`DROP OWNED BY ` + quote(role) + `;DROP ROLE ` + quote(role)); err != nil { + t.Error(err) + } + }) + u, err := url.Parse(owner) + if err != nil { + t.Fatal("invalid test owner URL") + } + u.User = url.UserPassword(role, password) + return u.String() +} + +func switchFixture(t *testing.T) (storageOptions, dbconfig.Selection) { + t.Helper() + telemetry, accounts := postgresDatabase(t), postgresDatabase(t) + t.Setenv("CORESCOPE_DATABASE_URL", telemetry) + t.Setenv("CORESCOPE_USERS_OWNER_DATABASE_URL", accounts) + t.Setenv("CORESCOPE_READER_DATABASE_URL", restrictedRole(t, telemetry)) + t.Setenv("CORESCOPE_WRITER_DATABASE_URL", restrictedRole(t, telemetry)) + t.Setenv("CORESCOPE_USERS_DATABASE_URL", restrictedRole(t, accounts)) + t.Setenv("CORESCOPE_APPROVED_CHANNELS_DATABASE_URL", restrictedRole(t, accounts)) + t.Setenv("CORESCOPE_DB_BACKEND", "") + t.Setenv("DB_PATH", "") + source, account := telemetrySource(t), accountSource(t, 5) + if err := applySQLiteTarget(context.Background(), source, "telemetry"); err != nil { + t.Fatal(err) + } + if err := applySQLiteTarget(context.Background(), account, "accounts"); err != nil { + t.Fatal(err) + } + dir := t.TempDir() + selected, err := dbconfig.NewSelection(dbconfig.Storage{Backend: dbconfig.SQLite, StateDir: dir, DBPath: source, UsersDBPath: account}) + if err != nil { + t.Fatal(err) + } + selection := dbconfig.SelectionPath(dir) + if _, err := dbconfig.AdoptSelection(selection, selected); err != nil { + t.Fatal(err) + } + return storageOptions{Action: "switch", SelectionFile: selection, Backend: "postgres", OwnerURL: telemetry, UsersOwnerURL: accounts, StateDir: t.TempDir(), Offline: true, ConfigDir: t.TempDir()}, selected +} + +func TestVerifiedTwoStoreSwitchRoundTripAndRuntimeGrants(t *testing.T) { + o, source := switchFixture(t) + ctx := context.Background() + before, err := sourceFingerprint(source.Telemetry.SQLitePath) + if err != nil { + t.Fatal(err) + } + beforeAccount, err := sourceFingerprint(source.Accounts.SQLitePath) + if err != nil { + t.Fatal(err) + } + var out bytes.Buffer + if err := runStorageAction(ctx, o, &out); err != nil { + t.Fatal("forward switch", err) + } + selected, err := dbconfig.ReadSelection(o.SelectionFile) + if err != nil || selected.Backend != dbconfig.Postgres { + t.Fatal("PG selection", err) + } + if got, err := sourceFingerprint(source.Telemetry.SQLitePath); err != nil || got != before { + t.Fatal("telemetry source changed", err) + } + if got, err := sourceFingerprint(source.Accounts.SQLitePath); err != nil || got != beforeAccount { + t.Fatal("account source changed", err) + } + for _, pair := range [][2]string{{"CORESCOPE_WRITER_DATABASE_URL", "corescope_schema"}, {"CORESCOPE_USERS_DATABASE_URL", "schema_version"}} { + db, err := pgutil.Open(os.Getenv(pair[0]), false) + if err != nil { + t.Fatal(err) + } + var allowed bool + if err := db.QueryRow(`SELECT has_table_privilege($1,'INSERT,UPDATE,DELETE,TRUNCATE')`, pair[1]).Scan(&allowed); err != nil || allowed { + t.Fatal("metadata DML granted", allowed, err) + } + db.Close() + } + channel, err := pgutil.Open(os.Getenv("CORESCOPE_APPROVED_CHANNELS_DATABASE_URL"), true) + if err != nil { + t.Fatal(err) + } + for _, view := range []string{"approved_channels", "linked_companions"} { + if _, err := channel.Exec(`SELECT * FROM ` + view + ` LIMIT 0`); err != nil { + t.Fatal("projection reader lacks its public view", view, err) + } + } + for _, table := range []string{"users", "sessions", "companion_links", "link_challenges"} { + if _, err := channel.Exec(`SELECT * FROM ` + table + ` LIMIT 0`); err == nil { + t.Fatal("projection reader can access private account table", table) + } + } + channel.Close() + o.Backend = "sqlite" + o.SQLitePath = filepath.Join(t.TempDir(), "telemetry.sqlite") + o.UsersSQLitePath = filepath.Join(t.TempDir(), "accounts.sqlite") + out.Reset() + if err := runStorageAction(ctx, o, &out); err != nil { + t.Fatal("reverse switch", err) + } + selected, err = dbconfig.ReadSelection(o.SelectionFile) + if err != nil || selected.Backend != dbconfig.SQLite || selected.StateDir != source.StateDir || selected.Telemetry.SQLitePath != o.SQLitePath { + t.Fatal("reverse selection", err) + } + db, err := openSQLite(o.UsersSQLitePath, "ro") + if err != nil { + t.Fatal(err) + } + defer db.Close() + var id int + if err := db.QueryRow(`SELECT id FROM mail_events WHERE mail_id=4`).Scan(&id); err != nil || id != 900 { + t.Fatal("round trip event", id, err) + } + if strings.Contains(out.String(), "postgres://") { + t.Fatal("CLI leaked credentials") + } +} + +func TestSwitchFailureKeepsOldSelectionAndResumeVerifiesAgain(t *testing.T) { + o, source := switchFixture(t) + ctx := context.Background() + o.afterStore = func(kind string) error { + if kind == "telemetry" { + return errors.New("injected failure before accounts") + } + return nil + } + if err := runStorageAction(ctx, o, &bytes.Buffer{}); err == nil { + t.Fatal("partial switch succeeded") + } + status, err := dbconfig.InspectSelection(o.SelectionFile) + if err != nil || status.State != "pending" || status.Backend != dbconfig.SQLite || status.Generation != source.Generation { + t.Fatal("partial switch selected target", status, err) + } + if _, _, err := dbconfig.OpenSelection(o.SelectionFile); !errors.Is(err, dbconfig.ErrSelectionInProgress) { + t.Fatal("partial switch allowed runtime", err) + } + o.Action = "resume" + o.JobID = status.JobID + o.afterStore = nil + o.beforeCommit = func() error { return errors.New("injected cutover failure") } + if err := runStorageAction(ctx, o, &bytes.Buffer{}); err == nil { + t.Fatal("cutover failure passed") + } + status, err = dbconfig.InspectSelection(o.SelectionFile) + if err != nil || status.Generation != source.Generation || status.State != "pending" { + t.Fatal("failed cutover changed selection", err) + } + o.beforeCommit = nil + if err := runStorageAction(ctx, o, &bytes.Buffer{}); err != nil { + t.Fatal("verified resume", err) + } + final, err := dbconfig.ReadSelection(o.SelectionFile) + if err != nil || final.Backend != dbconfig.Postgres { + t.Fatal("resume failed to select", err) + } +} + +func TestSQLiteSourceFencePreservesWALAndRejectsWriters(t *testing.T) { + source := telemetrySource(t) + ctx := context.Background() + if err := applySQLiteTarget(ctx, source, "telemetry"); err != nil { + t.Fatal(err) + } + writer, err := openSQLite(source, "rw") + if err != nil { + t.Fatal(err) + } + if _, err := writer.Exec(`PRAGMA journal_mode=WAL; PRAGMA wal_autocheckpoint=0; INSERT INTO observers(id) VALUES('wal-retained')`); err != nil { + t.Fatal(err) + } + keeper, err := openSQLite(source, "ro") + if err != nil { + t.Fatal(err) + } + defer keeper.Close() + if err := keeper.Ping(); err != nil { + t.Fatal(err) + } + writer.Close() + before, err := sourceFingerprint(source) + if err != nil { + t.Fatal(err) + } + f, err := fenceSQLite(ctx, source, "telemetry") + if err != nil { + t.Fatal(err) + } + other, err := openSQLite(source, "rw") + if err != nil { + t.Fatal(err) + } + if _, err := other.Exec(`PRAGMA busy_timeout=1; INSERT INTO observers(id) VALUES('must-not-write')`); err == nil { + t.Fatal("source accepted a concurrent writer") + } + other.Close() + f.Close() + keeper.Close() + after, err := sourceFingerprint(source) + if err != nil || after != before { + t.Fatal("source fence checkpointed or modified original WAL", err) + } +} + +func TestSwitchSourceMutationAndDestinationSubstitutionRefuse(t *testing.T) { + o, source := switchFixture(t) + ctx := context.Background() + o.afterStore = func(string) error { return errors.New("stop after first store") } + if err := runStorageAction(ctx, o, &bytes.Buffer{}); err == nil { + t.Fatal("expected interruption") + } + status, err := dbconfig.InspectSelection(o.SelectionFile) + if err != nil { + t.Fatal(err) + } + var job switchJob + jobPath := filepath.Join(o.StateDir, status.JobID, "job.json") + if err := readPrivateJSON(jobPath, &job); err != nil { + t.Fatal(err) + } + original := job + job.Next.Telemetry.Postgres = &dbconfig.PostgresTarget{Host: "substituted.invalid", Port: 5432, Database: "other", Schema: "public"} + data, err := json.Marshal(job) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(jobPath, data, 0600); err != nil { + t.Fatal(err) + } + o.Action = "resume" + o.JobID = status.JobID + o.afterStore = nil + if err := runStorageAction(ctx, o, &bytes.Buffer{}); err == nil || !strings.Contains(err.Error(), "manifest") { + t.Fatal("changed target accepted", err) + } + data, err = json.Marshal(original) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(jobPath, data, 0600); err != nil { + t.Fatal(err) + } + changed, err := openSQLite(source.Telemetry.SQLitePath, "rw") + if err != nil { + t.Fatal(err) + } + if _, err := changed.Exec(`UPDATE observations SET score=7.5 WHERE id=90`); err != nil { + t.Fatal(err) + } + changed.Close() + if err := runStorageAction(ctx, o, &bytes.Buffer{}); err == nil { + t.Fatal("changed original source accepted") + } + o.Action = "abort" + if err := runStorageAction(ctx, o, &bytes.Buffer{}); err != nil { + t.Fatal("abort", err) + } + selected, err := dbconfig.ReadSelection(o.SelectionFile) + if err != nil || selected.Generation != source.Generation { + t.Fatal("abort changed original selection", err) + } + if _, err := os.Stat(filepath.Join(o.StateDir, status.JobID, "telemetry", "recovery.sqlite")); err != nil { + t.Fatal("abort removed recovery", err) + } +} + +func TestSetupPostgresCannotAbandonExistingSQLite(t *testing.T) { + owner := postgresDatabase(t) + t.Setenv("CORESCOPE_DATABASE_URL", owner) + t.Setenv("CORESCOPE_USERS_DATABASE_URL", "") + t.Setenv("CORESCOPE_USERS_OWNER_DATABASE_URL", "") + t.Setenv("CORESCOPE_APPROVED_CHANNELS_DATABASE_URL", "") + t.Setenv("CORESCOPE_READER_DATABASE_URL", "") + t.Setenv("CORESCOPE_WRITER_DATABASE_URL", "") + t.Setenv("CORESCOPE_DB_BACKEND", "sqlite") + source := telemetrySource(t) + t.Setenv("DB_PATH", source) + before, err := sourceFingerprint(source) + if err != nil { + t.Fatal(err) + } + dir := t.TempDir() + config := filepath.Join(dir, "config.json") + if err := os.WriteFile(config, []byte(`{"db":{"backend":"sqlite"}}`), 0600); err != nil { + t.Fatal(err) + } + args := []string{"-storage-action=setup", "-backend=postgres", "-selection-file", filepath.Join(dir, "storage-selection.json"), "-config", config, "-offline"} + err = runCommand(context.Background(), args, &bytes.Buffer{}) + if err == nil || !strings.Contains(err.Error(), "primary SQLite data") { + t.Fatal("fresh PG abandoned source or explicit choice did not override hints", err) + } + db := openImportDB(t, owner) + var n int + if err := db.QueryRow(importDestinationObjects).Scan(&n); err != nil || n != 0 { + t.Fatal("empty PG target was initialized", n, err) + } + if after, err := sourceFingerprint(source); err != nil || before != after { + t.Fatal("SQLite source was changed", err) + } +} + +func TestFreshPostgresSetupOverridesBootstrapDefaultAndGrantsBeforeSelection(t *testing.T) { + owner, accounts := postgresDatabase(t), postgresDatabase(t) + t.Setenv("CORESCOPE_DATABASE_URL", owner) + t.Setenv("CORESCOPE_USERS_OWNER_DATABASE_URL", accounts) + t.Setenv("CORESCOPE_READER_DATABASE_URL", restrictedRole(t, owner)) + t.Setenv("CORESCOPE_WRITER_DATABASE_URL", restrictedRole(t, owner)) + t.Setenv("CORESCOPE_USERS_DATABASE_URL", restrictedRole(t, accounts)) + t.Setenv("CORESCOPE_APPROVED_CHANNELS_DATABASE_URL", restrictedRole(t, accounts)) + t.Setenv("CORESCOPE_DB_BACKEND", "sqlite") + t.Setenv("DB_PATH", "") + dir := t.TempDir() + config := filepath.Join(dir, "config.json") + if err := os.WriteFile(config, []byte(`{"db":{"backend":"sqlite"},"userManagement":{"enabled":false}}`), 0600); err != nil { + t.Fatal(err) + } + selection := filepath.Join(dir, "storage-selection.json") + args := []string{"-storage-action=setup", "-backend=postgres", "-selection-file", selection, "-config", config, "-offline"} + var out bytes.Buffer + if err := runCommand(context.Background(), args, &out); err != nil { + t.Fatal(err) + } + selected, err := dbconfig.ReadSelection(selection) + if err != nil || selected.Backend != dbconfig.Postgres || selected.Accounts == nil { + t.Fatal("explicit PG setup", err) + } + if bytes.Contains(out.Bytes(), []byte("postgres://")) { + t.Fatal("owner credentials in setup output") + } + // Updates retain PG although the unchanged bootstrap config/env still say SQLite. + args = []string{"-storage-action=setup", "-selection-file", selection, "-config", config, "-offline"} + if err := runCommand(context.Background(), args, &bytes.Buffer{}); err != nil { + t.Fatal("PG update", err) + } + after, err := dbconfig.ReadSelection(selection) + if err != nil || after.Generation != selected.Generation { + t.Fatal("update replaced installation", err) + } +} + +func TestInitialAdoptionRefusesImplicitRoleSchemaDrift(t *testing.T) { + owner := postgresDatabase(t) + config, err := pgutil.ParseConfig(owner) + if err != nil { + t.Fatal(err) + } + db := openImportDB(t, owner) + if _, err := db.Exec(`CREATE SCHEMA ` + quote(config.User)); err != nil { + t.Fatal(err) + } + t.Setenv("CORESCOPE_DATABASE_URL", owner) + t.Setenv("CORESCOPE_USERS_OWNER_DATABASE_URL", "") + t.Setenv("CORESCOPE_USERS_DATABASE_URL", "") + t.Setenv("CORESCOPE_APPROVED_CHANNELS_DATABASE_URL", "") + t.Setenv("CORESCOPE_READER_DATABASE_URL", "") + t.Setenv("CORESCOPE_WRITER_DATABASE_URL", "") + t.Setenv("DB_PATH", "") + dir := t.TempDir() + selection := filepath.Join(dir, "storage-selection.json") + err = runStorageAction(context.Background(), storageOptions{Action: "setup", Backend: "postgres", OwnerURL: owner, Offline: true, SelectionFile: selection, ConfigDir: dir}, &bytes.Buffer{}) + if err == nil || !strings.Contains(err.Error(), "database/schema differs") { + t.Fatal("implicit owner schema drift was accepted", err) + } + if _, err := os.Stat(selection); !os.IsNotExist(err) { + t.Fatal("ambiguous schema was selected", err) + } + var tables int + if err := db.QueryRow(`SELECT count(*) FROM information_schema.tables WHERE table_schema NOT IN ('pg_catalog','information_schema')`).Scan(&tables); err != nil || tables != 0 { + t.Fatal("ambiguous schema bootstrap wrote tables", tables, err) + } +} + +func TestReverseWithoutAccountSourceResumesAfterCutoverFailure(t *testing.T) { + owner := postgresDatabase(t) + db := openImportDB(t, owner) + if err := dbschema.ApplyPostgres(db, nil); err != nil { + t.Fatal(err) + } + t.Setenv("CORESCOPE_DATABASE_URL", owner) + t.Setenv("CORESCOPE_USERS_OWNER_DATABASE_URL", "") + t.Setenv("CORESCOPE_USERS_DATABASE_URL", "") + t.Setenv("CORESCOPE_APPROVED_CHANNELS_DATABASE_URL", "") + dir := t.TempDir() + selected, err := dbconfig.NewSelection(dbconfig.Storage{Backend: dbconfig.Postgres, StateDir: dir, WriterDatabaseURL: owner}) + if err != nil { + t.Fatal(err) + } + selection := dbconfig.SelectionPath(dir) + if _, err := dbconfig.AdoptSelection(selection, selected); err != nil { + t.Fatal(err) + } + o := storageOptions{Action: "switch", Backend: "sqlite", Offline: true, SelectionFile: selection, OwnerURL: owner, StateDir: t.TempDir(), ConfigDir: t.TempDir(), SQLitePath: filepath.Join(t.TempDir(), "telemetry.sqlite"), UsersSQLitePath: filepath.Join(t.TempDir(), "accounts.sqlite"), beforeCommit: func() error { return errors.New("cutover interrupted") }} + if err := runStorageAction(context.Background(), o, &bytes.Buffer{}); err == nil { + t.Fatal("injected failure passed") + } + status, err := dbconfig.InspectSelection(selection) + if err != nil || status.State != "pending" { + t.Fatal("pending job absent", err) + } + o.Action = "resume" + o.JobID = status.JobID + o.beforeCommit = nil + if err := runStorageAction(context.Background(), o, &bytes.Buffer{}); err != nil { + t.Fatal("empty optional account resume", err) + } + after, err := dbconfig.ReadSelection(selection) + if err != nil || after.Backend != dbconfig.SQLite || after.Accounts != nil { + t.Fatal("resume did not preserve uninitialized accounts", err) + } + if _, err := os.Stat(o.UsersSQLitePath); !os.IsNotExist(err) { + t.Fatal("reverse switch invented an account database", err) + } + newOwner := postgresDatabase(t) + t.Setenv("CORESCOPE_DATABASE_URL", newOwner) + t.Setenv("CORESCOPE_READER_DATABASE_URL", restrictedRole(t, newOwner)) + t.Setenv("CORESCOPE_WRITER_DATABASE_URL", restrictedRole(t, newOwner)) + o.Action = "switch" + o.Backend = "postgres" + o.OwnerURL = newOwner + o.JobID = "" + if err := runStorageAction(context.Background(), o, &bytes.Buffer{}); err != nil { + t.Fatal("switch nil accounts forward", err) + } + after, err = dbconfig.ReadSelection(selection) + if err != nil || after.Backend != dbconfig.Postgres || after.Accounts != nil { + t.Fatal("forward switch changed account absence", err) + } +} diff --git a/cmd/migrate/target_identity_test.go b/cmd/migrate/target_identity_test.go new file mode 100644 index 000000000..af4325690 --- /dev/null +++ b/cmd/migrate/target_identity_test.go @@ -0,0 +1,62 @@ +package main + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "net/url" + "os" + "path/filepath" + "testing" + + "github.com/meshcore-analyzer/pgutil" +) + +func TestImportIdentityUsesConfiguredEndpoint(t *testing.T) { + dsn := postgresSchema(t) + u, err := url.Parse(dsn) + if err != nil { + t.Fatal("parse test URL") + } + // Connect through the stable name instead of its transient resolved address. + u.Host = "localhost:" + u.Port() + o := importOptions{Source: accountSource(t, 5), DatabaseURL: u.String(), StateDir: t.TempDir(), Kind: "accounts"} + if _, err := importSQLite(context.Background(), o); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(filepath.Join(o.StateDir, o.Kind, "manifest.json")) + if err != nil { + t.Fatal(err) + } + var manifest sourceManifest + if err := json.Unmarshal(data, &manifest); err != nil { + t.Fatal(err) + } + db := openImportDB(t, o.DatabaseURL) + var database, schema string + var databaseOID, schemaOID int64 + if err := db.QueryRow(`SELECT current_database(),current_schema(),(SELECT oid::bigint FROM pg_database WHERE datname=current_database()),(SELECT oid::bigint FROM pg_namespace WHERE nspname=current_schema())`).Scan(&database, &schema, &databaseOID, &schemaOID); err != nil { + t.Fatal(err) + } + config, err := pgutil.ParseConfig(o.DatabaseURL) + if err != nil { + t.Fatal(err) + } + identity, _ := json.Marshal([]any{database, schema, databaseOID, schemaOID, config.Host, config.Port}) + want := sha256.Sum256(identity) + if manifest.TargetDigest != hex.EncodeToString(want[:]) { + t.Fatal("manifest identity used a transient resolved address instead of the configured endpoint") + } + o.Resume = true + if _, err := importSQLite(context.Background(), o); err != nil { + t.Fatalf("same configured endpoint did not resume: %v", err) + } + // The IP alias reaches the same database/OIDs but is a different configured + // endpoint. Operators must retain their original endpoint when resuming. + u.Host = "127.0.0.1:" + u.Port() + o.DatabaseURL = u.String() + if _, err := importSQLite(context.Background(), o); err == nil { + t.Fatal("changed configured endpoint accepted") + } +} diff --git a/cmd/server/account_test.go b/cmd/server/account_test.go index 554bb52e0..80dc75b3e 100644 --- a/cmd/server/account_test.go +++ b/cmd/server/account_test.go @@ -135,8 +135,7 @@ func TestAuthResetTokenStoreFailureEndsSessionsFirst(t *testing.T) { c := f.registerAndActivate(t, "pia@example.org", "Pia", pw) thief := f.login(t, "pia@example.org", pw) _, reset := pendingLinks(t, f, c, "pia@example.org", "attacker@example.org") - f.execDB(t, `CREATE TRIGGER down BEFORE UPDATE ON tokens WHEN OLD.purpose = 'email_change' BEGIN - SELECT RAISE(ABORT, 'token store down'); END`) + f.execDB(t, testNativeSQL(`CREATE TRIGGER down BEFORE UPDATE ON tokens WHEN OLD.purpose='email_change' BEGIN SELECT RAISE(ABORT,'token store down'); END`, `CREATE FUNCTION down() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'token store down'; END $$; CREATE TRIGGER down BEFORE UPDATE ON tokens FOR EACH ROW WHEN (OLD.purpose='email_change') EXECUTE FUNCTION down()`)) expectStatus(t, f.do("POST", "/api/auth/reset", resetRequest{Token: reset, Password: "a brand new secret"}), 500) expectStatus(t, f.do("GET", "/api/auth/me", nil, as(thief)), 401) expectStatus(t, f.do("GET", "/api/auth/me", nil, as(c)), 401) diff --git a/cmd/server/advert_evidence_scale_test.go b/cmd/server/advert_evidence_scale_test.go index b569d30b6..eed081e8d 100644 --- a/cmd/server/advert_evidence_scale_test.go +++ b/cmd/server/advert_evidence_scale_test.go @@ -18,7 +18,7 @@ func TestAdvertEvidenceCatchupScale(t *testing.T) { if path == "" { t.Fatal("ADVERT_SCALE_DB must name the completed ingestor scale fixture") } - db, err := OpenDB(path) + db, err := openFixtureReader(t, path) if err != nil { t.Fatal(err) } diff --git a/cmd/server/advert_route_evidence.go b/cmd/server/advert_route_evidence.go index fc8336cbc..9858483d5 100644 --- a/cmd/server/advert_route_evidence.go +++ b/cmd/server/advert_route_evidence.go @@ -16,7 +16,7 @@ func (db *DB) advertEvidencePresent() bool { return true } var exists int - if err := db.conn.QueryRow(`SELECT 1 FROM sqlite_master WHERE type='table' AND name='advert_route_evidence'`).Scan(&exists); err != nil { + if err := db.conn.QueryRow(db.nativeSQL("SELECT 1 FROM sqlite_master WHERE type='table' AND name='advert_route_evidence'", `SELECT 1 FROM information_schema.tables WHERE table_schema=current_schema() AND table_name='advert_route_evidence'`)).Scan(&exists); err != nil { return false } db.advertEvidenceTable.Store(true) @@ -39,7 +39,7 @@ func (db *DB) advertEvidenceForIDs(ids []int) (map[int]uint8, error) { for i, id := range ids[start:end] { args[i] = id } - query := `SELECT tx_id,SUM(bit) FROM advert_route_evidence WHERE tx_id IN (` + strings.TrimSuffix(strings.Repeat("?,", len(args)), ",") + `) GROUP BY tx_id` + query := `SELECT tx_id,SUM(bit) FROM advert_route_evidence WHERE tx_id IN (` + db.sqlPlaceholders(len(args)) + `) GROUP BY tx_id` if db.advertEvidenceReadHook != nil { db.advertEvidenceReadHook() } @@ -121,7 +121,7 @@ func (s *PacketStore) pollAdvertEvidence(limit int) error { if limit <= 0 || limit > 500 { limit = 500 } - rows, err := s.db.conn.Query(`SELECT id,tx_id,bit FROM advert_route_evidence WHERE id>? ORDER BY id LIMIT ?`, s.advertEvidenceCursor, limit) + rows, err := s.db.conn.Query(`SELECT id,tx_id,bit FROM advert_route_evidence WHERE id>`+s.db.parameter(1)+` ORDER BY id LIMIT `+s.db.parameter(2), s.advertEvidenceCursor, limit) if err != nil { return err } diff --git a/cmd/server/advert_route_evidence_test.go b/cmd/server/advert_route_evidence_test.go index 3fc57e06b..59041b3fc 100644 --- a/cmd/server/advert_route_evidence_test.go +++ b/cmd/server/advert_route_evidence_test.go @@ -15,29 +15,32 @@ import ( func advertEvidenceFixture(t *testing.T, raw string, bits ...int) (*DB, *sql.DB) { t.Helper() path := createTestDBWithResolvedPath(t, 1, []string{"fixture-relay"}) - w, err := sql.Open("sqlite3", path+"?_journal_mode=WAL&_foreign_keys=on") + w, err := openFixtureSQL(path) if err != nil { t.Fatal(err) } t.Cleanup(func() { w.Close() }) for _, stmt := range []string{ `ALTER TABLE transmissions ADD COLUMN from_pubkey TEXT`, - `CREATE TABLE advert_route_evidence (id INTEGER PRIMARY KEY AUTOINCREMENT, tx_id INTEGER NOT NULL REFERENCES transmissions(id) ON DELETE CASCADE, bit INTEGER NOT NULL CHECK (bit IN (1,2)), UNIQUE(tx_id,bit))`, + testNativeSQL(`CREATE TABLE advert_route_evidence (id INTEGER PRIMARY KEY AUTOINCREMENT, tx_id INTEGER NOT NULL REFERENCES transmissions(id) ON DELETE CASCADE, bit INTEGER NOT NULL CHECK (bit IN (1,2)), UNIQUE(tx_id,bit))`, `CREATE TABLE advert_route_evidence (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, tx_id INTEGER NOT NULL REFERENCES transmissions(id) ON DELETE CASCADE, bit INTEGER NOT NULL CHECK (bit IN (1,2)), UNIQUE(tx_id,bit))`), } { if _, err := w.Exec(stmt); err != nil { t.Fatal(err) } } now := time.Now().UTC().Add(-time.Minute).Format(time.RFC3339) - if _, err := w.Exec(`UPDATE transmissions SET raw_hex=?, route_type=?, first_seen=?, from_pubkey='fixture-origin'; UPDATE observations SET raw_hex=?, timestamp=?`, raw, int(raw[1]-'0')&3, now, raw, now); err != nil { + if _, err := w.Exec(`UPDATE transmissions SET raw_hex=$1, route_type=$2, first_seen=$3, from_pubkey='fixture-origin'`, raw, int(raw[1]-'0')&3, now); err != nil { + t.Fatal(err) + } + if _, err := w.Exec(`UPDATE observations SET raw_hex=$1, timestamp=$2`, raw, now); err != nil { t.Fatal(err) } for _, bit := range bits { - if _, err := w.Exec(`INSERT INTO advert_route_evidence(tx_id,bit) VALUES(1,?)`, bit); err != nil { + if _, err := w.Exec(`INSERT INTO advert_route_evidence(tx_id,bit) VALUES(1,$1)`, bit); err != nil { t.Fatal(err) } } - db, err := OpenDB(path) + db, err := openFixtureReader(t, path) if err != nil { t.Fatal(err) } @@ -48,13 +51,13 @@ func advertEvidenceFixture(t *testing.T, raw string, bits ...int) (*DB, *sql.DB) func TestAdvertRouteEvidenceNodeAPIHonorsRequestedLimit(t *testing.T) { db, w := advertEvidenceFixture(t, "1100aa", 1) for id := 2; id <= 24; id++ { - if _, err := w.Exec(`INSERT INTO transmissions(id,hash,raw_hex,first_seen,payload_type,route_type,from_pubkey) SELECT ?,?,'1100aa',first_seen,4,1,'fixture-origin' FROM transmissions WHERE id=1`, id, fmt.Sprintf("node-advert-%d", id)); err != nil { + if _, err := w.Exec(`INSERT INTO transmissions(id,hash,raw_hex,first_seen,payload_type,route_type,from_pubkey) SELECT $1,$2,'1100aa',first_seen,4,1,'fixture-origin' FROM transmissions WHERE id=1`, id, fmt.Sprintf("node-advert-%d", id)); err != nil { t.Fatal(err) } mask := id % 4 for _, bit := range []int{1, 2} { if mask&bit != 0 { - if _, err := w.Exec(`INSERT INTO advert_route_evidence(tx_id,bit) VALUES(?,?)`, id, bit); err != nil { + if _, err := w.Exec(`INSERT INTO advert_route_evidence(tx_id,bit) VALUES($1,$2)`, id, bit); err != nil { t.Fatal(err) } } @@ -79,18 +82,20 @@ func TestAdvertRouteEvidenceNodeAPIHonorsRequestedLimit(t *testing.T) { // Measures the added bulk-load work and an idle feed at a 30K-packet scale. // Both paths touch indexed compact evidence, never observation raw frames. func BenchmarkAdvertEvidence30K(b *testing.B) { - conn, err := sql.Open("sqlite3", ":memory:") + conn, err := openPostgresTestDB(b) if err != nil { b.Fatal(err) } conn.SetMaxOpenConns(1) defer conn.Close() - if _, err := conn.Exec(`CREATE TABLE advert_route_evidence(id INTEGER PRIMARY KEY AUTOINCREMENT,tx_id INTEGER,bit INTEGER,UNIQUE(tx_id,bit)); + if _, err := conn.Exec(testNativeSQL(`CREATE TABLE advert_route_evidence(id INTEGER PRIMARY KEY AUTOINCREMENT,tx_id INTEGER,bit INTEGER,UNIQUE(tx_id,bit)); WITH RECURSIVE ids(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM ids WHERE n<30000) - INSERT INTO advert_route_evidence(tx_id,bit) SELECT n,1 FROM ids UNION ALL SELECT n,2 FROM ids`); err != nil { + INSERT INTO advert_route_evidence(tx_id,bit) SELECT n,1 FROM ids UNION ALL SELECT n,2 FROM ids`, `CREATE TABLE advert_route_evidence(id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,tx_id INTEGER,bit INTEGER,UNIQUE(tx_id,bit)); + WITH RECURSIVE ids(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM ids WHERE n<30000) + INSERT INTO advert_route_evidence(tx_id,bit) SELECT n,1 FROM ids UNION ALL SELECT n,2 FROM ids`)); err != nil { b.Fatal(err) } - db := &DB{conn: conn} + db := &DB{backend: testBackendValue(), conn: conn} db.advertEvidenceTable.Store(true) ids := make([]int, 30000) for i := range ids { @@ -309,7 +314,21 @@ func TestAdvertRouteEvidencePollSameObservationID(t *testing.T) { maxObs, maxTx := db.GetMaxObservationID(), db.GetMaxTransmissionID() // This matches the ingestor's conflict update: neither observation // ID nor timestamp advances, and the final raw matches the first. - if _, err := w.Exec(`UPDATE observations SET raw_hex=? WHERE id=1; INSERT INTO advert_route_evidence(tx_id,bit) VALUES(1,?); UPDATE observations SET raw_hex=? WHERE id=1`, tc.second, tc.secondBit, tc.first); err != nil { + tx, err := w.Begin() + if err != nil { + t.Fatal(err) + } + defer tx.Rollback() + if _, err := tx.Exec(`UPDATE observations SET raw_hex=$1 WHERE id=1`, tc.second); err != nil { + t.Fatal(err) + } + if _, err := tx.Exec(`INSERT INTO advert_route_evidence(tx_id,bit) VALUES(1,$1)`, tc.secondBit); err != nil { + t.Fatal(err) + } + if _, err := tx.Exec(`UPDATE observations SET raw_hex=$1 WHERE id=1`, tc.first); err != nil { + t.Fatal(err) + } + if err := tx.Commit(); err != nil { t.Fatal(err) } s.IngestNewFromDB(maxTx, 100) @@ -352,7 +371,7 @@ func TestAdvertRouteEvidenceMissingTableIsUnknown(t *testing.T) { } assertAdvertEvidenceViews(t, s, "other") var count int - if err := w.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE name='advert_route_evidence'`).Scan(&count); err != nil { + if err := w.QueryRow(testNativeSQL(`SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='advert_route_evidence'`, `SELECT COUNT(*) FROM information_schema.tables WHERE table_schema=current_schema() AND table_name='advert_route_evidence'`)).Scan(&count); err != nil { t.Fatal(err) } if count != 0 { @@ -362,10 +381,7 @@ func TestAdvertRouteEvidenceMissingTableIsUnknown(t *testing.T) { func TestAdvertRouteEvidenceTableAppearsAfterStartup(t *testing.T) { db, w := advertEvidenceFixture(t, "1100aa") - var schema string - if err := w.QueryRow(`SELECT sql FROM sqlite_master WHERE name='advert_route_evidence'`).Scan(&schema); err != nil { - t.Fatal(err) - } + var schema = testNativeSQL(`CREATE TABLE advert_route_evidence (id INTEGER PRIMARY KEY AUTOINCREMENT, tx_id INTEGER NOT NULL REFERENCES transmissions(id) ON DELETE CASCADE, bit INTEGER NOT NULL CHECK (bit IN (1,2)), UNIQUE(tx_id,bit))`, `CREATE TABLE advert_route_evidence (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, tx_id INTEGER NOT NULL REFERENCES transmissions(id) ON DELETE CASCADE, bit INTEGER NOT NULL CHECK (bit IN (1,2)), UNIQUE(tx_id,bit))`) if _, err := w.Exec(`DROP TABLE advert_route_evidence`); err != nil { t.Fatal(err) } @@ -397,7 +413,7 @@ func TestAdvertRouteEvidencePollRejectsPartialBatch(t *testing.T) { t.Fatal(err) } before := s.advertEvidenceCursor - if _, err := w.Exec(`PRAGMA ignore_check_constraints=ON; INSERT INTO advert_route_evidence(tx_id,bit) VALUES(1,2),(1,3)`); err != nil { + if _, err := w.Exec(testNativeSQL(`PRAGMA ignore_check_constraints=ON; INSERT INTO advert_route_evidence(tx_id,bit) VALUES(1,2),(1,3)`, `ALTER TABLE advert_route_evidence DROP CONSTRAINT advert_route_evidence_bit_check; INSERT INTO advert_route_evidence(tx_id,bit) VALUES(1,2),(1,3)`)); err != nil { t.Fatal(err) } if err := s.pollAdvertEvidence(500); err == nil { @@ -406,7 +422,7 @@ func TestAdvertRouteEvidencePollRejectsPartialBatch(t *testing.T) { if s.advertEvidenceCursor != before || s.byTxID[1].AdvertRouteEvidence != 1 { t.Fatal("partial failed batch changed evidence or cursor") } - if _, err := w.Exec(`DELETE FROM advert_route_evidence WHERE bit=3; PRAGMA ignore_check_constraints=OFF`); err != nil { + if _, err := w.Exec(`DELETE FROM advert_route_evidence WHERE bit=3; ALTER TABLE advert_route_evidence ADD CONSTRAINT advert_route_evidence_bit_check CHECK (bit IN (1,2))`); err != nil { t.Fatal(err) } if err := s.pollAdvertEvidence(500); err != nil { diff --git a/cmd/server/advert_stats.go b/cmd/server/advert_stats.go index fcde4b54a..10a14e0ba 100644 --- a/cmd/server/advert_stats.go +++ b/cmd/server/advert_stats.go @@ -63,10 +63,13 @@ func (db *DB) CountFloodAdvertsForNode(pubkey string, windowHours float64, rowCa } floor := time.Now().UTC().Add(-time.Duration(windowHours*float64(time.Hour))).AddDate(0, 0, -1).Format("2006-01-02") rows, err := db.conn.Query( - `SELECT COALESCE(first_seen, ''), ?, COALESCE(hash, '') FROM transmissions t - WHERE from_pubkey=? AND payload_type=? AND first_seen>=? - AND EXISTS(SELECT 1 FROM advert_route_evidence e WHERE e.tx_id=t.id AND e.bit=?) - ORDER BY id DESC LIMIT ?`, + db.nativeSQL(`SELECT COALESCE(first_seen, ''), CAST(?1 AS INTEGER), COALESCE(hash, '') FROM transmissions t + WHERE from_pubkey=?2 AND payload_type=?3 AND first_seen>=?4 + AND EXISTS(SELECT 1 FROM advert_route_evidence e WHERE e.tx_id=t.id AND e.bit=?5) + ORDER BY id DESC LIMIT ?6`, `SELECT COALESCE(first_seen, ''), `+db.parameter(1)+`::integer, COALESCE(hash, '') FROM transmissions t + WHERE from_pubkey=`+db.parameter(2)+` AND payload_type=`+db.parameter(3)+` AND first_seen>=`+db.parameter(4)+` + AND EXISTS(SELECT 1 FROM advert_route_evidence e WHERE e.tx_id=t.id AND e.bit=`+db.parameter(5)+`) + ORDER BY id DESC LIMIT `+db.parameter(6)), packetpath.AdvertFlood, pubkey, payloadTypeAdvert, floor, packetpath.AdvertFlood, rowCap) if err != nil { return 0, err diff --git a/cmd/server/advert_stats_test.go b/cmd/server/advert_stats_test.go index 942a094ae..9ef3038fc 100644 --- a/cmd/server/advert_stats_test.go +++ b/cmd/server/advert_stats_test.go @@ -15,16 +15,16 @@ import ( func insertAdvertTx(t *testing.T, db *DB, pubkey, hash string, rt int, ts time.Time) { t.Helper() if _, err := db.conn.Exec( - `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, from_pubkey) VALUES ('00', ?, ?, ?, ?, ?)`, + `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, from_pubkey) VALUES ('00', $1, $2, $3, $4, $5)`, hash, ts.Format("2006-01-02T15:04:05.000Z"), rt, payloadTypeAdvert, pubkey); err != nil { t.Fatalf("insert transmission: %v", err) } // Fixtures represent ingestor output: canonical routing alone is not evidence. - if _, err := db.conn.Exec(`CREATE TABLE IF NOT EXISTS advert_route_evidence(id INTEGER PRIMARY KEY AUTOINCREMENT,tx_id INTEGER,bit INTEGER,UNIQUE(tx_id,bit))`); err != nil { + if _, err := db.conn.Exec(testNativeSQL(`CREATE TABLE IF NOT EXISTS advert_route_evidence(id INTEGER PRIMARY KEY AUTOINCREMENT,tx_id INTEGER,bit INTEGER,UNIQUE(tx_id,bit))`, `CREATE TABLE IF NOT EXISTS advert_route_evidence(id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,tx_id INTEGER,bit INTEGER,UNIQUE(tx_id,bit))`)); err != nil { t.Fatal(err) } if rt == RouteFlood { - if _, err := db.conn.Exec(`INSERT INTO advert_route_evidence(tx_id,bit) SELECT id,1 FROM transmissions WHERE hash=?`, hash); err != nil { + if _, err := db.conn.Exec(`INSERT INTO advert_route_evidence(tx_id,bit) SELECT id,1 FROM transmissions WHERE hash=$1`, hash); err != nil { t.Fatal(err) } } @@ -64,7 +64,7 @@ func TestCountFloodAdverts(t *testing.T) { // counting recent flood adverts only (zero-hop and out-of-window excluded). func TestNodeDetailIncludesFloodAdvertCount(t *testing.T) { srv, router := setupTestServer(t) - if _, err := srv.db.conn.Exec(`CREATE TABLE advert_route_evidence(id INTEGER PRIMARY KEY AUTOINCREMENT,tx_id INTEGER,bit INTEGER,UNIQUE(tx_id,bit))`); err != nil { + if _, err := srv.db.conn.Exec(testNativeSQL(`CREATE TABLE advert_route_evidence(id INTEGER PRIMARY KEY AUTOINCREMENT,tx_id INTEGER,bit INTEGER,UNIQUE(tx_id,bit))`, `CREATE TABLE advert_route_evidence(id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,tx_id INTEGER,bit INTEGER,UNIQUE(tx_id,bit))`)); err != nil { t.Fatal(err) } now := time.Now().UTC() @@ -131,11 +131,14 @@ func TestNodeDetailFloodCountAgreesWithRouteEvidence(t *testing.T) { for _, firstRoute := range []int{RouteDirect, RouteFlood, RouteTransportFlood} { t.Run(fmt.Sprint(firstRoute), func(t *testing.T) { srv, router := setupTestServer(t) - if _, err := srv.db.conn.Exec(`CREATE TABLE IF NOT EXISTS advert_route_evidence(id INTEGER PRIMARY KEY AUTOINCREMENT, tx_id INTEGER, bit INTEGER, UNIQUE(tx_id,bit))`); err != nil { + if _, err := srv.db.conn.Exec(testNativeSQL(`CREATE TABLE IF NOT EXISTS advert_route_evidence(id INTEGER PRIMARY KEY AUTOINCREMENT, tx_id INTEGER, bit INTEGER, UNIQUE(tx_id,bit))`, `CREATE TABLE IF NOT EXISTS advert_route_evidence(id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, tx_id INTEGER, bit INTEGER, UNIQUE(tx_id,bit))`)); err != nil { t.Fatal(err) } const pubkey = "aabbccdd11223344" - if _, err := srv.db.conn.Exec(`DELETE FROM transmissions WHERE from_pubkey=?`, pubkey); err != nil { + if _, err := srv.db.conn.Exec(`DELETE FROM observations WHERE transmission_id IN (SELECT id FROM transmissions WHERE from_pubkey=$1)`, pubkey); err != nil { + t.Fatal(err) + } + if _, err := srv.db.conn.Exec(`DELETE FROM transmissions WHERE from_pubkey=$1`, pubkey); err != nil { t.Fatal(err) } insertAdvertTx(t, srv.db, pubkey, "mixed-advert", firstRoute, time.Now().Add(-time.Hour)) diff --git a/cmd/server/analytics_recompute_after_load_test.go b/cmd/server/analytics_recompute_after_load_test.go index b2a54cb50..34ee530bf 100644 --- a/cmd/server/analytics_recompute_after_load_test.go +++ b/cmd/server/analytics_recompute_after_load_test.go @@ -18,7 +18,6 @@ import ( "encoding/json" "net/http" "net/http/httptest" - "path/filepath" "sync/atomic" "testing" "time" @@ -51,9 +50,9 @@ func waitForTest(t *testing.T, what string, cond func() bool) { // still running (LoadComplete is already true at that point) and close // once RunStartupLoad returns. func TestStartupLoadDone_ClosesOnlyAfterBackgroundFill(t *testing.T) { - dbPath := filepath.Join(t.TempDir(), "test.db") + dbPath := postgresTestDSN(t) createTestDBSpreadOverDays(t, dbPath, 100, 14, time.Now().UTC().Unix()) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatalf("OpenDB: %v", err) } @@ -85,9 +84,9 @@ func TestStartupLoadDone_ClosesOnlyAfterBackgroundFill(t *testing.T) { // A failed load is terminal too: nothing more will be loaded, so the // signal must close instead of leaving the recomputers waiting forever. func TestStartupLoadDone_ClosesWhenLoadFails(t *testing.T) { - dbPath := filepath.Join(t.TempDir(), "test.db") + dbPath := postgresTestDSN(t) createTestDBSpreadOverDays(t, dbPath, 10, 1, time.Now().UTC().Unix()) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatalf("OpenDB: %v", err) } @@ -367,9 +366,9 @@ func TestDistanceIndexBuild_RefreshesRecomputerBeforeReportingBuilt(t *testing.T // right after the load. func TestAnalyticsRecomputers_FullDataRightAfterStartupLoad(t *testing.T) { const totalRows = 100 - dbPath := filepath.Join(t.TempDir(), "test.db") + dbPath := postgresTestDSN(t) createTestDBSpreadOverDays(t, dbPath, totalRows, 14, time.Now().UTC().Unix()) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatalf("OpenDB: %v", err) } @@ -434,9 +433,18 @@ func TestAnalyticsRecomputers_FullDataRightAfterStartupLoad(t *testing.T) { t.Fatalf("fixture precondition: %d packets in memory after load, want %d", inMemory, totalRows) } - waitForTest(t, "rf gate opens after full load", func() bool { return get("/api/analytics/rf").Code == http.StatusOK }) + // Post-load passes run sequentially: RF completing does not imply that + // topology or channels has published its full snapshot yet. + for _, name := range []string{"rf", "topology", "channels"} { + rc := all[name] + waitForTest(t, name+" first full pass finishes", func() bool { return !rc.FirstPassDoneAt_1659().IsZero() }) + } + rfResponse := get("/api/analytics/rf") + if rfResponse.Code != http.StatusOK { + t.Fatalf("rf after full load: got %d, want 200", rfResponse.Code) + } var rf map[string]interface{} - if err := json.Unmarshal(get("/api/analytics/rf").Body.Bytes(), &rf); err != nil { + if err := json.Unmarshal(rfResponse.Body.Bytes(), &rf); err != nil { t.Fatalf("rf body: %v", err) } if got, _ := rf["totalTransmissions"].(float64); int(got) != totalRows { diff --git a/cmd/server/auth_fixture_test.go b/cmd/server/auth_fixture_test.go index 00ea6d0ae..fe87e3641 100644 --- a/cmd/server/auth_fixture_test.go +++ b/cmd/server/auth_fixture_test.go @@ -8,14 +8,17 @@ import ( "net/http" "net/http/httptest" "net/url" - "path/filepath" "regexp" + "strings" "testing" "time" "github.com/gorilla/mux" + "github.com/meshcore-analyzer/dbconfig" "github.com/meshcore-analyzer/mailer" + "github.com/meshcore-analyzer/pgutil/pgtest" "github.com/meshcore-analyzer/users" + "modernc.org/sqlite" ) const ( @@ -25,10 +28,11 @@ const ( ) type authFixture struct { - srv *Server - router *mux.Router - fake *mailer.Fake - st *users.Store + ownerURL string + srv *Server + router *mux.Router + fake *mailer.Fake + st *users.Store } // client is a browser: its session cookie and CSRF token. @@ -38,10 +42,73 @@ type client struct { me meResponse } +// Account owners must use the same SQLite library as users.Store, not the +// telemetry driver: https://sqlite.org/howtocorrupt.html#multiple_copies_of_sqlite_linked_into_the_same_application +func openAccountFixtureSQL(target string) (*sql.DB, error) { + if strings.HasPrefix(target, "postgres://") || strings.HasPrefix(target, "postgresql://") { + return openFixtureSQL(target) + } + uri, err := dbconfig.SQLiteURI(target, url.Values{"mode": {"rw"}, "_pragma": {"busy_timeout(5000)"}}) + if err != nil { + return nil, err + } + db, err := sql.Open("sqlite", uri) + if err == nil { + db.SetMaxOpenConns(1) + } + return db, err +} + +// Account fixture owners share the live users.Store file. Mixing SQLite +// implementations in one process bypasses their per-library lock bookkeeping. +func TestAccountFixtureUsesAccountSQLiteDriver(t *testing.T) { + if testBackend(t) != dbconfig.SQLite { + t.Skip("SQLite account driver boundary") + } + f := newAuthFixture(t) + db, err := openAccountFixtureSQL(f.ownerURL) + if err != nil { + t.Fatal(err) + } + defer db.Close() + if _, ok := db.Driver().(*sqlite.Driver); !ok { + t.Fatalf("account fixture uses %T; users.Store uses modernc.org/sqlite", db.Driver()) + } + if err := db.Ping(); err != nil { + t.Fatal(err) + } +} + func newTestAuthService(t *testing.T, adminEmails ...string) (*authService, *mailer.Fake) { + a, fake, _ := newTestAuthServiceWithURL(t, postgresTestDSN(t), adminEmails...) + return a, fake +} + +func newTestBackupAuthService(t *testing.T, adminEmails ...string) (*authService, *mailer.Fake) { + a, fake, _ := newTestAuthServiceWithURL(t, testDatabaseDSN(t), adminEmails...) + return a, fake +} + +func newTestAuthServiceWithURL(t *testing.T, ownerURL string, adminEmails ...string) (*authService, *mailer.Fake, string) { t.Helper() + runtimeURL := ownerURL + if testBackend(t) == dbconfig.Postgres { + owner, err := openAccountFixtureSQL(ownerURL) + if err != nil { + t.Fatal(err) + } + defer owner.Close() + if err := users.Apply(owner); err != nil { + t.Fatal(err) + } + runtimeURL = pgtest.Writer(t, ownerURL) + u, _ := url.Parse(runtimeURL) + if _, err := owner.Exec(`REVOKE INSERT,UPDATE,DELETE ON corescope_schema,schema_version FROM "` + u.User.Username() + `"`); err != nil { + t.Fatal(err) + } + } set := &userMgmtSettings{ - dbPath: filepath.Join(t.TempDir(), "users.db"), adminEmails: map[string]bool{}, + databaseURL: runtimeURL, adminEmails: map[string]bool{}, sessionTTL: 30 * 24 * time.Hour, provider: "fake", fromEmail: "noreply@example.org", fromName: "CoreScope", webhookSecret: testHook, } @@ -50,7 +117,7 @@ func newTestAuthService(t *testing.T, adminEmails ...string) (*authService, *mai for _, e := range adminEmails { set.adminEmails[e] = true } - st, err := users.Open(set.dbPath) + st, err := users.Open(set.databaseURL) if err != nil { t.Fatal(err) } @@ -60,17 +127,25 @@ func newTestAuthService(t *testing.T, adminEmails ...string) (*authService, *mai a.waitAudits() st.Close() }) - return a, fake + return a, fake, ownerURL } // newAuthFixture builds a Server with auth on and only the auth routes. func newAuthFixture(t *testing.T, adminEmails ...string) *authFixture { + return newAuthFixtureWithURL(t, postgresTestDSN(t), adminEmails...) +} + +func newBackupAuthFixture(t *testing.T, adminEmails ...string) *authFixture { + return newAuthFixtureWithURL(t, testDatabaseDSN(t), adminEmails...) +} + +func newAuthFixtureWithURL(t *testing.T, databaseURL string, adminEmails ...string) *authFixture { t.Helper() - a, fake := newTestAuthService(t, adminEmails...) + a, fake, ownerURL := newTestAuthServiceWithURL(t, databaseURL, adminEmails...) srv := &Server{cfg: &Config{APIKey: testAPIKey}, perfStats: NewPerfStats(), auth: a} r := mux.NewRouter() srv.registerAuthRoutes(r) - return &authFixture{srv: srv, router: r, fake: fake, st: a.st} + return &authFixture{srv: srv, router: r, fake: fake, st: a.st, ownerURL: ownerURL} } type reqMod func(*http.Request) @@ -187,7 +262,7 @@ func (f *authFixture) login(t *testing.T, email, password string) *client { // store call that touches it fails with a DB error (not ErrNotFound). func (f *authFixture) breakTable(t *testing.T, table string) { t.Helper() - db, err := sql.Open("sqlite", f.srv.auth.set.dbPath) + db, err := openAccountFixtureSQL(f.ownerURL) if err != nil { t.Fatal(err) } @@ -201,13 +276,13 @@ func (f *authFixture) breakTable(t *testing.T, table string) { // from users.db (the raw tokens are not observable when no mail left). func (f *authFixture) unusedTokens(t *testing.T, uid int64, p users.Purpose) int { t.Helper() - db, err := sql.Open("sqlite", f.srv.auth.set.dbPath) + db, err := openAccountFixtureSQL(f.ownerURL) if err != nil { t.Fatal(err) } defer db.Close() var n int - if err := db.QueryRow(`SELECT COUNT(*) FROM tokens WHERE user_id = ? AND purpose = ? AND used_at IS NULL`, uid, string(p)).Scan(&n); err != nil { + if err := db.QueryRow(`SELECT COUNT(*) FROM tokens WHERE user_id = $1 AND purpose = $2 AND used_at IS NULL`, uid, string(p)).Scan(&n); err != nil { t.Fatal(err) } return n @@ -217,7 +292,7 @@ func (f *authFixture) unusedTokens(t *testing.T, uid int64, p users.Purpose) int // simulate a concurrent writer or a failing statement). func (f *authFixture) execDB(t *testing.T, stmt string) { t.Helper() - db, err := sql.Open("sqlite", f.srv.auth.set.dbPath) + db, err := openAccountFixtureSQL(f.ownerURL) if err != nil { t.Fatal(err) } diff --git a/cmd/server/auth_flow_test.go b/cmd/server/auth_flow_test.go index 3776bd79f..ef270c2ef 100644 --- a/cmd/server/auth_flow_test.go +++ b/cmd/server/auth_flow_test.go @@ -223,8 +223,7 @@ func TestAuthActivateHashChangedMidwayIs409(t *testing.T) { f := newAuthFixture(t) expectStatus(t, f.do("POST", "/api/auth/register", registerRequest{Email: "ned@example.org", DisplayName: "Ned", Password: pw}), 200) tok := f.lastToken(t) - f.execDB(t, `CREATE TRIGGER race AFTER UPDATE OF used_at ON tokens BEGIN - UPDATE users SET password_hash = 'replaced-by-a-re-register' WHERE id = NEW.user_id; END`) + f.execDB(t, testNativeSQL(`CREATE TRIGGER race AFTER UPDATE OF used_at ON tokens BEGIN UPDATE users SET password_hash='replaced-by-a-re-register' WHERE id=NEW.user_id; END`, `CREATE FUNCTION race() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN UPDATE users SET password_hash='replaced-by-a-re-register' WHERE id=NEW.user_id; RETURN NEW; END $$; CREATE TRIGGER race AFTER UPDATE OF used_at ON tokens FOR EACH ROW EXECUTE FUNCTION race()`)) w := f.do("POST", "/api/auth/activate", activateRequest{Token: tok, Password: pw}) expectStatus(t, w, 409) if !strings.Contains(w.Body.String(), "account changed, try again") { diff --git a/cmd/server/auth_login_audit_test.go b/cmd/server/auth_login_audit_test.go index 493c7542e..29f6076ca 100644 --- a/cmd/server/auth_login_audit_test.go +++ b/cmd/server/auth_login_audit_test.go @@ -2,7 +2,6 @@ package main import ( "context" - "database/sql" "strings" "testing" "time" @@ -134,7 +133,7 @@ func TestLoginAnswersBeforeTheAuditWrite(t *testing.T) { f := newAuthFixture(t) dave := f.registerAndActivate(t, "dave@example.org", "Dave", pw) - db, err := sql.Open("sqlite", f.srv.auth.set.dbPath) + db, err := openAccountFixtureSQL(f.ownerURL) if err != nil { t.Fatal(err) } @@ -144,7 +143,7 @@ func TestLoginAnswersBeforeTheAuditWrite(t *testing.T) { t.Fatal(err) } defer conn.Close() - if _, err := conn.ExecContext(context.Background(), "BEGIN IMMEDIATE"); err != nil { + if _, err := conn.ExecContext(context.Background(), testNativeSQL("BEGIN IMMEDIATE", "BEGIN; LOCK TABLE audit_log IN SHARE MODE")); err != nil { t.Fatal(err) } diff --git a/cmd/server/auth_mail_test.go b/cmd/server/auth_mail_test.go index 1314f77b9..23677dba0 100644 --- a/cmd/server/auth_mail_test.go +++ b/cmd/server/auth_mail_test.go @@ -5,7 +5,6 @@ import ( "context" "errors" "log" - "path/filepath" "strings" "testing" @@ -60,22 +59,16 @@ func TestActivationMailMentionsPassword(t *testing.T) { } } -// The startup line shows the absolute users.db path, so a relative path -// resolved against another working directory than the ingestor's is visible. -func TestLogStartupShowsAbsoluteUsersDBPath(t *testing.T) { +func TestLogStartupRedactsDatabaseURL(t *testing.T) { a, _ := newTestAuthService(t) - a.set.dbPath = "users.db" + a.set.databaseURL = "postgresql://private:secret@host/accounts?password=hidden" var buf bytes.Buffer prev := log.Writer() log.SetOutput(&buf) defer log.SetOutput(prev) a.logStartup() - want, err := filepath.Abs("users.db") - if err != nil { - t.Fatal(err) - } - if !strings.Contains(buf.String(), "[users] user management enabled: db="+want+",") { - t.Fatalf("log line = %q, want the absolute path %q", buf.String(), want) + if !strings.Contains(buf.String(), string(a.st.Backend())) || strings.Contains(buf.String(), "secret") || strings.Contains(buf.String(), "hidden") { + t.Fatalf("unsafe startup log: %q", buf.String()) } } diff --git a/cmd/server/auth_reregister_test.go b/cmd/server/auth_reregister_test.go index 6c157e82c..1cf8a6549 100644 --- a/cmd/server/auth_reregister_test.go +++ b/cmd/server/auth_reregister_test.go @@ -16,7 +16,10 @@ func TestAuthReregisterProfileFailureKeepsPassword(t *testing.T) { } sent := len(f.fake.Sent()) token := f.lastToken(t) - f.execDB(t, `CREATE TRIGGER reject_name BEFORE UPDATE OF display_name ON users BEGIN SELECT RAISE(ABORT, 'profile update failed'); END`) + f.execDB(t, testNativeSQL( + `CREATE TRIGGER reject_name BEFORE UPDATE OF display_name ON users BEGIN SELECT RAISE(ABORT, 'profile update failed'); END`, + `CREATE FUNCTION reject_name() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'profile update failed'; END $$; + CREATE TRIGGER reject_name BEFORE UPDATE OF display_name ON users FOR EACH ROW EXECUTE FUNCTION reject_name()`)) w := f.do("POST", "/api/auth/register", registerRequest{Email: u.Email, DisplayName: "Replacement", Password: "replacement password"}) expectStatus(t, w, 200) if response := decode[okResponse](t, w); !response.OK || response.Message != msgCheckMail { @@ -46,10 +49,14 @@ func TestAuthReregisterChangedAccountDoesNotIssueActivation(t *testing.T) { t.Fatal(err) } sent := len(f.fake.Sent()) - f.execDB(t, `CREATE TRIGGER activate_first BEFORE UPDATE OF password_hash ON users BEGIN + f.execDB(t, testNativeSQL(`CREATE TRIGGER activate_first BEFORE UPDATE OF password_hash ON users BEGIN UPDATE users SET status='active', role='admin', activated_at=123 WHERE id=NEW.id; SELECT RAISE(IGNORE); - END`) + END`, `CREATE FUNCTION activate_first() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN + UPDATE users SET status='active', role='admin', activated_at=123 WHERE id=NEW.id; + RETURN NULL; + END $$; + CREATE TRIGGER activate_first BEFORE UPDATE OF password_hash ON users FOR EACH ROW EXECUTE FUNCTION activate_first()`)) w := f.do("POST", "/api/auth/register", registerRequest{Email: u.Email, DisplayName: "Replacement", Password: "replacement password"}) expectStatus(t, w, 200) if response := decode[okResponse](t, w); !response.OK || response.Message != msgCheckMail { diff --git a/cmd/server/auth_service.go b/cmd/server/auth_service.go index 274edcab3..d63120754 100644 --- a/cmd/server/auth_service.go +++ b/cmd/server/auth_service.go @@ -1,9 +1,11 @@ package main import ( + "github.com/meshcore-analyzer/dbconfig" "log" "os" "path/filepath" + "strings" "sync" "sync/atomic" "time" @@ -60,20 +62,34 @@ func newAuthService(set *userMgmtSettings, st *users.Store, m mailer.Mailer) *au } // initUserManagement builds s.auth when the feature is on. Call it after -// NewServer and before RegisterRoutes. measurementDBPath is passed to -// users.Open as a forbidden path, so users.db can never be the analyzer DB. -func (s *Server) initUserManagement(measurementDBPath string) error { +// NewServer and before RegisterRoutes. measurementDatabaseURL is passed to +// users.Open as a forbidden database, so accounts cannot share telemetry storage. +func (s *Server) initUserManagement(measurementTarget string) error { if !s.cfg.UserManagementEnabled() { if s.cfg.ClientRxRequireLinkedCompanionSet() { log.Printf("[users] clientRxCoverage.requireLinkedCompanion is set but userManagement is off: the setting is ignored") } return nil } - set, err := resolveUserManagement(s.cfg.UserManagement, measurementDBPath, os.Getenv) + set, err := resolveUserManagement(s.cfg.UserManagement, runtimeStateDir, os.Getenv) if err != nil { return err } - st, err := users.Open(set.dbPath, measurementDBPath) + storage := dbconfig.Storage{Backend: dbconfig.SQLite, DBPath: measurementTarget, UsersDBPath: set.databaseURL, StateDir: runtimeStateDir} + if strings.HasPrefix(measurementTarget, "postgres://") || strings.HasPrefix(measurementTarget, "postgresql://") { + storage = dbconfig.Storage{Backend: dbconfig.Postgres, ReaderDatabaseURL: measurementTarget, UsersDatabaseURL: set.databaseURL, StateDir: runtimeStateDir} + } + return s.initUserManagementStorage(storage) +} +func (s *Server) initUserManagementStorage(storage dbconfig.Storage) error { + if !s.cfg.UserManagementEnabled() { + return nil + } + set, err := resolveUserManagement(s.cfg.UserManagement, storage.StateDir, os.Getenv) + if err != nil { + return err + } + st, err := users.OpenStorage(storage) if err != nil { return err } @@ -106,7 +122,7 @@ func (s *Server) initUserManagement(measurementDBPath string) error { return nil } -// closeUserManagement stops the janitor and closes users.db. Safe when off. +// closeUserManagement stops the janitor and closes the account connection pool. Safe when off. func (s *Server) closeUserManagement() { if s.auth == nil { return @@ -122,8 +138,8 @@ func (s *Server) closeUserManagement() { } func (a *authService) logStartup() { - log.Printf("[users] user management enabled: db=%s, %d config admin(s), webhook=%v", - absForLog(a.set.dbPath), len(a.set.adminEmails), a.set.webhookSecret != "") + log.Printf("[users] user management enabled: %s, %d config admin(s), webhook=%v", + a.st.Backend(), len(a.set.adminEmails), a.set.webhookSecret != "") admins, err := a.st.List(users.ListFilter{Role: users.RoleAdmin}) if err != nil { log.Printf("[users] list admins: %v", err) diff --git a/cmd/server/backup.go b/cmd/server/backup.go index 91915ed1a..a362118da 100644 --- a/cmd/server/backup.go +++ b/cmd/server/backup.go @@ -7,29 +7,16 @@ import ( "net/http" "os" "path/filepath" - "strings" "time" + + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/pgutil" + "github.com/meshcore-analyzer/sqliteutil" ) -// handleBackup streams a consistent SQLite snapshot of the analyzer DB. -// -// Requires API-key authentication (mounted via requireAPIKey in routes.go). -// -// Strategy: SQLite's `VACUUM INTO 'path'` produces an atomic, defragmented -// copy of the current database into a new file. It runs at READ ISOLATION -// against the source DB (works on our read-only connection) and never -// blocks concurrent writers — the ingestor keeps writing to the WAL while -// the snapshot is taken from a consistent read transaction. -// -// Response: -// -// 200 OK -// Content-Type: application/octet-stream -// Content-Disposition: attachment; filename="corescope-backup-.db" -// -// -// The temp file is removed after the response is fully written, regardless -// of whether the client successfully consumed the stream. +// handleBackup stages a consistent native SQLite file or PostgreSQL archive. +// pgutil passes credentials through the child environment, bounds its lifetime, +// removes partial output, and returns redacted errors. func (s *Server) handleBackup(w http.ResponseWriter, r *http.Request) { if s.db == nil || s.db.conn == nil { writeError(w, http.StatusServiceUnavailable, "database unavailable") @@ -56,12 +43,18 @@ func (s *Server) handleBackup(w http.ResponseWriter, r *http.Request) { } }() - snapshotPath := filepath.Join(tmpDir, fmt.Sprintf("corescope-backup-%d.db", ts)) + extension := ".dump" + if s.db.Backend() == dbconfig.SQLite { + extension = ".db" + } + snapshotPath := filepath.Join(tmpDir, fmt.Sprintf("corescope-backup-%d%s", ts, extension)) - // SQLite parses the path literal — escape any single quotes defensively. - // (mkdtemp output won't contain quotes, but be paranoid for future-proofing.) - escaped := strings.ReplaceAll(snapshotPath, "'", "''") - if _, err := s.db.conn.ExecContext(r.Context(), fmt.Sprintf("VACUUM INTO '%s'", escaped)); err != nil { + if s.db.Backend() == dbconfig.SQLite { + err = sqliteutil.Snapshot(r.Context(), s.db.path, snapshotPath) + } else { + err = pgutil.Dump(r.Context(), s.db.path, snapshotPath) + } + if err != nil { writeError(w, http.StatusInternalServerError, "snapshot failed: "+err.Error()) return } @@ -78,8 +71,9 @@ func (s *Server) handleBackup(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Length", fmt.Sprintf("%d", stat.Size())) } w.Header().Set("Content-Type", "application/octet-stream") - w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"corescope-backup-%d.db\"", ts)) + w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"corescope-backup-%d%s\"", ts, extension)) w.Header().Set("X-Content-Type-Options", "nosniff") + w.Header().Set("Cache-Control", "no-store") w.WriteHeader(http.StatusOK) if _, err := io.Copy(w, f); err != nil { diff --git a/cmd/server/backup_test.go b/cmd/server/backup_test.go index 024ef3cb1..d4c4f0c8f 100644 --- a/cmd/server/backup_test.go +++ b/cmd/server/backup_test.go @@ -3,13 +3,32 @@ package main import ( "net/http" "net/http/httptest" + "net/url" + "os" + "path/filepath" "strings" "testing" ) -// sqliteMagic is the 16-byte file header identifying a valid SQLite 3 database. -// See https://www.sqlite.org/fileformat.html#magic_header_string -const sqliteMagic = "SQLite format 3\x00" +// postgresMagic is the signature of a PostgreSQL custom-format archive. +var postgresMagic = testNativeSQL("SQLite format 3\x00", "PGDMP") + +// Installed PostgreSQL selections pin even the default schema explicitly. +func selectedBackupTestDSN(t *testing.T) string { + t.Helper() + dsn := testDatabaseDSN(t) + if testBackendValue() == "postgres" { + u, err := url.Parse(dsn) + if err != nil { + t.Fatal("invalid fixture URL") + } + q := u.Query() + q.Set("search_path", `"public"`) + u.RawQuery = q.Encode() + dsn = u.String() + } + return dsn +} func TestBackupRequiresAPIKey(t *testing.T) { _, router := setupTestServerWithAPIKey(t, "test-secret-key-strong-enough") @@ -22,9 +41,18 @@ func TestBackupRequiresAPIKey(t *testing.T) { } } -func TestBackupReturnsValidSQLiteSnapshot(t *testing.T) { +func TestBackupRestoresNativePostgresSnapshot(t *testing.T) { const apiKey = "test-secret-key-strong-enough" - _, router := setupTestServerWithAPIKey(t, apiKey) + srv, router := setupTestServerWithAPIKey(t, apiKey) + srv.db = setupTestDBAtURL(t, selectedBackupTestDSN(t)) + seedTestData(t, srv.db) + writerURL := srv.db.path + reader, err := openFixtureReader(t, writerURL) + if err != nil { + t.Fatal(err) + } + defer reader.Close() + srv.db = reader req := httptest.NewRequest("GET", "/api/backup", nil) req.Header.Set("X-API-Key", apiKey) @@ -41,15 +69,31 @@ func TestBackupReturnsValidSQLiteSnapshot(t *testing.T) { } cd := w.Header().Get("Content-Disposition") - if !strings.HasPrefix(cd, "attachment;") || !strings.Contains(cd, "filename=\"corescope-backup-") || !strings.HasSuffix(cd, ".db\"") { - t.Errorf("expected Content-Disposition attachment with corescope-backup-.db filename, got %q", cd) + if !strings.HasPrefix(cd, "attachment;") || !strings.Contains(cd, "filename=\"corescope-backup-") || !strings.HasSuffix(cd, testNativeSQL(".db\"", ".dump\"")) { + t.Errorf("expected Content-Disposition attachment with corescope-backup-.dump filename, got %q", cd) } body := w.Body.Bytes() - if len(body) < len(sqliteMagic) { - t.Fatalf("backup body too short (%d bytes) — expected SQLite file", len(body)) + if len(body) < len(postgresMagic) { + t.Fatalf("backup body too short (%d bytes) — expected PostgreSQL file", len(body)) + } + if got := string(body[:len(postgresMagic)]); got != postgresMagic { + t.Fatalf("expected PostgreSQL magic header %q, got %q", postgresMagic, got) } - if got := string(body[:len(sqliteMagic)]); got != sqliteMagic { - t.Fatalf("expected SQLite magic header %q, got %q", sqliteMagic, got) + path := filepath.Join(t.TempDir(), "telemetry.dump") + if err := os.WriteFile(path, body, 0o600); err != nil { + t.Fatal(err) } + restored := restorePostgresTestBackup(t, path) + var transmissions, observations int + if err := restored.QueryRow(`SELECT COUNT(*) FROM transmissions`).Scan(&transmissions); err != nil { + t.Fatal(err) + } + if err := restored.QueryRow(`SELECT COUNT(*) FROM observations`).Scan(&observations); err != nil { + t.Fatal(err) + } + if transmissions != 3 || observations != 4 { + t.Fatalf("restored counts = %d/%d", transmissions, observations) + } + } diff --git a/cmd/server/bounded_load_test.go b/cmd/server/bounded_load_test.go index ef383d549..1c5379429 100644 --- a/cmd/server/bounded_load_test.go +++ b/cmd/server/bounded_load_test.go @@ -1,21 +1,15 @@ package main import ( - "database/sql" "fmt" - "os" - "path/filepath" "testing" "time" - - _ "github.com/mattn/go-sqlite3" ) // createTestDB creates a temporary SQLite database with N transmissions (1 obs each). func createTestDB(t *testing.T, numTx int) string { t.Helper() - dir := t.TempDir() - dbPath := filepath.Join(dir, "test.db") + dbPath := postgresTestDSN(t) createTestDBAt(t, dbPath, numTx) return dbPath } @@ -23,7 +17,7 @@ func createTestDB(t *testing.T, numTx int) string { // loadStore creates a PacketStore from a test DB with given maxMemoryMB. func loadStore(t *testing.T, dbPath string, maxMemMB int) *PacketStore { t.Helper() - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } @@ -37,7 +31,6 @@ func loadStore(t *testing.T, dbPath string, maxMemMB int) *PacketStore { func TestBoundedLoad_LimitedMemory(t *testing.T) { dbPath := createTestDB(t, 5000) - defer os.RemoveAll(filepath.Dir(dbPath)) // Use 1MB budget — should load far fewer than 5000 packets store := loadStore(t, dbPath, 1) @@ -47,15 +40,14 @@ func TestBoundedLoad_LimitedMemory(t *testing.T) { if loaded >= 5000 { t.Errorf("expected bounded load to limit packets, got %d/5000", loaded) } - if loaded < 1000 { - t.Errorf("expected at least 1000 packets (minimum), got %d", loaded) + if loaded == 0 || store.trackedBytes > 1048576 { + t.Errorf("expected a nonempty cache within 1 MiB, got %d packets / %d bytes", loaded, store.trackedBytes) } t.Logf("Loaded %d/5000 packets with 1MB budget", loaded) } func TestBoundedLoad_NewestFirst(t *testing.T) { dbPath := createTestDB(t, 5000) - defer os.RemoveAll(filepath.Dir(dbPath)) store := loadStore(t, dbPath, 1) defer store.db.conn.Close() @@ -85,7 +77,6 @@ func TestBoundedLoad_NewestFirst(t *testing.T) { func TestBoundedLoad_OldestLoadedSet(t *testing.T) { dbPath := createTestDB(t, 5000) - defer os.RemoveAll(filepath.Dir(dbPath)) store := loadStore(t, dbPath, 1) defer store.db.conn.Close() @@ -93,15 +84,18 @@ func TestBoundedLoad_OldestLoadedSet(t *testing.T) { if store.oldestLoaded == "" { t.Fatal("oldestLoaded should be set after bounded load") } - if len(store.packets) > 0 && store.oldestLoaded != store.packets[0].FirstSeen { - t.Errorf("oldestLoaded (%s) should match first packet (%s)", store.oldestLoaded, store.packets[0].FirstSeen) + if len(store.packets) > 0 && store.oldestLoaded > store.packets[0].FirstSeen { + t.Errorf("coverage floor (%s) should include first packet (%s)", store.oldestLoaded, store.packets[0].FirstSeen) + } + oldestOmitted := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC).Add(time.Duration(5000-len(store.packets)) * time.Minute).Format(time.RFC3339) + if store.oldestLoaded <= oldestOmitted { + t.Errorf("coverage floor %s claims omitted packet %s", store.oldestLoaded, oldestOmitted) } t.Logf("oldestLoaded = %s", store.oldestLoaded) } func TestBoundedLoad_UnlimitedWithZero(t *testing.T) { dbPath := createTestDB(t, 200) - defer os.RemoveAll(filepath.Dir(dbPath)) store := loadStore(t, dbPath, 0) defer store.db.conn.Close() @@ -113,7 +107,6 @@ func TestBoundedLoad_UnlimitedWithZero(t *testing.T) { func TestBoundedLoad_AscendingOrder(t *testing.T) { dbPath := createTestDB(t, 3000) - defer os.RemoveAll(filepath.Dir(dbPath)) store := loadStore(t, dbPath, 1) defer store.db.conn.Close() @@ -130,7 +123,7 @@ func TestBoundedLoad_AscendingOrder(t *testing.T) { // loadStoreWithRetention creates a PacketStore with retentionHours set. func loadStoreWithRetention(t *testing.T, dbPath string, retentionHours float64) *PacketStore { t.Helper() - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } @@ -146,10 +139,9 @@ func loadStoreWithRetention(t *testing.T, dbPath string, retentionHours float64) // the last hour and numOld packets with timestamps 48 hours ago. func createTestDBWithAgedPackets(t *testing.T, numRecent, numOld int) string { t.Helper() - dir := t.TempDir() - dbPath := filepath.Join(dir, "test.db") + dbPath := postgresTestDSN(t) - conn, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL") + conn, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } @@ -160,10 +152,10 @@ func createTestDBWithAgedPackets(t *testing.T, numRecent, numOld int) string { t.Fatalf("setup: %v\nSQL: %s", err, s) } } - execOrFail(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT)`) - execOrFail(`CREATE TABLE observations (id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, direction TEXT, snr REAL, rssi REAL, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT)`) - execOrFail(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) - execOrFail(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, frequency REAL)`) + execOrFail(testNativeSQL(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY AUTOINCREMENT, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT)`, `CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT)`)) + execOrFail(testNativeSQL(`CREATE TABLE observations (id INTEGER PRIMARY KEY AUTOINCREMENT, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp BIGINT, raw_hex TEXT)`, `CREATE TABLE observations (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp BIGINT, raw_hex TEXT)`)) + execOrFail(testNativeSQL(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY AUTOINCREMENT, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, `CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`)) + execOrFail(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, first_seen TEXT, frequency DOUBLE PRECISION)`) execOrFail(`CREATE TABLE schema_version (version INTEGER)`) execOrFail(`INSERT INTO schema_version (version) VALUES (1)`) execOrFail(`CREATE INDEX idx_tx_first_seen ON transmissions(first_seen)`) @@ -174,28 +166,29 @@ func createTestDBWithAgedPackets(t *testing.T, numRecent, numOld int) string { // (auto-commit per Exec fsyncs per row). numOld+numRecent // is small here today, but wrapping keeps the fixture robust if callers scale // it up, and is consistent with the other builders. - if _, err := conn.Exec("BEGIN"); err != nil { + fixtureTx, err := conn.Begin() + if err != nil { t.Fatalf("test DB BEGIN: %v", err) } // Insert old packets (48 hours ago) for i := 0; i < numOld; i++ { oldT := now.Add(-48 * time.Hour).Add(time.Duration(i) * time.Second) ts := oldT.Format(time.RFC3339) - conn.Exec("INSERT INTO transmissions VALUES (?,?,?,?,0,4,1,?)", id, "aa", fmt.Sprintf("old%d", i), ts, `{}`) + fixtureTx.Exec("INSERT INTO transmissions VALUES ($1,$2,$3,$4,0,4,1,$5)", id, "aa", fmt.Sprintf("old%d", i), ts, `{}`) // observations.timestamp is INTEGER (unix seconds) in production schema // — keep the fixture consistent so the RFC3339 subquery matches. - conn.Exec("INSERT INTO observations VALUES (?,?,?,?,?,?,?,?,?,?,?)", id, id, "obs1", "Obs1", "RX", -10.0, -80.0, 5, `[]`, oldT.Unix(), "") + fixtureTx.Exec("INSERT INTO observations VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11)", id, id, "obs1", "Obs1", "RX", -10.0, -80.0, 5, `[]`, oldT.Unix(), "") id++ } // Insert recent packets (within last hour) for i := 0; i < numRecent; i++ { newT := now.Add(-30 * time.Minute).Add(time.Duration(i) * time.Second) ts := newT.Format(time.RFC3339) - conn.Exec("INSERT INTO transmissions VALUES (?,?,?,?,0,4,1,?)", id, "bb", fmt.Sprintf("new%d", i), ts, `{}`) - conn.Exec("INSERT INTO observations VALUES (?,?,?,?,?,?,?,?,?,?,?)", id, id, "obs1", "Obs1", "RX", -10.0, -80.0, 5, `[]`, newT.Unix(), "") + fixtureTx.Exec("INSERT INTO transmissions VALUES ($1,$2,$3,$4,0,4,1,$5)", id, "bb", fmt.Sprintf("new%d", i), ts, `{}`) + fixtureTx.Exec("INSERT INTO observations VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11)", id, id, "obs1", "Obs1", "RX", -10.0, -80.0, 5, `[]`, newT.Unix(), "") id++ } - if _, err := conn.Exec("COMMIT"); err != nil { + if err := fixtureTx.Commit(); err != nil { t.Fatalf("test DB COMMIT: %v", err) } return dbPath @@ -203,7 +196,6 @@ func createTestDBWithAgedPackets(t *testing.T, numRecent, numOld int) string { func TestRetentionLoad_OnlyLoadsRecentPackets(t *testing.T) { dbPath := createTestDBWithAgedPackets(t, 50, 100) - defer os.RemoveAll(filepath.Dir(dbPath)) // retention = 2 hours — should load only the 50 recent packets, not the 100 old ones store := loadStoreWithRetention(t, dbPath, 2) @@ -216,7 +208,6 @@ func TestRetentionLoad_OnlyLoadsRecentPackets(t *testing.T) { func TestRetentionLoad_ZeroRetentionLoadsAll(t *testing.T) { dbPath := createTestDBWithAgedPackets(t, 50, 100) - defer os.RemoveAll(filepath.Dir(dbPath)) // retention = 0 (unlimited) — should load all 150 packets store := loadStoreWithRetention(t, dbPath, 0) @@ -242,13 +233,12 @@ func TestEstimateStoreTxBytesTypical(t *testing.T) { } func BenchmarkLoad_Bounded(b *testing.B) { - dir := b.TempDir() - dbPath := filepath.Join(dir, "bench.db") + dbPath := postgresTestDSN(b) createTestDBAt(b, dbPath, 5000) b.ResetTimer() for i := 0; i < b.N; i++ { - db, _ := OpenDB(dbPath) + db, _ := openFixtureReader(b, dbPath) cfg := &PacketStoreConfig{MaxMemoryMB: 1} store := NewPacketStore(db, cfg) store.Load() @@ -257,13 +247,12 @@ func BenchmarkLoad_Bounded(b *testing.B) { } func BenchmarkLoad_Unlimited(b *testing.B) { - dir := b.TempDir() - dbPath := filepath.Join(dir, "bench.db") + dbPath := postgresTestDSN(b) createTestDBAt(b, dbPath, 5000) b.ResetTimer() for i := 0; i < b.N; i++ { - db, _ := OpenDB(dbPath) + db, _ := openFixtureReader(b, dbPath) cfg := &PacketStoreConfig{MaxMemoryMB: 0} store := NewPacketStore(db, cfg) store.Load() @@ -274,13 +263,12 @@ func BenchmarkLoad_Unlimited(b *testing.B) { // BenchmarkLoad_30K_Bounded benchmarks bounded Load() with 30K transmissions // and realistic observation counts (1–5 per transmission). func BenchmarkLoad_30K_Bounded(b *testing.B) { - dir := b.TempDir() - dbPath := filepath.Join(dir, "bench30k.db") + dbPath := postgresTestDSN(b) createTestDBWithObs(b, dbPath, 30000) b.ResetTimer() for i := 0; i < b.N; i++ { - db, _ := OpenDB(dbPath) + db, _ := openFixtureReader(b, dbPath) cfg := &PacketStoreConfig{MaxMemoryMB: 50} store := NewPacketStore(db, cfg) store.Load() @@ -291,13 +279,12 @@ func BenchmarkLoad_30K_Bounded(b *testing.B) { // BenchmarkLoad_30K_Unlimited benchmarks unlimited Load() with 30K transmissions // and realistic observation counts (1–5 per transmission). func BenchmarkLoad_30K_Unlimited(b *testing.B) { - dir := b.TempDir() - dbPath := filepath.Join(dir, "bench30k.db") + dbPath := postgresTestDSN(b) createTestDBWithObs(b, dbPath, 30000) b.ResetTimer() for i := 0; i < b.N; i++ { - db, _ := OpenDB(dbPath) + db, _ := openFixtureReader(b, dbPath) cfg := &PacketStoreConfig{MaxMemoryMB: 0} store := NewPacketStore(db, cfg) store.Load() @@ -308,7 +295,7 @@ func BenchmarkLoad_30K_Unlimited(b *testing.B) { // createTestDBAt is like createTestDB but writes to a specific path. func createTestDBAt(tb testing.TB, dbPath string, numTx int) { tb.Helper() - conn, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL") + conn, err := openFixtureSQL(dbPath) if err != nil { tb.Fatal(err) } @@ -319,67 +306,52 @@ func createTestDBAt(tb testing.TB, dbPath string, numTx int) { tb.Fatalf("test DB setup exec failed: %v\nSQL: %s", err, sql) } } - execOrFail(`CREATE TABLE IF NOT EXISTS transmissions ( - id INTEGER PRIMARY KEY, + execOrFail(testNativeSQL(`CREATE TABLE IF NOT EXISTS transmissions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT - )`) - execOrFail(`CREATE TABLE IF NOT EXISTS observations ( - id INTEGER PRIMARY KEY, + )`, `CREATE TABLE IF NOT EXISTS transmissions ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + raw_hex TEXT, hash TEXT, first_seen TEXT, + route_type INTEGER, payload_type INTEGER, + payload_version INTEGER, decoded_json TEXT + )`)) + execOrFail(testNativeSQL(`CREATE TABLE IF NOT EXISTS observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, - direction TEXT, snr REAL, rssi REAL, score INTEGER, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT - )`) - execOrFail(`CREATE TABLE IF NOT EXISTS observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) + )`, `CREATE TABLE IF NOT EXISTS observations ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + transmission_id INTEGER, observer_id TEXT, observer_name TEXT, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, + path_json TEXT, timestamp TEXT, raw_hex TEXT + )`)) + execOrFail(testNativeSQL(`CREATE TABLE IF NOT EXISTS observers (rowid INTEGER PRIMARY KEY AUTOINCREMENT, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, `CREATE TABLE IF NOT EXISTS observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`)) execOrFail(`CREATE TABLE IF NOT EXISTS nodes ( - public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, - last_seen TEXT, first_seen TEXT, frequency REAL + public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, + last_seen TEXT, first_seen TEXT, frequency DOUBLE PRECISION )`) execOrFail(`CREATE TABLE IF NOT EXISTS schema_version (version INTEGER)`) execOrFail(`INSERT INTO schema_version (version) VALUES (1)`) execOrFail(`CREATE INDEX IF NOT EXISTS idx_tx_first_seen ON transmissions(first_seen)`) - txStmt, err := conn.Prepare("INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) VALUES (?, ?, ?, ?, ?, ?, ?, ?)") - if err != nil { - tb.Fatalf("test DB prepare transmissions insert: %v", err) - } - obsStmt, err := conn.Prepare("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") - if err != nil { - tb.Fatalf("test DB prepare observations insert: %v", err) - } - defer txStmt.Close() - defer obsStmt.Close() - base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) - // Wrap the inserts in a single transaction. Without this, SQLite - // (pure-Go driver) auto-commits every Exec → one fsync per row → ~2N fsyncs - // for N transmissions (tx + obs). At numTx=5000 that is ~10k fsyncs and the - // fixture blows past the test timeout (the #1741 hang). A single - // BEGIN/COMMIT makes the whole build one commit, finishing in well under a - // second regardless of numTx. - if _, err := conn.Exec("BEGIN"); err != nil { - tb.Fatalf("test DB BEGIN: %v", err) - } - for i := 1; i <= numTx; i++ { - ts := base.Add(time.Duration(i) * time.Minute).Format(time.RFC3339) - hash := fmt.Sprintf("h%04d", i) - if _, err := txStmt.Exec(i, "aabb", hash, ts, 0, 4, 1, fmt.Sprintf(`{"pubKey":"pk%04d"}`, i)); err != nil { - tb.Fatalf("test DB insert transmission %d: %v", i, err) - } - if _, err := obsStmt.Exec(i, i, "obs1", "Obs1", "RX", -10.0, -80.0, 5, `["aa","bb"]`, ts); err != nil { - tb.Fatalf("test DB insert observation %d: %v", i, err) - } - } - if _, err := conn.Exec("COMMIT"); err != nil { - tb.Fatalf("test DB COMMIT: %v", err) - } + copyTestRows(tb, conn, "transmissions", []string{"id", "raw_hex", "hash", "first_seen", "route_type", "payload_type", "payload_version", "decoded_json"}, numTx, func(n int) []any { + i := n + 1 + return []any{i, "aabb", fmt.Sprintf("h%04d", i), base.Add(time.Duration(i) * time.Minute).Format(time.RFC3339), 0, 4, 1, fmt.Sprintf(`{"pubKey":"pk%04d"}`, i)} + }) + copyTestRows(tb, conn, "observations", []string{"id", "transmission_id", "observer_id", "observer_name", "direction", "snr", "rssi", "score", "path_json", "timestamp"}, numTx, func(n int) []any { + i := n + 1 + return []any{i, i, "obs1", "Obs1", "RX", -10.0, -80.0, 5, `["aa","bb"]`, base.Add(time.Duration(i) * time.Minute).Format(time.RFC3339)} + }) } // createTestDBWithObs creates a test DB with realistic observation counts (1–5 per tx). func createTestDBWithObs(tb testing.TB, dbPath string, numTx int) { tb.Helper() - conn, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL") + conn, err := openFixtureSQL(dbPath) if err != nil { tb.Fatal(err) } @@ -390,61 +362,48 @@ func createTestDBWithObs(tb testing.TB, dbPath string, numTx int) { tb.Fatalf("test DB setup exec failed: %v\nSQL: %s", err, sqlStr) } } - execOrFail(`CREATE TABLE IF NOT EXISTS transmissions ( - id INTEGER PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, + execOrFail(testNativeSQL(`CREATE TABLE IF NOT EXISTS transmissions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT - )`) - execOrFail(`CREATE TABLE IF NOT EXISTS observations ( - id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, - direction TEXT, snr REAL, rssi REAL, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT - )`) - execOrFail(`CREATE TABLE IF NOT EXISTS observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) + )`, `CREATE TABLE IF NOT EXISTS transmissions ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, + route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT + )`)) + execOrFail(testNativeSQL(`CREATE TABLE IF NOT EXISTS observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT + )`, `CREATE TABLE IF NOT EXISTS observations ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT + )`)) + execOrFail(testNativeSQL(`CREATE TABLE IF NOT EXISTS observers (rowid INTEGER PRIMARY KEY AUTOINCREMENT, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, `CREATE TABLE IF NOT EXISTS observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`)) execOrFail(`CREATE TABLE IF NOT EXISTS nodes ( - public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, - last_seen TEXT, first_seen TEXT, frequency REAL + public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, + last_seen TEXT, first_seen TEXT, frequency DOUBLE PRECISION )`) execOrFail(`CREATE TABLE IF NOT EXISTS schema_version (version INTEGER)`) execOrFail(`INSERT INTO schema_version (version) VALUES (1)`) execOrFail(`CREATE INDEX IF NOT EXISTS idx_tx_first_seen ON transmissions(first_seen)`) - txStmt, err := conn.Prepare("INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) VALUES (?, ?, ?, ?, ?, ?, ?, ?)") - if err != nil { - tb.Fatalf("test DB prepare transmissions: %v", err) - } - obsStmt, err := conn.Prepare("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") - if err != nil { - tb.Fatalf("test DB prepare observations: %v", err) - } - defer txStmt.Close() - defer obsStmt.Close() - - observers := []string{"obs1", "obs2", "obs3", "obs4", "obs5"} - obsNames := []string{"Alpha", "Bravo", "Charlie", "Delta", "Echo"} - obsID := 1 base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) - // Single transaction for all inserts — see createTestDBAt for the rationale - // (auto-commit per Exec would fsync per row; at numTx=30000 - // the benchmarks would otherwise stall for minutes). One BEGIN/COMMIT. - if _, err := conn.Exec("BEGIN"); err != nil { - tb.Fatalf("test DB BEGIN: %v", err) - } + copyTestRows(tb, conn, "transmissions", []string{"id", "raw_hex", "hash", "first_seen", "route_type", "payload_type", "payload_version", "decoded_json"}, numTx, func(n int) []any { + i := n + 1 + return []any{i, "aabb", fmt.Sprintf("h%06d", i), base.Add(time.Duration(i) * time.Minute).Format(time.RFC3339), 0, 4, 1, fmt.Sprintf(`{"pubKey":"pk%06d"}`, i)} + }) + observers := []string{"obs1", "obs2", "obs3", "obs4", "obs5"} + names := []string{"Alpha", "Bravo", "Charlie", "Delta", "Echo"} + totalObs := 0 for i := 1; i <= numTx; i++ { - ts := base.Add(time.Duration(i) * time.Minute).Format(time.RFC3339) - hash := fmt.Sprintf("h%06d", i) - if _, err := txStmt.Exec(i, "aabb", hash, ts, 0, 4, 1, fmt.Sprintf(`{"pubKey":"pk%06d"}`, i)); err != nil { - tb.Fatalf("test DB insert transmission %d: %v", i, err) - } - nObs := (i % 5) + 1 // 1–5 observations per transmission - for j := 0; j < nObs; j++ { - snr := -5.0 + float64(j)*2.5 - rssi := -90.0 + float64(j)*5.0 - if _, err := obsStmt.Exec(obsID, i, observers[j], obsNames[j], "RX", snr, rssi, 5-j, `["aa","bb"]`, ts); err != nil { - tb.Fatalf("test DB insert observation %d: %v", obsID, err) - } - obsID++ + totalObs += i%5 + 1 + } + txID, j := 1, 0 + copyTestRows(tb, conn, "observations", []string{"id", "transmission_id", "observer_id", "observer_name", "direction", "snr", "rssi", "score", "path_json", "timestamp"}, totalObs, func(n int) []any { + row := []any{n + 1, txID, observers[j], names[j], "RX", -5.0 + float64(j)*2.5, -90.0 + float64(j)*5.0, 5 - j, `["aa","bb"]`, base.Add(time.Duration(txID) * time.Minute).Format(time.RFC3339)} + j++ + if j > txID%5 { + txID++ + j = 0 } - } - if _, err := conn.Exec("COMMIT"); err != nil { - tb.Fatalf("test DB COMMIT: %v", err) - } + return row + }) } diff --git a/cmd/server/bridge_handle_nodes_test.go b/cmd/server/bridge_handle_nodes_test.go index b88b7b1b6..b08376f45 100644 --- a/cmd/server/bridge_handle_nodes_test.go +++ b/cmd/server/bridge_handle_nodes_test.go @@ -34,7 +34,7 @@ func TestBridgeScore_HandleNodesSurface(t *testing.T) { for _, pk := range pks { if _, err := db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, ?, 'repeater', 37.5, -122.0, ?, ?, 10)`, + VALUES ($1, $2, 'repeater', 37.5, -122.0, $3, $4, 10)`, pk, "node-"+pk[:4], recent, recent); err != nil { t.Fatal(err) } diff --git a/cmd/server/bulk_health_region_test.go b/cmd/server/bulk_health_region_test.go index 70c5bb1fa..a073adaf6 100644 --- a/cmd/server/bulk_health_region_test.go +++ b/cmd/server/bulk_health_region_test.go @@ -28,27 +28,27 @@ func TestGetBulkHealthRegionKeepsNodesHeardThere(t *testing.T) { t.Fatalf("%v\n%s", err, q) } } - exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) VALUES ('obsE', 'Edinburgh', 'EDI', ?, ?, 1)`, ts, ts) - exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) VALUES ('obsG', 'Glasgow', 'GLA', ?, ?, 1)`, ts, ts) + exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) VALUES ('obsE', 'Edinburgh', 'EDI', $1, $2, 1)`, ts, ts) + exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) VALUES ('obsG', 'Glasgow', 'GLA', $1, $2, 1)`, ts, ts) for _, pk := range []string{a, b, c} { - exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) VALUES (?, ?, 'repeater', 56.0, -3.5, ?, ?, 1)`, pk, "R-"+pk[:4], ts, ts) + exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) VALUES ($1, $2, 'repeater', 56.0, -3.5, $3, $4, 1)`, pk, "R-"+pk[:4], ts, ts) } advert := func(id int, pk string) { - exec(`INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, decoded_json, from_pubkey) VALUES (?, 'AA00', ?, ?, 1, 4, ?, ?)`, + exec(`INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, decoded_json, from_pubkey) VALUES ($1, 'AA00', $2, $3, 1, 4, $4, $5)`, id, "h"+pk, ts, `{"pubKey":"`+pk+`","name":"R","type":"ADVERT"}`, pk) } advert(10, a) advert(20, b) advert(30, c) // A channel message heard in EDI that c relayed. - exec(`INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, decoded_json) VALUES (40, 'CC00', 'hchan', ?, 1, 5, '{"type":"CHAN","text":"x"}')`, ts) + exec(`INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, decoded_json) VALUES (40, 'CC00', 'hchan', $1, 1, 5, '{"type":"CHAN","text":"x"}')`, ts) // Observation IDs 1-4 never equal a transmission ID (10-40). observer_idx // is the observers rowid: 1 = obsE (EDI), 2 = obsG (GLA). - exec(`INSERT INTO observations (id, transmission_id, observer_idx, snr, rssi, path_json, timestamp) VALUES (1, 10, 1, 5, -90, '[]', ?)`, epoch) - exec(`INSERT INTO observations (id, transmission_id, observer_idx, snr, rssi, path_json, timestamp) VALUES (2, 20, 2, 5, -90, '[]', ?)`, epoch) - exec(`INSERT INTO observations (id, transmission_id, observer_idx, snr, rssi, path_json, timestamp) VALUES (3, 30, 2, 5, -90, '[]', ?)`, epoch) - exec(`INSERT INTO observations (id, transmission_id, observer_idx, snr, rssi, path_json, timestamp, resolved_path) VALUES (4, 40, 1, 5, -90, '["cc"]', ?, ?)`, epoch, `["`+c+`"]`) + exec(`INSERT INTO observations (id, transmission_id, observer_idx, snr, rssi, path_json, timestamp) VALUES (1, 10, 1, 5, -90, '[]', $1)`, epoch) + exec(`INSERT INTO observations (id, transmission_id, observer_idx, snr, rssi, path_json, timestamp) VALUES (2, 20, 2, 5, -90, '[]', $1)`, epoch) + exec(`INSERT INTO observations (id, transmission_id, observer_idx, snr, rssi, path_json, timestamp) VALUES (3, 30, 2, 5, -90, '[]', $1)`, epoch) + exec(`INSERT INTO observations (id, transmission_id, observer_idx, snr, rssi, path_json, timestamp, resolved_path) VALUES (4, 40, 1, 5, -90, '["cc"]', $1, $2)`, epoch, `["`+c+`"]`) store := NewPacketStore(db, nil) if err := store.Load(); err != nil { diff --git a/cmd/server/channel_message_hash_size_test.go b/cmd/server/channel_message_hash_size_test.go index 6a53a345a..c4226915a 100644 --- a/cmd/server/channel_message_hash_size_test.go +++ b/cmd/server/channel_message_hash_size_test.go @@ -44,18 +44,19 @@ func seedChannelHashSize(t *testing.T, db *DB) *DB { } for i, r := range rows { ts := now.Add(time.Duration(i-len(rows)) * time.Minute) - res, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash) - VALUES (?, ?, ?, ?, 5, '{"type":"CHAN","channel":"#hashsize","text":"Alice: msg"}', '#hashsize')`, - r.rawHex, r.hash, ts.Format(time.RFC3339), r.routeType) + var txID int64 + err := db.conn.QueryRow(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash) + VALUES ($1, $2, $3, $4, 5, '{"type":"CHAN","channel":"#hashsize","text":"Alice: msg"}', '#hashsize') RETURNING id`, + r.rawHex, r.hash, ts.Format(time.RFC3339), r.routeType).Scan(&txID) if err != nil { t.Fatalf("insert tx %s: %v", r.hash, err) } - txID, _ := res.LastInsertId() + obsSQL := `INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (?, 1, 9.5, -90, '[]', ?)` + VALUES ($1, 1, 9.5, -90, '[]', $2)` if !db.isV3 { obsSQL = `INSERT INTO observations (transmission_id, observer_id, observer_name, snr, rssi, path_json, timestamp) - VALUES (?, 'obs1', 'Observer One', 9.5, -90, '[]', ?)` + VALUES ($1, 'obs1', 'Observer One', 9.5, -90, '[]', $2)` } if _, err := db.conn.Exec(obsSQL, txID, ts.Unix()); err != nil { t.Fatalf("insert obs %s: %v", r.hash, err) diff --git a/cmd/server/channel_message_scope_name_test.go b/cmd/server/channel_message_scope_name_test.go index 3d2af4719..ab834ee5e 100644 --- a/cmd/server/channel_message_scope_name_test.go +++ b/cmd/server/channel_message_scope_name_test.go @@ -54,16 +54,17 @@ func setupChannelScopeDB(t *testing.T) *DB { } for i, r := range rows { ts := now.Add(time.Duration(i-3) * time.Minute) - res, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash, scope_name) - VALUES ('AABB', ?, ?, ?, 5, ?, '#scopetest', ?)`, + var txID int64 + err := db.conn.QueryRow(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash, scope_name) + VALUES ('AABB', $1, $2, $3, 5, $4, '#scopetest', $5) RETURNING id`, r.hash, ts.Format(time.RFC3339), r.routeType, - `{"type":"CHAN","channel":"#scopetest","text":"`+r.text+`"}`, r.scope) + `{"type":"CHAN","channel":"#scopetest","text":"`+r.text+`"}`, r.scope).Scan(&txID) if err != nil { t.Fatalf("insert tx %s: %v", r.hash, err) } - txID, _ := res.LastInsertId() + if _, err := db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (?, 1, 9.5, -90, '[]', ?)`, txID, ts.Unix()); err != nil { + VALUES ($1, 1, 9.5, -90, '[]', $2)`, txID, ts.Unix()); err != nil { t.Fatalf("insert obs %s: %v", r.hash, err) } } @@ -229,15 +230,16 @@ func TestBroadcastMapsCarryScopeName(t *testing.T) { t.Run("IngestNewFromDB", func(t *testing.T) { txMax := store.MaxTransmissionID() now := time.Now().UTC() - res, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash, scope_name) - VALUES ('AABB', 'cccccccccccccc04', ?, 0, 5, '{"type":"CHAN","channel":"#scopetest","text":"Dave: live"}', '#scopetest', '#belgium')`, - now.Format(time.RFC3339)) + var txID int64 + err := db.conn.QueryRow(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash, scope_name) + VALUES ('AABB', 'cccccccccccccc04', $1, 0, 5, '{"type":"CHAN","channel":"#scopetest","text":"Dave: live"}', '#scopetest', '#belgium') RETURNING id`, + now.Format(time.RFC3339)).Scan(&txID) if err != nil { t.Fatalf("insert tx: %v", err) } - txID, _ := res.LastInsertId() + if _, err := db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (?, 1, 7.0, -95, '[]', ?)`, txID, now.Unix()); err != nil { + VALUES ($1, 1, 7.0, -95, '[]', $2)`, txID, now.Unix()); err != nil { t.Fatalf("insert obs: %v", err) } maps, _ := store.IngestNewFromDB(txMax, 100) @@ -247,11 +249,11 @@ func TestBroadcastMapsCarryScopeName(t *testing.T) { t.Run("IngestNewObservations", func(t *testing.T) { obsMax := db.GetMaxObservationID() var txID int - if err := db.conn.QueryRow(`SELECT id FROM transmissions WHERE hash = ?`, chScopeHashUnknown).Scan(&txID); err != nil { + if err := db.conn.QueryRow(`SELECT id FROM transmissions WHERE hash = $1`, chScopeHashUnknown).Scan(&txID); err != nil { t.Fatalf("lookup tx: %v", err) } if _, err := db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (?, 1, 3.0, -110, '["aa"]', ?)`, txID, time.Now().Unix()); err != nil { + VALUES ($1, 1, 3.0, -110, '["aa"]', $2)`, txID, time.Now().Unix()); err != nil { t.Fatalf("insert obs: %v", err) } maps := store.IngestNewObservations(obsMax, 100) diff --git a/cmd/server/channels_message_order_1366_test.go b/cmd/server/channels_message_order_1366_test.go index 0fde9bfbb..5f307443b 100644 --- a/cmd/server/channels_message_order_1366_test.go +++ b/cmd/server/channels_message_order_1366_test.go @@ -34,20 +34,20 @@ func TestChannelMessages_TimestampUsesLatestSeen(t *testing.T) { _ = laterEpoch db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obsA', 'ObsA', 'SJC', ?, '2026-01-01T00:00:00Z', 10)`, firstSeen) + VALUES ('obsA', 'ObsA', 'SJC', $1, '2026-01-01T00:00:00Z', 10)`, firstSeen) db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obsB', 'ObsB', 'LAX', ?, '2026-01-01T00:00:00Z', 10)`, firstSeen) + VALUES ('obsB', 'ObsB', 'LAX', $1, '2026-01-01T00:00:00Z', 10)`, firstSeen) // One transmission with two observations: T0 (7h ago) and T1 (5m ago). db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash) - VALUES ('AA01', 'hash_repeated_msg', ?, 1, 5, + VALUES ('AA01', 'hash_repeated_msg', $1, 1, 5, '{"type":"CHAN","channel":"#test","text":"Heartbeat: ping","sender":"Heartbeat","sender_timestamp":`+ strconv.FormatInt(firstSeenEpoch, 10)+`}', '#test')`, firstSeen) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 10.0, -90, '["aa"]', ?)`, firstSeenEpoch) + VALUES (1, 1, 10.0, -90, '["aa"]', $1)`, firstSeenEpoch) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 2, 11.0, -88, '["bb"]', ?)`, laterEpoch) + VALUES (1, 2, 11.0, -88, '["bb"]', $1)`, laterEpoch) store := NewPacketStore(db, nil) store.Load() @@ -102,14 +102,14 @@ func TestChannelMessages_TimestampNotSenderTimestamp(t *testing.T) { badSenderTs := int64(946684800) // 2000-01-01 UTC db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obsX', 'ObsX', 'SJC', ?, '2026-01-01T00:00:00Z', 1)`, firstSeen) + VALUES ('obsX', 'ObsX', 'SJC', $1, '2026-01-01T00:00:00Z', 1)`, firstSeen) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash) - VALUES ('BB01', 'hash_bad_clock', ?, 1, 5, + VALUES ('BB01', 'hash_bad_clock', $1, 1, 5, '{"type":"CHAN","channel":"#bad","text":"Alice: ping","sender":"Alice","sender_timestamp":`+ strconv.FormatInt(badSenderTs, 10)+`}', '#bad')`, firstSeen) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 10.0, -90, '["aa"]', ?)`, firstSeenEpoch) + VALUES (1, 1, 10.0, -90, '["aa"]', $1)`, firstSeenEpoch) store := NewPacketStore(db, nil) store.Load() @@ -143,13 +143,13 @@ func TestChannelMessages_TimestampIsUTCZ(t *testing.T) { ep := now.Add(-30 * time.Minute).Unix() db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obsZ', 'ObsZ', 'SJC', ?, '2026-01-01T00:00:00Z', 1)`, fs) + VALUES ('obsZ', 'ObsZ', 'SJC', $1, '2026-01-01T00:00:00Z', 1)`, fs) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash) - VALUES ('ZZ01', 'hash_zone_check', ?, 1, 5, + VALUES ('ZZ01', 'hash_zone_check', $1, 1, 5, '{"type":"CHAN","channel":"#zone","text":"Carol: ping","sender":"Carol"}', '#zone')`, fs) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 11.0, -89, '["zz"]', ?)`, ep) + VALUES (1, 1, 11.0, -89, '["zz"]', $1)`, ep) store := NewPacketStore(db, nil) store.Load() @@ -197,30 +197,30 @@ func TestChannelMessages_OrderedByLatestSeen_InMemory(t *testing.T) { tNewestStr := tNewest.Format(time.RFC3339) db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obsO', 'ObsO', 'SJC', ?, '2026-01-01T00:00:00Z', 10)`, tOldStr) + VALUES ('obsO', 'ObsO', 'SJC', $1, '2026-01-01T00:00:00Z', 10)`, tOldStr) // tx-A: FirstSeen 24h ago, LatestSeen NOW (T-1m). Old insertion order. db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash) - VALUES ('AAAA', 'order_hash_a', ?, 1, 5, + VALUES ('AAAA', 'order_hash_a', $1, 1, 5, '{"type":"CHAN","channel":"#ord","text":"Alpha: hb","sender":"Alpha"}', '#ord')`, tOldStr) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 10.0, -90, '["aa"]', ?)`, tOld.Unix()) + VALUES (1, 1, 10.0, -90, '["aa"]', $1)`, tOld.Unix()) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 11.0, -88, '["aa"]', ?)`, tFresh.Unix()) + VALUES (1, 1, 11.0, -88, '["aa"]', $1)`, tFresh.Unix()) // tx-B: FirstSeen 1h ago, LatestSeen 1h ago. OLDEST. db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash) - VALUES ('BBBB', 'order_hash_b', ?, 1, 5, + VALUES ('BBBB', 'order_hash_b', $1, 1, 5, '{"type":"CHAN","channel":"#ord","text":"Bravo: msg","sender":"Bravo"}', '#ord')`, tMidStr) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (2, 1, 9.0, -91, '["bb"]', ?)`, tMid.Unix()) + VALUES (2, 1, 9.0, -91, '["bb"]', $1)`, tMid.Unix()) // tx-C: FirstSeen 30m ago, LatestSeen 30m ago. Middle. db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash) - VALUES ('CCCC', 'order_hash_c', ?, 1, 5, + VALUES ('CCCC', 'order_hash_c', $1, 1, 5, '{"type":"CHAN","channel":"#ord","text":"Charlie: msg","sender":"Charlie"}', '#ord')`, tNewestStr) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (3, 1, 9.0, -91, '["cc"]', ?)`, tNewest.Unix()) + VALUES (3, 1, 9.0, -91, '["cc"]', $1)`, tNewest.Unix()) store := NewPacketStore(db, nil) store.Load() @@ -276,34 +276,34 @@ func TestChannelMessages_OrderedByLatestSeen_DB(t *testing.T) { tNewestStr := tNewest.Format(time.RFC3339) db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obsD', 'ObsD', 'SJC', ?, '2026-01-01T00:00:00Z', 10)`, tOldStr) + VALUES ('obsD', 'ObsD', 'SJC', $1, '2026-01-01T00:00:00Z', 10)`, tOldStr) // tx-A: FirstSeen 24h ago, observations at T-24h and T-1m (LatestSeen // = T-1m, the FRESHEST). Despite the freshest LatestSeen, a // FirstSeen-DESC selection would push it OFF a small page. db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash) - VALUES ('AADB', 'order_db_hash_a', ?, 1, 5, + VALUES ('AADB', 'order_db_hash_a', $1, 1, 5, '{"type":"CHAN","channel":"#ordb","text":"Alpha: hb","sender":"Alpha"}', '#ordb')`, tOldStr) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 10.0, -90, '["aa"]', ?)`, tOld.Unix()) + VALUES (1, 1, 10.0, -90, '["aa"]', $1)`, tOld.Unix()) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 11.0, -88, '["aa"]', ?)`, tFresh.Unix()) + VALUES (1, 1, 11.0, -88, '["aa"]', $1)`, tFresh.Unix()) // tx-B: FirstSeen 1h ago, LatestSeen 1h ago. OLDEST LatestSeen. db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash) - VALUES ('BBDB', 'order_db_hash_b', ?, 1, 5, + VALUES ('BBDB', 'order_db_hash_b', $1, 1, 5, '{"type":"CHAN","channel":"#ordb","text":"Bravo: msg","sender":"Bravo"}', '#ordb')`, tMidStr) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (2, 1, 9.0, -91, '["bb"]', ?)`, tMid.Unix()) + VALUES (2, 1, 9.0, -91, '["bb"]', $1)`, tMid.Unix()) // tx-C: FirstSeen 30m ago, LatestSeen 30m ago. Middle LatestSeen. // With FirstSeen-DESC selection + limit=2, page = [tx-C, tx-B] and // tx-A is EXCLUDED — that's the selection bug fix #2 gates. db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash) - VALUES ('CCDB', 'order_db_hash_c', ?, 1, 5, + VALUES ('CCDB', 'order_db_hash_c', $1, 1, 5, '{"type":"CHAN","channel":"#ordb","text":"Charlie: msg","sender":"Charlie"}', '#ordb')`, tNewestStr) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (3, 1, 9.0, -91, '["cc"]', ?)`, tNewest.Unix()) + VALUES (3, 1, 9.0, -91, '["cc"]', $1)`, tNewest.Unix()) msgs, total, err := db.GetChannelMessages("#ordb", 2, 0) if err != nil { diff --git a/cmd/server/channels_no_unknown_bucket_1373_test.go b/cmd/server/channels_no_unknown_bucket_1373_test.go index 3f709f399..d917e4e4b 100644 --- a/cmd/server/channels_no_unknown_bucket_1373_test.go +++ b/cmd/server/channels_no_unknown_bucket_1373_test.go @@ -67,7 +67,7 @@ func TestGetChannels_DB_NoUnknownBucket_1373(t *testing.T) { for i := 0; i < 5; i++ { _, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash) - VALUES (?, ?, '2026-05-25T12:00:00Z', 1, 5, + VALUES ($1, $2, '2026-05-25T12:00:00Z', 1, 5, '{"type":"CHAN","channel":"","text":"","sender":""}', NULL)`, "AA", sqlHashFor(i)) if err != nil { @@ -79,7 +79,7 @@ func TestGetChannels_DB_NoUnknownBucket_1373(t *testing.T) { for i := 0; i < 3; i++ { _, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash) - VALUES (?, ?, '2026-05-25T12:00:00Z', 1, 5, + VALUES ($1, $2, '2026-05-25T12:00:00Z', 1, 5, '{"type":"CHAN","channel":"#real","text":"Alice: hi","sender":"Alice"}', '#real')`, "BB", sqlHashFor(100+i)) if err != nil { diff --git a/cmd/server/chunked_load.go b/cmd/server/chunked_load.go index db9313a54..8ab7d648f 100644 --- a/cmd/server/chunked_load.go +++ b/cmd/server/chunked_load.go @@ -3,9 +3,10 @@ package main // Chunked startup load + early HTTP readiness for issue #1009. // // Design: -// * LoadChunked paginates transmissions in id-ordered chunks of +// * LoadChunked paginates transmissions in bounded chunks of // `chunkSize` (default 10000 via Config.DBLoadChunkSize). After the // first chunk is merged into the store, FirstChunkReady is closed. +// Memory-capped stores prioritize newest rows; unlimited stores use IDs. // main.go binds the HTTP listener on that signal and serves // partial data while remaining chunks stream in the background. // * RunStartupLoad is the orchestrator: it runs LoadChunked @@ -22,15 +23,13 @@ package main // logging / tests. // // Concurrency: each chunk acquires s.mu.Lock() ONLY while merging the -// chunk's rows into store-shared maps. SQLite reads run lock-free so +// chunk's rows into store-shared maps. Database reads run without the store lock so // HTTP handlers (which take s.mu.RLock) stay responsive. import ( - "database/sql" "fmt" "log" "net/http" - "sort" "strings" "sync" "sync/atomic" @@ -102,7 +101,7 @@ func (s *PacketStore) chunkedLoadInit() { // StartupLoadDone returns a channel closed once RunStartupLoad has // returned: LoadChunked AND the background fill loader are finished, -// whether they succeeded or not. Nothing more is loaded from SQLite +// whether they succeeded or not. Nothing more is loaded from the database // after it closes. LoadComplete() is not a substitute: it flips at the // end of the hot window, before the background fill starts. func (s *PacketStore) StartupLoadDone() <-chan struct{} { @@ -187,8 +186,8 @@ func (s *PacketStore) fireChunkCallbacks(rowsThisChunk, totalRows int) { // (no background work was needed). Coverage is computed against // row count so /api/perf does not report ratio=0.0 (dij #2). // - hotStartupHours > 0 success: terminal state is whatever -// loadBackgroundChunks set (done=true on full coverage, -// failed=true on partial / chunk errors — see #1690). +// loadBackgroundChunks set (done=true on full coverage or the configured +// memory limit; failed=true on unexpected partial coverage/chunk errors). // // Issue #1809 root cause: previously main.go spawned loadBackgroundChunks // at FirstChunkReady while LoadChunked was still merging the remainder @@ -200,6 +199,7 @@ func (s *PacketStore) fireChunkCallbacks(rowsThisChunk, totalRows int) { // bg loader starts. func (s *PacketStore) RunStartupLoad(chunkSize int) error { defer s.signalStartupLoadDone() + defer s.logStartupAdmission() // Clear any stale error from a previous invocation (single-call // invariant — see godoc above). Production never re-enters but // test fixtures may construct fresh stores that share no state; @@ -265,8 +265,43 @@ func (s *PacketStore) RunStartupLoad(chunkSize int) error { return nil } -// LoadChunked streams transmissions + observations from SQLite into -// the in-memory store in id-ordered chunks of `chunkSize` rows. Pass +// logStartupAdmission reports the terminal cache state after both startup phases. +// The first_seen endpoints describe retained rows, not complete time coverage: +// a long-lived hash may be older than coverageFrom. All inputs are existing +// counters or sorted-slice endpoints; this adds no database query or store scan. +func (s *PacketStore) logStartupAdmission() { + s.mu.RLock() + txCount, obsCount, trackedBytes := len(s.packets), s.totalObs, s.trackedBytes + budgetMB, limited := s.maxMemoryMB, s.startupMemoryLimited + coverageFrom := "" + if floor, _ := s.coverageStart(time.Now().UTC()); !floor.IsZero() { + coverageFrom = floor.Format(time.RFC3339) + } + first, last := "", "" + if txCount > 0 { + first, last = s.packets[0].FirstSeen, s.packets[txCount-1].FirstSeen + } + s.mu.RUnlock() + reason := "window_complete" + if s.backgroundLoadFailed.Load() { + reason = "error" + } else if limited { + reason = "memory_budget" + } + // This is the former count-admission policy, not measured memory usage. + // It assumed ten observations per transmission and always admitted at + // least 1,000 transmissions, even when their real fanout exceeded budget. + formerEstimate := "unlimited" + if budgetMB > 0 { + typicalBytes := max(int64(1000), estimateStoreTxBytesTypical(10)) + formerEstimate = fmt.Sprint(max(int64(1000), int64(budgetMB)*1048576/typicalBytes)) + } + log.Printf("[store] startup admission: budgetMB=%d transmissions=%d observations=%d trackedMB=%.2f firstSeenRange=[%q,%q] coverageFrom=%q stop=%s formerPolicyTransmissionEstimate=%s (assumedObservations=10 minimumTransmissions=1000)", + budgetMB, txCount, obsCount, float64(trackedBytes)/1048576, first, last, coverageFrom, reason, formerEstimate) +} + +// LoadChunked streams transmissions + observations from the selected backend into +// the in-memory store in bounded chunks of `chunkSize` rows. Pass // 0 to use the default (10000). // // After the first chunk is merged, FirstChunkReady is closed and the @@ -327,7 +362,7 @@ func (s *PacketStore) LoadChunked(chunkSize int) error { // the prior `t2.first_seen >= cutoff` query loaded only hashes // first-inserted within the window (0.3% of DB on prod). // - // Test/legacy DBs without the column (PRAGMA-detected as + // Test/legacy DBs without the column (schema-detected as // hasLastSeen=false) fall back to the legacy first_seen axis to // keep existing fixtures green. Production goes through // dbschema.AssertReady which fail-fasts when the column is @@ -353,155 +388,49 @@ func (s *PacketStore) LoadChunked(chunkSize int) error { totalInDB = -1 } - // Memory cap honoured by clamping the maximum cursor walk. - var maxPackets int64 + // Polling starts after the first chunk. Capture the complete database's + // high-water marks before loading a subset, so older omitted rows are not + // mistaken for new ingest and replayed into the cache immediately. if s.maxMemoryMB > 0 { - avgBytes := int64(1000) - if sample := estimateStoreTxBytesTypical(10); sample > avgBytes { - avgBytes = sample - } - maxPackets = (int64(s.maxMemoryMB) * 1048576) / avgBytes - if maxPackets < 1000 { - maxPackets = 1000 + var maxTxID, maxObsID int + if err := s.db.conn.QueryRow(`SELECT + (SELECT COALESCE(MAX(id), 0) FROM transmissions), + (SELECT COALESCE(MAX(id), 0) FROM observations)`).Scan(&maxTxID, &maxObsID); err != nil { + return fmt.Errorf("load watermarks: %w", err) } + s.mu.Lock() + s.maxTxID, s.maxObsID = maxTxID, maxObsID + s.mu.Unlock() + } + var from time.Time + if hotCutoffStr != "" { + from, _ = time.Parse(time.RFC3339, hotCutoffStr) } - chunkIdx := 0 - totalLoaded := 0 - capped := false // the memory cap stopped the walk with rows left (#748) - // Start the id cursor BELOW the minimum possible row id so the - // first chunk's `t2.id > cursorID` predicate includes id=0. The - // e2e fixture seed for issue #1486 inserts the grouped-packet row - // with id=0 (so it sorts LAST in the default packets view via - // `ORDER BY id DESC` / oldest first_seen). Seeding the cursor at - // 0 silently excluded that row, leaving the page with no - // tr[data-hash] and timing out the playwright wait. Legacy Load() - // had no id cursor and loaded id=0 unconditionally — we restore - // that semantic by starting one below SQLite's minimum rowid (-1). - var cursorID int64 = -1 - - // Relay-hop fallback inputs, fetched ONCE before the chunk-query loop. - // getCachedNodesAndPM issues its own DB query, so calling it while a - // chunk cursor is open would deadlock on a single-connection SQLite - // pool. resolved_path is never persisted post-#1287, so scanAndMergeChunk - // re-resolves relay hops from path_json using these snapshots. - // PR #1643 R1 munger #1: cold load uses unique_prefix-only gate, so - // the neighbor graph is no longer consulted here (affinity-tier - // resolution against ≤168h-old observations would silently mis-attribute). - s.mu.RLock() - _, relayPM := s.getCachedNodesAndPM() - s.mu.RUnlock() - var coldLoadAmbiguousHopsSkipped int - - for { - conds := append([]string{}, loadConditions...) - conds = append(conds, fmt.Sprintf("t2.id > %d", cursorID)) - whereClause := "WHERE " + strings.Join(conds, " AND ") - - rpCol := "" - if s.db.hasResolvedPath { - rpCol = ", o.resolved_path" - } - obsRawHexCol := "" - if s.db.hasObsRawHex { - obsRawHexCol = ", o.raw_hex" - } - // #1751: scope_name is on the transmission row, so appending it as the - // last selected column is safe regardless of the observation fan-out. - scopeNameCol := "" - if s.db.hasScopeName { - scopeNameCol = ", t.scope_name" - } - - var chunkSQL string - if s.db.isV3 { - chunkSQL = `SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type, - t.payload_type, t.payload_version, t.decoded_json, - o.id, obs.id, obs.name, COALESCE(obs.iata, ''), o.direction, - o.snr, o.rssi, o.score, o.path_json, strftime('%Y-%m-%dT%H:%M:%fZ', o.timestamp, 'unixepoch')` + obsRawHexCol + rpCol + scopeNameCol + ` - FROM (SELECT * FROM transmissions t2 ` + whereClause + ` ORDER BY t2.id ASC LIMIT ` + fmt.Sprintf("%d", chunkSize) + `) AS t - LEFT JOIN observations o ON o.transmission_id = t.id - LEFT JOIN observers obs ON obs.rowid = o.observer_idx - ORDER BY t.id ASC, o.timestamp DESC` - } else { - chunkSQL = `SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type, - t.payload_type, t.payload_version, t.decoded_json, - o.id, o.observer_id, o.observer_name, COALESCE(obs.iata, ''), o.direction, - o.snr, o.rssi, o.score, o.path_json, o.timestamp` + obsRawHexCol + rpCol + scopeNameCol + ` - FROM (SELECT * FROM transmissions t2 ` + whereClause + ` ORDER BY t2.id ASC LIMIT ` + fmt.Sprintf("%d", chunkSize) + `) AS t - LEFT JOIN observations o ON o.transmission_id = t.id - LEFT JOIN observers obs ON obs.id = o.observer_id - ORDER BY t.id ASC, o.timestamp DESC` - } - - // Acquire before opening the cursor: a feed waiting for the same - // connection must never hold this gate while we hold its cursor. - s.advertEvidenceMu.Lock() - rows, err := s.db.conn.Query(chunkSQL) - if err != nil { - s.advertEvidenceMu.Unlock() - return fmt.Errorf("chunk %d: query: %w", chunkIdx, err) - } - - chunkTxCount, lastID, err := s.scanAndMergeChunk(rows, relayPM, &coldLoadAmbiguousHopsSkipped) - rows.Close() - s.advertEvidenceMu.Unlock() - if err != nil { - return fmt.Errorf("chunk %d: scan: %w", chunkIdx, err) - } - - if chunkTxCount == 0 { - break - } - - cursorID = lastID - totalLoaded += chunkTxCount + totalLoaded, err := s.loadPacketRange(from, time.Time{}, chunkSize, func(count, total int) { chunkIdx++ - s.loadProgressRows.Store(int64(totalLoaded)) + s.loadProgressRows.Store(int64(total)) s.signalFirstChunk() - s.fireChunkCallbacks(chunkTxCount, totalLoaded) - - if maxPackets > 0 && int64(totalLoaded) >= maxPackets { - capped = totalInDB < 0 || totalLoaded < totalInDB - break - } - if chunkTxCount < chunkSize { - break - } + s.fireChunkCallbacks(count, total) + }) + if err != nil { + return fmt.Errorf("chunk %d: %w", chunkIdx, err) } - // Post-load: pick best observation, build indexes — same shape as - // legacy Load(). + // Each chunk publishes complete packets and indexes atomically in + // first_seen order. Build the derived analytics indexes once at the end. s.mu.Lock() - for _, tx := range s.packets { - pickBestObservation(tx) - s.trackedBytes += rechargeTx(tx) - s.indexByNode(tx) - } - // Restore the "s.packets sorted oldest-first by FirstSeen" invariant - // that legacy Load() got for free from "ORDER BY t.first_seen ASC". - // LoadChunked walks chunks in id-ASC order so the slice ends up - // id-ordered, which only equals first_seen-ordered when ids and - // timestamps are correlated. After tools/freshen-fixture.sh (or any - // real-world out-of-order ingest) they're not, leaving - // s.packets[0].FirstSeen pointing at the newest row — which then - // poisons oldestLoaded below and routes legitimate in-memory queries - // to the SQL fallback. GetTimestamps (store.go) and QueryPackets - // both rely on this invariant. See PR #1596 / mobile e2e regression. - sort.SliceStable(s.packets, func(i, j int) bool { - return s.packets[i].FirstSeen < s.packets[j].FirstSeen - }) s.buildPathHopIndex() s.buildDistanceIndex() - if s.hotStartupHours > 0 { - s.oldestLoaded = hotCutoffStr - } else if len(s.packets) > 0 { - s.oldestLoaded = s.packets[0].FirstSeen - } - if capped { - s.loadCapEdge = time.Now().UTC().Format(time.RFC3339) + if !s.startupMemoryLimited { + if s.hotStartupHours > 0 { + s.oldestLoaded = hotCutoffStr + } else if len(s.packets) > 0 { + s.oldestLoaded = s.packets[0].FirstSeen + } } s.loaded = true + totalObs := s.totalObs s.mu.Unlock() // #1009 / PR #1596: flip the subpath + pathHop ready flags now that @@ -514,11 +443,7 @@ func (s *PacketStore) LoadChunked(chunkSize int) error { elapsed := time.Since(t0) log.Printf("[store] LoadChunked: %d transmissions (%d observations) across %d chunk(s) in %v (chunkSize=%d, DB total=%d)", - totalLoaded, s.totalObs, chunkIdx, elapsed, chunkSize, totalInDB) - if coldLoadAmbiguousHopsSkipped > 0 { - log.Printf("[store] LoadChunked: skipped %d ambiguous-prefix relay hops (unique_prefix gate, PR #1643 R1)", - coldLoadAmbiguousHopsSkipped) - } + totalLoaded, totalObs, chunkIdx, elapsed, chunkSize, totalInDB) s.loadMultibyteCapFromDB() // Mark complete on the success path only — see the function-level // defer above for why this is NOT in a deferred call. Probes that @@ -528,165 +453,6 @@ func (s *PacketStore) LoadChunked(chunkSize int) error { return nil } -// scanAndMergeChunk consumes one chunk's rows under s.mu.Lock and -// returns the number of distinct transmissions seen + the max -// transmission id (cursor for the next chunk). -func (s *PacketStore) scanAndMergeChunk(rows *sql.Rows, relayPM *prefixMap, coldLoadAmbiguousHopsSkipped *int) (int, int64, error) { - s.mu.Lock() - defer s.mu.Unlock() - - hopsSeen := make(map[string]bool) - seenTxIDs := make(map[int]bool) - var maxID int64 - - // Scan overwrites every target on each row, so one set (and one - // argument slice) serves the whole loop instead of escaping per row. - var txID int - var rawHex, hash, firstSeen, decodedJSON sql.NullString - var routeType, payloadType, payloadVersion sql.NullInt64 - var obsID sql.NullInt64 - var observerID, observerName, observerIATA, direction, pathJSON, obsTimestamp sql.NullString - var snr, rssi sql.NullFloat64 - var score sql.NullInt64 - var obsRawHex sql.NullString - var resolvedPathStr sql.NullString - var scopeName sql.NullString - - scanArgs := []interface{}{&txID, &rawHex, &hash, &firstSeen, &routeType, &payloadType, - &payloadVersion, &decodedJSON, - &obsID, &observerID, &observerName, &observerIATA, &direction, - &snr, &rssi, &score, &pathJSON, &obsTimestamp} - if s.db.hasObsRawHex { - scanArgs = append(scanArgs, &obsRawHex) - } - if s.db.hasResolvedPath { - scanArgs = append(scanArgs, &resolvedPathStr) - } - if s.db.hasScopeName { - scanArgs = append(scanArgs, &scopeName) - } - - for rows.Next() { - if err := rows.Scan(scanArgs...); err != nil { - log.Printf("[store] LoadChunked scan error: %v", err) - continue - } - - if int64(txID) > maxID { - maxID = int64(txID) - } - seenTxIDs[txID] = true - - hashStr := nullStrVal(hash) - tx := s.byHash[hashStr] - if tx == nil { - tx = &StoreTx{ - ID: txID, - RawHex: nullStrVal(rawHex), - Hash: hashStr, - FirstSeen: nullStrVal(firstSeen), - LatestSeen: nullStrVal(firstSeen), - RouteType: nullIntPtr(routeType), - PayloadType: nullIntPtr(payloadType), - DecodedJSON: nullStrVal(decodedJSON), - ScopeName: nullStrPtr(scopeName), - } - s.byHash[hashStr] = tx - s.packets = append(s.packets, tx) - s.byTxID[txID] = tx - if txID > s.maxTxID { - s.maxTxID = txID - } - s.indexByNode(tx) - if tx.PayloadType != nil { - pt := *tx.PayloadType - s.byPayloadType[pt] = append(s.byPayloadType[pt], tx) - } - s.trackAdvertPubkey(tx) - s.trackedBytes += rechargeTx(tx) - } - - if obsID.Valid { - oid := int(obsID.Int64) - obsIDStr := nullStrVal(observerID) - obsPJ := nullStrVal(pathJSON) - - if tx.hasObservation(obsIDStr, obsPJ) { - continue - } - - obs := &StoreObs{ - ID: oid, - TransmissionID: txID, - ObserverID: internStr(obsIDStr), - ObserverName: internStr(nullStrVal(observerName)), - ObserverIATA: internStr(nullStrVal(observerIATA)), - Direction: internStr(nullStrVal(direction)), - SNR: nullFloatPtr(snr), - RSSI: nullFloatPtr(rssi), - Score: nullIntPtr(score), - PathJSON: obsPJ, - // Raw frames stay in SQLite; hash equality does not imply route - // equality. Only compact advert evidence is retained per tx. - // Packet-detail queries can read the original observation raw. - Timestamp: normalizeTimestamp(nullStrVal(obsTimestamp)), - } - - rpStr := nullStrVal(resolvedPathStr) - if rpStr != "" { - pks := resolvedPubkeysFromJSON(rpStr) - s.indexResolvedPathHops(tx, pks, hopsSeen) - } else if relayPM != nil && obsPJ != "" && obsPJ != "[]" { - // resolved_path is NULL on live (since #1287 relay data is - // persisted as neighbor_edges, not per-observation). Re-resolve - // relay-hop attribution from path_json so relay nodes keep their - // analytics history across a restart instead of rebuilding only - // from post-restart live traffic. relayPM is passed in from - // LoadChunked (fetched before any chunk cursor opened). - // byNode ONLY — see the Load() counterpart for why the - // resolved_path/path-hop indexes must NOT be populated here. - // PR #1643 R1 munger #1: unique_prefix-only gate. - rp := resolvePathForObsColdLoad(obsPJ, obsIDStr, tx, relayPM, coldLoadAmbiguousHopsSkipped) - for _, pk := range extractResolvedPubkeys(rp) { - s.addToByNode(tx, pk) - } - } - - tx.addObservation(obs) - if obs.Timestamp > tx.LatestSeen { - tx.LatestSeen = obs.Timestamp - } - - s.byObsID[oid] = obs - if oid > s.maxObsID { - s.maxObsID = oid - } - if obsIDStr != "" { - s.byObserver[obsIDStr] = append(s.byObserver[obsIDStr], obs) - } - s.totalObs++ - s.trackedBytes += estimateStoreObsBytes(obs) - } - } - if err := rows.Err(); err != nil { - return len(seenTxIDs), maxID, err - } - rows.Close() - txs := make([]*StoreTx, 0, len(seenTxIDs)) - for id := range seenTxIDs { - txs = append(txs, s.byTxID[id]) - } - ids := advertTxIDs(txs) - s.mu.Unlock() - masks, err := s.db.advertEvidenceForIDs(ids) - s.mu.Lock() - if err != nil { - return len(seenTxIDs), maxID, err - } - s.mergeAdvertEvidence(masks) - return len(seenTxIDs), maxID, nil -} - // loadStatusMiddleware sets X-CoreScope-Load-Status on every response. // While LoadChunked is in flight the header reports // "loading; progress="; after completion it reports "ready". diff --git a/cmd/server/chunked_load_id_zero_test.go b/cmd/server/chunked_load_id_zero_test.go index 7b1fb87e7..47b7c6ec2 100644 --- a/cmd/server/chunked_load_id_zero_test.go +++ b/cmd/server/chunked_load_id_zero_test.go @@ -20,38 +20,46 @@ package main // row into s.byHash. import ( - "database/sql" "fmt" - "path/filepath" "testing" "time" ) func createTestDBWithIDZero(tb testing.TB, dbPath string, extraTx int) { tb.Helper() - conn, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL") + conn, err := openFixtureSQL(dbPath) if err != nil { tb.Fatal(err) } defer conn.Close() stmts := []string{ - `CREATE TABLE IF NOT EXISTS transmissions ( - id INTEGER PRIMARY KEY, + testNativeSQL(`CREATE TABLE IF NOT EXISTS transmissions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT - )`, - `CREATE TABLE IF NOT EXISTS observations ( - id INTEGER PRIMARY KEY, + )`, `CREATE TABLE IF NOT EXISTS transmissions ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + raw_hex TEXT, hash TEXT, first_seen TEXT, + route_type INTEGER, payload_type INTEGER, + payload_version INTEGER, decoded_json TEXT + )`), + testNativeSQL(`CREATE TABLE IF NOT EXISTS observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, - direction TEXT, snr REAL, rssi REAL, score INTEGER, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT - )`, - `CREATE TABLE IF NOT EXISTS observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, + )`, `CREATE TABLE IF NOT EXISTS observations ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + transmission_id INTEGER, observer_id TEXT, observer_name TEXT, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, + path_json TEXT, timestamp TEXT, raw_hex TEXT + )`), + testNativeSQL(`CREATE TABLE IF NOT EXISTS observers (rowid INTEGER PRIMARY KEY AUTOINCREMENT, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, `CREATE TABLE IF NOT EXISTS observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`), `CREATE TABLE IF NOT EXISTS nodes ( - public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, - last_seen TEXT, first_seen TEXT, frequency REAL + public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, + last_seen TEXT, first_seen TEXT, frequency DOUBLE PRECISION )`, `CREATE TABLE IF NOT EXISTS schema_version (version INTEGER)`, `INSERT INTO schema_version (version) VALUES (1)`, @@ -63,8 +71,8 @@ func createTestDBWithIDZero(tb testing.TB, dbPath string, extraTx int) { } } - txStmt, _ := conn.Prepare("INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) VALUES (?, ?, ?, ?, ?, ?, ?, ?)") - obsStmt, _ := conn.Prepare("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") + txStmt, _ := conn.Prepare("INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)") + obsStmt, _ := conn.Prepare("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)") defer txStmt.Close() defer obsStmt.Close() @@ -89,11 +97,10 @@ func createTestDBWithIDZero(tb testing.TB, dbPath string, extraTx int) { // rows are silently dropped. This breaks the #1486 e2e fixture seed // which uses id=0 to sort the grouped row last in the default view. func TestLoadChunked_IncludesIDZero(t *testing.T) { - dir := t.TempDir() - dbPath := filepath.Join(dir, "idzero.db") + dbPath := postgresTestDSN(t) createTestDBWithIDZero(t, dbPath, 10) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatalf("OpenDB: %v", err) } diff --git a/cmd/server/chunked_load_oldest_test.go b/cmd/server/chunked_load_oldest_test.go index 8b3edce7f..23bd97437 100644 --- a/cmd/server/chunked_load_oldest_test.go +++ b/cmd/server/chunked_load_oldest_test.go @@ -19,9 +19,7 @@ package main // transmissions, not the FirstSeen of the first row in s.packets. import ( - "database/sql" "fmt" - "path/filepath" "testing" "time" ) @@ -31,29 +29,39 @@ import ( // This mirrors the freshen-fixture-shifted e2e DB exactly. func createTestDBReverseTime(tb testing.TB, dbPath string, numTx int) { tb.Helper() - conn, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL") + conn, err := openFixtureSQL(dbPath) if err != nil { tb.Fatal(err) } defer conn.Close() stmts := []string{ - `CREATE TABLE IF NOT EXISTS transmissions ( - id INTEGER PRIMARY KEY, + testNativeSQL(`CREATE TABLE IF NOT EXISTS transmissions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT - )`, - `CREATE TABLE IF NOT EXISTS observations ( - id INTEGER PRIMARY KEY, + )`, `CREATE TABLE IF NOT EXISTS transmissions ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + raw_hex TEXT, hash TEXT, first_seen TEXT, + route_type INTEGER, payload_type INTEGER, + payload_version INTEGER, decoded_json TEXT + )`), + testNativeSQL(`CREATE TABLE IF NOT EXISTS observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, - direction TEXT, snr REAL, rssi REAL, score INTEGER, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT - )`, - `CREATE TABLE IF NOT EXISTS observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, + )`, `CREATE TABLE IF NOT EXISTS observations ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + transmission_id INTEGER, observer_id TEXT, observer_name TEXT, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, + path_json TEXT, timestamp TEXT, raw_hex TEXT + )`), + testNativeSQL(`CREATE TABLE IF NOT EXISTS observers (rowid INTEGER PRIMARY KEY AUTOINCREMENT, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, `CREATE TABLE IF NOT EXISTS observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`), `CREATE TABLE IF NOT EXISTS nodes ( - public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, - last_seen TEXT, first_seen TEXT, frequency REAL + public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, + last_seen TEXT, first_seen TEXT, frequency DOUBLE PRECISION )`, `CREATE TABLE IF NOT EXISTS schema_version (version INTEGER)`, `INSERT INTO schema_version (version) VALUES (1)`, @@ -65,29 +73,37 @@ func createTestDBReverseTime(tb testing.TB, dbPath string, numTx int) { } } - txStmt, _ := conn.Prepare("INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) VALUES (?, ?, ?, ?, ?, ?, ?, ?)") - obsStmt, _ := conn.Prepare("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") + txStmt, err := conn.Prepare("INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)") + if err != nil { + tb.Fatal(err) + } defer txStmt.Close() + obsStmt, err := conn.Prepare("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)") + if err != nil { + tb.Fatal(err) + } defer obsStmt.Close() // id=1 is the NEWEST (now); id=numTx is the OLDEST (numTx minutes ago). now := time.Now().UTC().Truncate(time.Second) for i := 1; i <= numTx; i++ { ts := now.Add(-time.Duration(i-1) * time.Minute).Format(time.RFC3339) - unixTs := now.Add(-time.Duration(i-1) * time.Minute).Unix() hash := fmt.Sprintf("h%04d", i) - txStmt.Exec(i, "aabb", hash, ts, 0, 4, 1, fmt.Sprintf(`{"pubKey":"pk%04d"}`, i)) - obsStmt.Exec(i, i, "obs1", "Obs1", "RX", -10.0, -80.0, 5, `["aa","bb"]`, unixTs) + if _, err := txStmt.Exec(i, "aabb", hash, ts, 0, 4, 1, fmt.Sprintf(`{"pubKey":"pk%04d"}`, i)); err != nil { + tb.Fatal(err) + } + if _, err := obsStmt.Exec(i, i, "obs1", "Obs1", "RX", -10.0, -80.0, 5, `["aa","bb"]`, ts); err != nil { + tb.Fatal(err) + } } } func openReverseTimeStore(t *testing.T, numTx int) *PacketStore { t.Helper() - dir := t.TempDir() - dbPath := filepath.Join(dir, "rev.db") + dbPath := postgresTestDSN(t) createTestDBReverseTime(t, dbPath, numTx) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatalf("OpenDB: %v", err) } diff --git a/cmd/server/chunked_load_test.go b/cmd/server/chunked_load_test.go index c603acd3f..bd33c2aa4 100644 --- a/cmd/server/chunked_load_test.go +++ b/cmd/server/chunked_load_test.go @@ -19,20 +19,16 @@ package main import ( "net/http" "net/http/httptest" - "os" - "path/filepath" "testing" "time" ) func openChunkedTestStore(t *testing.T, numTx int) *PacketStore { t.Helper() - dir := t.TempDir() - dbPath := filepath.Join(dir, "chunked.db") + dbPath := postgresTestDSN(t) createTestDBAt(t, dbPath, numTx) - t.Cleanup(func() { os.RemoveAll(dir) }) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatalf("OpenDB: %v", err) } diff --git a/cmd/server/collision_details_test.go b/cmd/server/collision_details_test.go index c59492e27..cbf6e931a 100644 --- a/cmd/server/collision_details_test.go +++ b/cmd/server/collision_details_test.go @@ -21,14 +21,14 @@ func TestCollisionDetailsIncludeNodePairs(t *testing.T) { // Add advert transmissions with hash_size=3 path bytes (0x80 = bits 10 → size 3) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('0180aabb11ccdd', 'col_hash_01', ?, 1, 4, '{"pubKey":"aabb11ccdd001122","name":"Node Alpha","type":"ADVERT"}')`, recent) + VALUES ('0180aabb11ccdd', 'col_hash_01', $1, 1, 4, '{"pubKey":"aabb11ccdd001122","name":"Node Alpha","type":"ADVERT"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 10.0, -91, '["aabb11"]', ?)`, recentEpoch) + VALUES (1, 1, 10.0, -91, '["aabb11"]', $1)`, recentEpoch) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('0180aabb11eeff', 'col_hash_02', ?, 1, 4, '{"pubKey":"aabb11eeff334455","name":"Node Beta","type":"ADVERT"}')`, recent) + VALUES ('0180aabb11eeff', 'col_hash_02', $1, 1, 4, '{"pubKey":"aabb11eeff334455","name":"Node Beta","type":"ADVERT"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (2, 1, 9.0, -93, '["aabb11"]', ?)`, recentEpoch) + VALUES (2, 1, 9.0, -93, '["aabb11"]', $1)`, recentEpoch) store := NewPacketStore(db, nil) store.Load() @@ -102,9 +102,9 @@ func TestCollisionDetailsEmptyWhenNoCollisions(t *testing.T) { db.conn.Exec(`INSERT INTO nodes (public_key, name, role) VALUES ('aabb11ccdd001122', 'Solo Node', 'repeater')`) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('0180aabb11ccdd', 'solo_hash_01', ?, 1, 4, '{"pubKey":"aabb11ccdd001122","name":"Solo Node","type":"ADVERT"}')`, recent) + VALUES ('0180aabb11ccdd', 'solo_hash_01', $1, 1, 4, '{"pubKey":"aabb11ccdd001122","name":"Solo Node","type":"ADVERT"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 10.0, -91, '["aabb11"]', ?)`, recentEpoch) + VALUES (1, 1, 10.0, -91, '["aabb11"]', $1)`, recentEpoch) store := NewPacketStore(db, nil) store.Load() diff --git a/cmd/server/companion_handlers.go b/cmd/server/companion_handlers.go index 25a47cd46..99fbe731a 100644 --- a/cmd/server/companion_handlers.go +++ b/cmd/server/companion_handlers.go @@ -218,7 +218,7 @@ func (s *Server) companionLastSeen(pks []string) map[string]string { args[i] = pk } rows, err := s.db.conn.Query(`SELECT rx_pubkey, MAX(rx_at) FROM client_receptions WHERE rx_pubkey IN (`+ - sqlPlaceholders(len(pks))+`) GROUP BY rx_pubkey`, args...) + s.db.sqlPlaceholders(len(pks))+`) GROUP BY rx_pubkey`, args...) if err != nil { if !strings.Contains(err.Error(), "no such table") { log.Printf("[users] companion last seen: %v", err) diff --git a/cmd/server/config.go b/cmd/server/config.go index 3d92fbbed..2a7adbe18 100644 --- a/cmd/server/config.go +++ b/cmd/server/config.go @@ -37,10 +37,12 @@ type ListLimitsConfig struct { // Config mirrors the Node.js config.json structure (read-only fields). type Config struct { - Port int `json:"port"` - APIKey string `json:"apiKey"` - DBPath string `json:"dbPath"` - ListLimits *ListLimitsConfig `json:"listLimits"` + Port int `json:"port"` + APIKey string `json:"apiKey"` + DBPath string `json:"dbPath,omitempty"` // legacy SQLite path; an installed selection is authoritative + DatabaseURL string `json:"databaseURL"` + StateDir string `json:"stateDir"` + ListLimits *ListLimitsConfig `json:"listLimits"` // NodeBlacklist is a list of public keys to exclude from all API responses. // Blacklisted nodes are hidden from node lists, search, detail, map, and stats. @@ -396,7 +398,7 @@ type RetentionConfig struct { MetricsDays int `json:"metricsDays"` } -// DBConfig is the shared SQLite vacuum/maintenance config (#919, #921). +// DBConfig shares startup-load settings and accepts obsolete SQLite config keys. type DBConfig = dbconfig.DBConfig // IncrementalVacuumPages returns the configured pages per vacuum or 1024 default. @@ -520,10 +522,13 @@ func LoadConfig(baseDirs ...string) (*Config, error) { for _, p := range paths { data, err := os.ReadFile(p) if err != nil { - continue + if os.IsNotExist(err) { + continue + } + return nil, fmt.Errorf("cannot read %s; refusing configuration fallback", filepath.Base(p)) } if err := json.Unmarshal(data, cfg); err != nil { - continue + return nil, fmt.Errorf("%s is malformed or has invalid field types; refusing configuration fallback", filepath.Base(p)) } cfg.NormalizeTimestampConfig() cfg.migrateDeprecatedConfig() @@ -762,14 +767,27 @@ func (h HealthThresholds) ToClientMs() map[string]int { } } -func (c *Config) ResolveDBPath(baseDir string) string { - if c.DBPath != "" { - return c.DBPath +// ResolveDatabaseURL refuses legacy paths rather than interpreting them as DSNs. +func (c *Config) ResolveDatabaseURL() (string, error) { + if c.DBPath != "" || os.Getenv("DB_PATH") != "" { + return "", fmt.Errorf("dbPath and DB_PATH are no longer supported; import the SQLite database and configure databaseURL or CORESCOPE_DATABASE_URL") + } + v := envOrValue(os.Getenv, "CORESCOPE_READER_DATABASE_URL", envOrValue(os.Getenv, "CORESCOPE_DATABASE_URL", c.DatabaseURL)) + if v == "" { + return "", fmt.Errorf("configure databaseURL or CORESCOPE_DATABASE_URL for the PostgreSQL telemetry reader") + } + return v, nil +} + +func (c *Config) ResolveStateDir(baseDir string) string { + v := envOrValue(os.Getenv, "CORESCOPE_STATE_DIR", c.StateDir) + if v == "" { + v = "data" } - if v := os.Getenv("DB_PATH"); v != "" { + if filepath.IsAbs(v) { return v } - return filepath.Join(baseDir, "data", "meshcore.db") + return filepath.Join(baseDir, v) } func (c *Config) NormalizeTimestampConfig() { diff --git a/cmd/server/config_mode_test.go b/cmd/server/config_mode_test.go index 0abb6bfc0..eb604971e 100644 --- a/cmd/server/config_mode_test.go +++ b/cmd/server/config_mode_test.go @@ -19,6 +19,11 @@ func TestSaveGeoFilterPreservesFileMode(t *testing.T) { if err := os.WriteFile(path, []byte(`{"apiKey":"secret-key-that-is-long-enough"}`+"\n"), 0600); err != nil { t.Fatal(err) } + before, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + want := before.Mode().Perm() gf := &GeoFilterConfig{Polygon: [][2]float64{{0, 0}, {0, 1}, {1, 1}, {1, 0}}} if err := SaveGeoFilter(dir, gf); err != nil { t.Fatalf("SaveGeoFilter: %v", err) @@ -27,18 +32,20 @@ func TestSaveGeoFilterPreservesFileMode(t *testing.T) { if err != nil { t.Fatal(err) } - if got := fi.Mode().Perm(); got != 0600 { - t.Fatalf("config.json mode after save = %o, want 0600", got) + if got := fi.Mode().Perm(); got != want { + t.Fatalf("config.json mode after save = %o, want %o", got, want) } // Default stays 0644 for a file that was 0644. if err := os.Chmod(path, 0644); err != nil { t.Fatal(err) } + before, _ = os.Stat(path) + want = before.Mode().Perm() if err := SaveGeoFilter(dir, nil); err != nil { t.Fatalf("SaveGeoFilter(nil): %v", err) } fi, _ = os.Stat(path) - if got := fi.Mode().Perm(); got != 0644 { - t.Fatalf("config.json mode after save = %o, want 0644", got) + if got := fi.Mode().Perm(); got != want { + t.Fatalf("config.json mode after save = %o, want %o", got, want) } } diff --git a/cmd/server/config_test.go b/cmd/server/config_test.go index b78968ab6..b241e445d 100644 --- a/cmd/server/config_test.go +++ b/cmd/server/config_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "os" "path/filepath" + "strings" "testing" ) @@ -103,15 +104,47 @@ func TestLoadConfigNoFiles(t *testing.T) { func TestLoadConfigInvalidJSON(t *testing.T) { dir := t.TempDir() - os.WriteFile(filepath.Join(dir, "config.json"), []byte("{invalid"), 0644) - - cfg, err := LoadConfig(dir) - if err != nil { + if err := os.WriteFile(filepath.Join(dir, "config.json"), []byte("{invalid"), 0600); err != nil { t.Fatal(err) } - // Should return defaults when JSON is invalid - if cfg.Port != 3000 { - t.Errorf("expected default port 3000, got %d", cfg.Port) + if cfg, err := LoadConfig(dir); err == nil || cfg != nil { + t.Fatal("malformed primary config fell back to defaults") + } +} + +func TestLoadConfigNeverUsesFallbackAfterExistingConfigError(t *testing.T) { + for _, kind := range []string{"invalid JSON", "invalid field type", "unreadable file"} { + t.Run(kind, func(t *testing.T) { + dir := t.TempDir() + if err := os.Mkdir(filepath.Join(dir, "data"), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "data", "config.json"), []byte(`{"stateDir":"unintended-installation","dbPath":"other.db"}`), 0600); err != nil { + t.Fatal(err) + } + primary := filepath.Join(dir, "config.json") + switch kind { + case "invalid JSON": + if err := os.WriteFile(primary, []byte(`{"stateDir":`), 0600); err != nil { + t.Fatal(err) + } + case "invalid field type": + if err := os.WriteFile(primary, []byte(`{"databaseURL":"postgresql://user:private-password@host/database","port":"invalid"}`), 0600); err != nil { + t.Fatal(err) + } + case "unreadable file": + if err := os.Mkdir(primary, 0700); err != nil { + t.Fatal(err) + } + } + cfg, err := LoadConfig(dir) + if err == nil || cfg != nil { + t.Fatal("existing config error selected fallback storage") + } + if strings.Contains(err.Error(), "private-password") { + t.Fatal("config error exposed credentials") + } + }) } } @@ -320,33 +353,45 @@ func TestGetHealthMs(t *testing.T) { } } -func TestResolveDBPath(t *testing.T) { - t.Run("DBPath set", func(t *testing.T) { - cfg := &Config{DBPath: "/explicit/path.db"} - got := cfg.ResolveDBPath("/base") - if got != "/explicit/path.db" { - t.Errorf("expected /explicit/path.db, got %s", got) - } - }) - - t.Run("env var", func(t *testing.T) { - cfg := &Config{} - t.Setenv("DB_PATH", "/env/path.db") - got := cfg.ResolveDBPath("/base") - if got != "/env/path.db" { - t.Errorf("expected /env/path.db, got %s", got) - } - }) +func TestResolveDatabaseURL(t *testing.T) { + t.Setenv("CORESCOPE_READER_DATABASE_URL", "") + t.Setenv("DB_PATH", "") + t.Setenv("CORESCOPE_DATABASE_URL", "") + cfg := &Config{DatabaseURL: "postgresql://reader@localhost/telemetry"} + if got, err := cfg.ResolveDatabaseURL(); err != nil || got != cfg.DatabaseURL { + t.Fatalf("config URL: %v", err) + } + t.Setenv("CORESCOPE_DATABASE_URL", "postgresql://reader@localhost/envdb") + if got, err := cfg.ResolveDatabaseURL(); err != nil || got != "postgresql://reader@localhost/envdb" { + t.Fatalf("environment URL: %v", err) + } + t.Setenv("CORESCOPE_READER_DATABASE_URL", "postgresql://reader@localhost/role") + if got, err := cfg.ResolveDatabaseURL(); err != nil || got != "postgresql://reader@localhost/role" { + t.Fatalf("reader URL precedence: %v", err) + } + t.Setenv("CORESCOPE_READER_DATABASE_URL", "") + cfg.DBPath = "private.db" + if _, err := cfg.ResolveDatabaseURL(); err == nil || strings.Contains(err.Error(), cfg.DBPath) { + t.Fatalf("legacy path must be rejected without echoing input: %v", err) + } + cfg.DBPath = "" + t.Setenv("CORESCOPE_DATABASE_URL", "") + cfg.DatabaseURL = "" + if _, err := cfg.ResolveDatabaseURL(); err == nil { + t.Fatal("missing URL accepted") + } +} - t.Run("default", func(t *testing.T) { - cfg := &Config{} - t.Setenv("DB_PATH", "") - got := cfg.ResolveDBPath("/base") - expected := filepath.Join("/base", "data", "meshcore.db") - if got != expected { - t.Errorf("expected %s, got %s", expected, got) - } - }) +func TestResolveStateDir(t *testing.T) { + t.Setenv("CORESCOPE_STATE_DIR", "") + cfg := &Config{} + if got := cfg.ResolveStateDir("base"); got != filepath.Join("base", "data") { + t.Fatal(got) + } + t.Setenv("CORESCOPE_STATE_DIR", "queues") + if got := cfg.ResolveStateDir("base"); got != filepath.Join("base", "queues") { + t.Fatal(got) + } } func TestPropagationBufferMs(t *testing.T) { diff --git a/cmd/server/coverage_test.go b/cmd/server/coverage_test.go index e65566da3..44a39c5d9 100644 --- a/cmd/server/coverage_test.go +++ b/cmd/server/coverage_test.go @@ -15,60 +15,85 @@ import ( "time" "github.com/gorilla/mux" - _ "github.com/mattn/go-sqlite3" ) // --- helpers --- func setupTestDBv2(t *testing.T) *DB { t.Helper() - conn, err := sql.Open("sqlite3", ":memory:") + conn, err := openPostgresTestDB(t) if err != nil { t.Fatal(err) } // Force single connection so all goroutines share the same in-memory DB conn.SetMaxOpenConns(1) - schema := ` + schema := testNativeSQL(` CREATE TABLE nodes ( public_key TEXT PRIMARY KEY, name TEXT, role TEXT, - lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, advert_count INTEGER DEFAULT 0, - battery_mv INTEGER, temperature_c REAL, foreign_advert INTEGER DEFAULT 0 + lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, first_seen TEXT, advert_count INTEGER DEFAULT 0, + battery_mv INTEGER, temperature_c DOUBLE PRECISION, foreign_advert INTEGER DEFAULT 0 ); - CREATE TABLE observers ( - id TEXT PRIMARY KEY, name TEXT, iata TEXT, last_seen TEXT, first_seen TEXT, + CREATE TABLE observers (id TEXT PRIMARY KEY, name TEXT, iata TEXT, last_seen TEXT, first_seen TEXT, packet_count INTEGER DEFAULT 0, model TEXT, firmware TEXT, - client_version TEXT, radio TEXT, battery_mv INTEGER, uptime_secs INTEGER, noise_floor REAL, + client_version TEXT, radio TEXT, battery_mv INTEGER, uptime_secs INTEGER, noise_floor DOUBLE PRECISION, inactive INTEGER DEFAULT 0 ); CREATE TABLE transmissions ( id INTEGER PRIMARY KEY AUTOINCREMENT, raw_hex TEXT NOT NULL, hash TEXT NOT NULL UNIQUE, first_seen TEXT NOT NULL, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, - decoded_json TEXT, channel_hash TEXT DEFAULT NULL, from_pubkey TEXT DEFAULT NULL, created_at TEXT DEFAULT (datetime('now')) + decoded_json TEXT, channel_hash TEXT DEFAULT NULL, from_pubkey TEXT DEFAULT NULL, created_at TEXT DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ','now')) ); CREATE TABLE observations ( id INTEGER PRIMARY KEY AUTOINCREMENT, transmission_id INTEGER NOT NULL REFERENCES transmissions(id), observer_id TEXT, observer_name TEXT, direction TEXT, - snr REAL, rssi REAL, score INTEGER, path_json TEXT, timestamp INTEGER NOT NULL, raw_hex TEXT + snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp INTEGER NOT NULL, raw_hex TEXT + ); + CREATE TRIGGER test_from_pubkey_advert AFTER INSERT ON transmissions + WHEN NEW.from_pubkey IS NULL AND NEW.payload_type=4 AND + CASE WHEN json_valid(NEW.decoded_json) THEN json_type(NEW.decoded_json)='object' ELSE 0 END + BEGIN UPDATE transmissions SET from_pubkey=NULLIF(json_extract(NEW.decoded_json,'$.pubKey'),'') WHERE id=NEW.id; END; + CREATE INDEX IF NOT EXISTS idx_transmissions_from_pubkey ON transmissions(from_pubkey); + `, ` + CREATE TABLE nodes ( + public_key TEXT PRIMARY KEY, name TEXT, role TEXT, + lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, first_seen TEXT, advert_count INTEGER DEFAULT 0, + battery_mv INTEGER, temperature_c DOUBLE PRECISION, foreign_advert INTEGER DEFAULT 0 + ); + CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY UNIQUE, + id TEXT PRIMARY KEY, name TEXT, iata TEXT, last_seen TEXT, first_seen TEXT, + packet_count INTEGER DEFAULT 0, model TEXT, firmware TEXT, + client_version TEXT, radio TEXT, battery_mv INTEGER, uptime_secs INTEGER, noise_floor DOUBLE PRECISION, + inactive INTEGER DEFAULT 0 + ); + CREATE TABLE transmissions ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, raw_hex TEXT NOT NULL, + hash TEXT NOT NULL UNIQUE, first_seen TEXT NOT NULL, + route_type INTEGER, payload_type INTEGER, payload_version INTEGER, + decoded_json TEXT, channel_hash TEXT DEFAULT NULL, from_pubkey TEXT DEFAULT NULL, created_at TEXT DEFAULT (to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"')) + ); + CREATE TABLE observations ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + transmission_id INTEGER NOT NULL REFERENCES transmissions(id), + observer_id TEXT, observer_name TEXT, direction TEXT, + snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp INTEGER NOT NULL, raw_hex TEXT ); - CREATE TRIGGER IF NOT EXISTS test_from_pubkey_advert - AFTER INSERT ON transmissions - FOR EACH ROW - WHEN NEW.from_pubkey IS NULL AND NEW.payload_type = 4 AND NEW.decoded_json IS NOT NULL - AND json_extract(NEW.decoded_json, '$.pubKey') IS NOT NULL - AND json_extract(NEW.decoded_json, '$.pubKey') <> '' + CREATE FUNCTION test_from_pubkey_advert() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN - UPDATE transmissions - SET from_pubkey = json_extract(NEW.decoded_json, '$.pubKey') - WHERE id = NEW.id; - END; + IF NEW.from_pubkey IS NULL AND NEW.payload_type = 4 AND NEW.decoded_json IS JSON OBJECT THEN + NEW.from_pubkey := NULLIF(NEW.decoded_json::json ->> 'pubKey', ''); + END IF; + RETURN NEW; + END $$; + CREATE TRIGGER test_from_pubkey_advert BEFORE INSERT ON transmissions + FOR EACH ROW EXECUTE FUNCTION test_from_pubkey_advert(); CREATE INDEX IF NOT EXISTS idx_transmissions_from_pubkey ON transmissions(from_pubkey); - ` + `) if _, err := conn.Exec(schema); err != nil { t.Fatal(err) } - return &DB{conn: conn, isV3: false} + return &DB{backend: testBackendValue(), conn: conn, isV3: false} } func seedV2Data(t *testing.T, db *DB) { @@ -78,13 +103,13 @@ func seedV2Data(t *testing.T, db *DB) { epoch := now.Add(-1 * time.Hour).Unix() db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obs1', 'Obs One', 'SJC', ?, '2026-01-01T00:00:00Z', 100)`, recent) + VALUES ('obs1', 'Obs One', 'SJC', $1, '2026-01-01T00:00:00Z', 100)`, recent) db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES ('aabbccdd11223344', 'TestRepeater', 'repeater', 37.5, -122.0, ?, '2026-01-01T00:00:00Z', 50)`, recent) + VALUES ('aabbccdd11223344', 'TestRepeater', 'repeater', 37.5, -122.0, $1, '2026-01-01T00:00:00Z', 50)`, recent) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('AABB', 'abc123def4567890', ?, 1, 4, '{"pubKey":"aabbccdd11223344","name":"TestRepeater","type":"ADVERT"}')`, recent) + VALUES ('AABB', 'abc123def4567890', $1, 1, 4, '{"pubKey":"aabbccdd11223344","name":"TestRepeater","type":"ADVERT"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_id, observer_name, snr, rssi, path_json, timestamp) - VALUES (1, 'obs1', 'Obs One', 12.5, -90, '["aa","bb"]', ?)`, epoch) + VALUES (1, 'obs1', 'Obs One', 12.5, -90, '["aa","bb"]', $1)`, epoch) } func setupNoStoreServer(t *testing.T) (*Server, *mux.Router) { @@ -339,13 +364,13 @@ func TestIngestNewFromDB(t *testing.T) { // Insert a new transmission in DB now := time.Now().UTC().Format(time.RFC3339) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('EEFF', 'newhash123456abcd', ?, 1, 4, '{"pubKey":"aabbccdd11223344","type":"ADVERT"}')`, now) + VALUES ('EEFF', 'newhash123456abcd', $1, 1, 4, '{"pubKey":"aabbccdd11223344","type":"ADVERT"}')`, now) newTxID := 0 db.conn.QueryRow("SELECT MAX(id) FROM transmissions").Scan(&newTxID) // Add observation for the new transmission db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (?, 1, 10.0, -92, '["cc"]', ?)`, newTxID, time.Now().Unix()) + VALUES ($1, 1, 10.0, -92, '["cc"]', $2)`, newTxID, time.Now().Unix()) // Ingest broadcastMaps, newMax := store.IngestNewFromDB(initialMax, 100) @@ -409,11 +434,11 @@ func TestIngestNewFromDBv2(t *testing.T) { now := time.Now().UTC().Format(time.RFC3339) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('EEFF', 'v2newhash12345678', ?, 1, 4, '{"pubKey":"aabbccdd11223344","type":"ADVERT"}')`, now) + VALUES ('EEFF', 'v2newhash12345678', $1, 1, 4, '{"pubKey":"aabbccdd11223344","type":"ADVERT"}')`, now) newTxID := 0 db.conn.QueryRow("SELECT MAX(id) FROM transmissions").Scan(&newTxID) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_id, observer_name, snr, rssi, path_json, timestamp) - VALUES (?, 'obs1', 'Obs One', 10.0, -92, '["cc"]', ?)`, newTxID, time.Now().Unix()) + VALUES ($1, 'obs1', 'Obs One', 10.0, -92, '["cc"]', $2)`, newTxID, time.Now().Unix()) broadcastMaps, newMax := store.IngestNewFromDB(initialMax, 100) if newMax <= initialMax { @@ -729,15 +754,15 @@ func TestGetChannelMessagesDedupe(t *testing.T) { // Insert a duplicate channel message with the same hash as existing db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('DDEE', 'dupchannelhash1234', ?, 1, 5, '{"type":"CHAN","channel":"#test","text":"Hello: World","sender":"TestUser"}')`, recent) + VALUES ('DDEE', 'dupchannelhash1234', $1, 1, 5, '{"type":"CHAN","channel":"#test","text":"Hello: World","sender":"TestUser"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (3, 1, 11.0, -91, '["aa"]', ?)`, epoch) + VALUES (3, 1, 11.0, -91, '["aa"]', $1)`, epoch) // Insert another dupe same hash as above (should dedup) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('DDFF', 'dupchannelhash5678', ?, 1, 5, '{"type":"CHAN","channel":"#test","text":"Hello: World","sender":"TestUser"}')`, recent) + VALUES ('DDFF', 'dupchannelhash5678', $1, 1, 5, '{"type":"CHAN","channel":"#test","text":"Hello: World","sender":"TestUser"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (4, 2, 9.0, -93, '[]', ?)`, epoch) + VALUES (4, 2, 9.0, -93, '[]', $1)`, epoch) store := NewPacketStore(db, nil) store.Load() @@ -1130,9 +1155,9 @@ func TestPollerStartWithStore(t *testing.T) { // Insert new data while poller running now := time.Now().UTC().Format(time.RFC3339) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type) - VALUES ('FFEE', 'pollerhash12345678', ?, 1, 4)`, now) + VALUES ('FFEE', 'pollerhash12345678', $1, 1, 4)`, now) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES ((SELECT MAX(id) FROM transmissions), 1, 10.0, -92, '[]', ?)`, time.Now().Unix()) + VALUES ((SELECT MAX(id) FROM transmissions), 1, 10.0, -92, '[]', $1)`, time.Now().Unix()) time.Sleep(200 * time.Millisecond) poller.Stop() @@ -1374,7 +1399,7 @@ func TestBuildTransmissionWhereRFC3339(t *testing.T) { // PR #1187 r2: RFC3339 since/until MUST use observations.timestamp // subquery so re-observed packets (older first_seen but recent // observation) are still included. Anything else breaks semantics. - if !strings.Contains(where[0], "observations") || !strings.Contains(where[0], "timestamp >= ?") { + if !strings.Contains(where[0], "observations") || !strings.Contains(where[0], "timestamp >= "+db.parameter(1)+"") { t.Errorf("expected observations.timestamp subquery for RFC3339 since, got %q", where[0]) } }) @@ -1388,7 +1413,7 @@ func TestBuildTransmissionWhereRFC3339(t *testing.T) { if len(args) != 1 { t.Errorf("expected 1 arg, got %d", len(args)) } - if !strings.Contains(where[0], "observations") || !strings.Contains(where[0], "timestamp <= ?") { + if !strings.Contains(where[0], "observations") || !strings.Contains(where[0], "timestamp <= "+db.parameter(1)+"") { t.Errorf("expected observations.timestamp subquery for RFC3339 until, got %q", where[0]) } }) @@ -2188,21 +2213,21 @@ func setupRichTestDB(t *testing.T) *DB { // route_type 1 = FLOOD, path byte at position 1 (hex index 2..3) // header: 0x01 (route_type=1), path byte: 0x40 (hashSize bits=01 → size 2) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('0140aabbccdd', 'hash_with_path_01', ?, 1, 4, '{"pubKey":"aabbccdd11223344","name":"TestRepeater","type":"ADVERT"}')`, recent) + VALUES ('0140aabbccdd', 'hash_with_path_01', $1, 1, 4, '{"pubKey":"aabbccdd11223344","name":"TestRepeater","type":"ADVERT"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (3, 1, 10.0, -91, '["aabb","ccdd"]', ?)`, recentEpoch) + VALUES (3, 1, 10.0, -91, '["aabb","ccdd"]', $1)`, recentEpoch) // Another advert with 3-byte hash size: header 0x01, path byte 0x80 (bits=10 → size 3) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('0180eeff0011', 'hash_with_path_02', ?, 1, 4, '{"pubKey":"eeff00112233aabb","name":"TestCompanion","type":"ADVERT"}')`, yesterday) + VALUES ('0180eeff0011', 'hash_with_path_02', $1, 1, 4, '{"pubKey":"eeff00112233aabb","name":"TestCompanion","type":"ADVERT"}')`, yesterday) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (4, 2, 8.5, -94, '["eeff","0011","2233"]', ?)`, yesterdayEpoch) + VALUES (4, 2, 8.5, -94, '["eeff","0011","2233"]', $1)`, yesterdayEpoch) // Another channel message with different sender for analytics db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('CC01', 'chan_msg_hash_001', ?, 1, 5, '{"type":"CHAN","channel":"#test","text":"User2: Another msg","sender":"User2","channelHash":"abc123"}')`, recent) + VALUES ('CC01', 'chan_msg_hash_001', $1, 1, 5, '{"type":"CHAN","channel":"#test","text":"User2: Another msg","sender":"User2","channelHash":"abc123"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (5, 1, 14.0, -88, '["aa"]', ?)`, recentEpoch) + VALUES (5, 1, 14.0, -88, '["aa"]', $1)`, recentEpoch) return db } @@ -2525,20 +2550,20 @@ func TestStoreGetAnalyticsChannelsNumericHash(t *testing.T) { // Insert GRP_TXT packets with numeric channelHash (matches decoder.js output) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('DD01', 'grp_num_hash_1', ?, 1, 5, '{"type":"GRP_TXT","channelHash":97,"channelHashHex":"61","decryptionStatus":"no_key"}')`, recent) + VALUES ('DD01', 'grp_num_hash_1', $1, 1, 5, '{"type":"GRP_TXT","channelHash":97,"channelHashHex":"61","decryptionStatus":"no_key"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (4, 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (4, 1, 10.0, -90, '[]', $1)`, recentEpoch) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('DD02', 'grp_num_hash_2', ?, 1, 5, '{"type":"GRP_TXT","channelHash":42,"channelHashHex":"2A","decryptionStatus":"no_key"}')`, recent) + VALUES ('DD02', 'grp_num_hash_2', $1, 1, 5, '{"type":"GRP_TXT","channelHash":42,"channelHashHex":"2A","decryptionStatus":"no_key"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (5, 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (5, 1, 10.0, -90, '[]', $1)`, recentEpoch) // Also a decrypted CHAN with numeric channelHash — use hash 198 which is the real hash for #general db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('DD03', 'chan_num_hash_3', ?, 1, 5, '{"type":"CHAN","channel":"general","channelHash":198,"channelHashHex":"C6","text":"hello","sender":"Alice"}')`, recent) + VALUES ('DD03', 'chan_num_hash_3', $1, 1, 5, '{"type":"CHAN","channel":"general","channelHash":198,"channelHashHex":"C6","text":"hello","sender":"Alice"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (6, 1, 12.0, -88, '[]', ?)`, recentEpoch) + VALUES (6, 1, 12.0, -88, '[]', $1)`, recentEpoch) store := NewPacketStore(db, nil) store.Load() @@ -3189,64 +3214,64 @@ func TestGetNodeHashSizeInfoEdgeCases(t *testing.T) { // Observers db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obs1', 'Obs', 'SJC', ?, '2026-01-01T00:00:00Z', 10)`, recent) + VALUES ('obs1', 'Obs', 'SJC', $1, '2026-01-01T00:00:00Z', 10)`, recent) // Adverts with various edge cases // 1. Valid advert with pubKey db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('0140aabbccdd', 'hs_valid_1', ?, 1, 4, '{"pubKey":"aabbccdd11223344","name":"NodeA","type":"ADVERT"}')`, recent) + VALUES ('0140aabbccdd', 'hs_valid_1', $1, 1, 4, '{"pubKey":"aabbccdd11223344","name":"NodeA","type":"ADVERT"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (1, 1, 10.0, -90, '[]', $1)`, recentEpoch) // 2. Short raw_hex (< 4 chars) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('01', 'hs_short_hex', ?, 1, 4, '{"pubKey":"eeff00112233aabb","name":"NodeB","type":"ADVERT"}')`, recent) + VALUES ('01', 'hs_short_hex', $1, 1, 4, '{"pubKey":"eeff00112233aabb","name":"NodeB","type":"ADVERT"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (2, 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (2, 1, 10.0, -90, '[]', $1)`, recentEpoch) // 3. Invalid hex in path byte position db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('01GGHHII', 'hs_bad_hex', ?, 1, 4, '{"pubKey":"1122334455667788","name":"NodeC","type":"ADVERT"}')`, recent) + VALUES ('01GGHHII', 'hs_bad_hex', $1, 1, 4, '{"pubKey":"1122334455667788","name":"NodeC","type":"ADVERT"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (3, 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (3, 1, 10.0, -90, '[]', $1)`, recentEpoch) // 4. Invalid JSON db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('0140aabb', 'hs_bad_json', ?, 1, 4, 'not-json')`, recent) + VALUES ('0140aabb', 'hs_bad_json', $1, 1, 4, 'not-json')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (4, 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (4, 1, 10.0, -90, '[]', $1)`, recentEpoch) // 5. JSON with public_key field instead of pubKey db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('0180eeff', 'hs_alt_key', ?, 1, 4, '{"public_key":"aabbccdd11223344","name":"NodeA","type":"ADVERT"}')`, recent) + VALUES ('0180eeff', 'hs_alt_key', $1, 1, 4, '{"public_key":"aabbccdd11223344","name":"NodeA","type":"ADVERT"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (5, 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (5, 1, 10.0, -90, '[]', $1)`, recentEpoch) // 6. JSON with no pubKey at all db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('01C0ffee', 'hs_no_pk', ?, 1, 4, '{"name":"NodeZ","type":"ADVERT"}')`, recent) + VALUES ('01C0ffee', 'hs_no_pk', $1, 1, 4, '{"name":"NodeZ","type":"ADVERT"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (6, 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (6, 1, 10.0, -90, '[]', $1)`, recentEpoch) // 7. Empty decoded_json db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('0140bbcc', 'hs_empty_json', ?, 1, 4, '')`, recent) + VALUES ('0140bbcc', 'hs_empty_json', $1, 1, 4, '')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (7, 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (7, 1, 10.0, -90, '[]', $1)`, recentEpoch) // 8-10. Multiple adverts for same node with different hash sizes (flip-flop test) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('0140dd01', 'hs_flip_1', ?, 1, 4, '{"pubKey":"ffff000011112222","name":"Flipper","type":"ADVERT"}')`, recent) + VALUES ('0140dd01', 'hs_flip_1', $1, 1, 4, '{"pubKey":"ffff000011112222","name":"Flipper","type":"ADVERT"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (8, 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (8, 1, 10.0, -90, '[]', $1)`, recentEpoch) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('0180dd02', 'hs_flip_2', ?, 1, 4, '{"pubKey":"ffff000011112222","name":"Flipper","type":"ADVERT"}')`, recent) + VALUES ('0180dd02', 'hs_flip_2', $1, 1, 4, '{"pubKey":"ffff000011112222","name":"Flipper","type":"ADVERT"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (9, 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (9, 1, 10.0, -90, '[]', $1)`, recentEpoch) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('0140dd03', 'hs_flip_3', ?, 1, 4, '{"pubKey":"ffff000011112222","name":"Flipper","type":"ADVERT"}')`, recent) + VALUES ('0140dd03', 'hs_flip_3', $1, 1, 4, '{"pubKey":"ffff000011112222","name":"Flipper","type":"ADVERT"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (10, 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (10, 1, 10.0, -90, '[]', $1)`, recentEpoch) store := NewPacketStore(db, nil) store.Load() @@ -3290,7 +3315,7 @@ func TestHashSizeTransportRoutePathByteOffset(t *testing.T) { recentEpoch := now.Add(-1 * time.Hour).Unix() db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obs1', 'Obs', 'SJC', ?, '2026-01-01T00:00:00Z', 10)`, recent) + VALUES ('obs1', 'Obs', 'SJC', $1, '2026-01-01T00:00:00Z', 10)`, recent) // Route type 0 (TRANSPORT_FLOOD): header=0x04 (payload_type=1, route_type=0) // 4 transport bytes + path byte at offset 5. @@ -3298,25 +3323,25 @@ func TestHashSizeTransportRoutePathByteOffset(t *testing.T) { // If bug is present, code reads byte 1 (0xAA) → hash_size bits = 10 → size 3 (coincidence) // Use path byte 0x40 (hash_size=2) and transport byte 0x01 at offset 1 (hash_size=1 if misread) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('100102030440aabb', 'tf_offset', ?, 0, 4, '{"pubKey":"aaaa000000000001","name":"TF-Node","type":"ADVERT"}')`, recent) + VALUES ('100102030440aabb', 'tf_offset', $1, 0, 4, '{"pubKey":"aaaa000000000001","name":"TF-Node","type":"ADVERT"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (1, 1, 10.0, -90, '[]', $1)`, recentEpoch) // Route type 3 (TRANSPORT_DIRECT): header=0x13 (payload_type=4, route_type=3) // 4 transport bytes + path byte at offset 5. // Path byte 0xC1 → hash_size bits = 11 → size 4, hop_count = 1 (not zero-hop) // Byte 1 = 0x05 → hash_size bits = 00 → size 1 if misread db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('1305060708C1bbcc', 'td_offset', ?, 3, 4, '{"pubKey":"aaaa000000000002","name":"TD-Node","type":"ADVERT"}')`, recent) + VALUES ('1305060708C1bbcc', 'td_offset', $1, 3, 4, '{"pubKey":"aaaa000000000002","name":"TD-Node","type":"ADVERT"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (2, 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (2, 1, 10.0, -90, '[]', $1)`, recentEpoch) // Route type 1 (FLOOD): header=0x11 (payload_type=4, route_type=1) // Path byte at offset 1. Path byte 0x80 → hash_size bits = 10 → size 3 db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('1180aabbccdd', 'flood_offset', ?, 1, 4, '{"pubKey":"aaaa000000000003","name":"Flood-Node","type":"ADVERT"}')`, recent) + VALUES ('1180aabbccdd', 'flood_offset', $1, 1, 4, '{"pubKey":"aaaa000000000003","name":"Flood-Node","type":"ADVERT"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (3, 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (3, 1, 10.0, -90, '[]', $1)`, recentEpoch) store := NewPacketStore(db, nil) store.Load() @@ -3355,7 +3380,7 @@ func TestHashSizeTransportDirectZeroHopSkipped(t *testing.T) { recentEpoch := now.Add(-1 * time.Hour).Unix() db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obs1', 'Obs', 'SJC', ?, '2026-01-01T00:00:00Z', 10)`, recent) + VALUES ('obs1', 'Obs', 'SJC', $1, '2026-01-01T00:00:00Z', 10)`, recent) // RouteDirect (2) zero-hop with an all-zero path byte: the sender wiped the // size bits, so there is nothing to read. Should be skipped. @@ -3363,23 +3388,23 @@ func TestHashSizeTransportDirectZeroHopSkipped(t *testing.T) { // a deliberate declaration — see TestZeroHopDirectAdvertDeclaredSizeIsEvidence // — so the skip is keyed on the byte's content, not on the route type.) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('1200aabbccdd', 'direct_zh', ?, 2, 4, '{"pubKey":"bbbb000000000001","name":"Direct-ZH","type":"ADVERT"}')`, recent) + VALUES ('1200aabbccdd', 'direct_zh', $1, 2, 4, '{"pubKey":"bbbb000000000001","name":"Direct-ZH","type":"ADVERT"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (1, 1, 10.0, -90, '[]', $1)`, recentEpoch) // RouteTransportDirect (3) zero-hop: 4 transport bytes + all-zero path byte // Should ALSO be skipped (this was the missing case in #747) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('130102030400aabb', 'tdirect_zh', ?, 3, 4, '{"pubKey":"bbbb000000000002","name":"TDirect-ZH","type":"ADVERT"}')`, recent) + VALUES ('130102030400aabb', 'tdirect_zh', $1, 3, 4, '{"pubKey":"bbbb000000000002","name":"TDirect-ZH","type":"ADVERT"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (2, 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (2, 1, 10.0, -90, '[]', $1)`, recentEpoch) // RouteDirect (2) non-zero-hop: path byte 0x41 → hop_count=1 // Should NOT be skipped db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('1241aabbccdd', 'direct_1h', ?, 2, 4, '{"pubKey":"bbbb000000000003","name":"Direct-1H","type":"ADVERT"}')`, recent) + VALUES ('1241aabbccdd', 'direct_1h', $1, 2, 4, '{"pubKey":"bbbb000000000003","name":"Direct-1H","type":"ADVERT"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (3, 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (3, 1, 10.0, -90, '[]', $1)`, recentEpoch) store := NewPacketStore(db, nil) store.Load() @@ -3415,10 +3440,10 @@ func TestAnalyticsHashSizesZeroHopSkip(t *testing.T) { recentEpoch := now.Add(-1 * time.Hour).Unix() db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obs1', 'Obs', 'SJC', ?, '2026-01-01T00:00:00Z', 10)`, recent) + VALUES ('obs1', 'Obs', 'SJC', $1, '2026-01-01T00:00:00Z', 10)`, recent) pk := "cccc000000000001" - db.conn.Exec(`INSERT INTO nodes (public_key, name, role) VALUES (?, 'ZH-Analytics', 'repeater')`, pk) + db.conn.Exec(`INSERT INTO nodes (public_key, name, role) VALUES ($1, 'ZH-Analytics', 'repeater')`, pk) decoded := `{"pubKey":"` + pk + `","name":"ZH-Analytics","type":"ADVERT"}` @@ -3426,16 +3451,16 @@ func TestAnalyticsHashSizesZeroHopSkip(t *testing.T) { // header 0x11 = route_type 1 (flood), payload_type 4 // pathByte 0x41 = hashSize bits 01 → size 2, hop_count 1 db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('1141aabbccdd', 'az_flood', ?, 1, 4, ?)`, recent, decoded) + VALUES ('1141aabbccdd', 'az_flood', $1, 1, 4, $2)`, recent, decoded) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 10.0, -90, '["aabb"]', ?)`, recentEpoch) + VALUES (1, 1, 10.0, -90, '["aabb"]', $1)`, recentEpoch) // Second: a direct zero-hop advert with pathByte=0x00 → would give hashSize=1 // header 0x12 = route_type 2 (direct), payload_type 4 db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('1200aabbccdd', 'az_direct', ?, 2, 4, ?)`, recent, decoded) + VALUES ('1200aabbccdd', 'az_direct', $1, 2, 4, $2)`, recent, decoded) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (2, 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (2, 1, 10.0, -90, '[]', $1)`, recentEpoch) store := NewPacketStore(db, nil) store.Load() @@ -3569,32 +3594,32 @@ func TestGetChannelMessagesDedupeRepeats(t *testing.T) { recentEpoch := now.Add(-1 * time.Hour).Unix() db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obs1', 'Obs1', 'SJC', ?, '2026-01-01T00:00:00Z', 10)`, recent) + VALUES ('obs1', 'Obs1', 'SJC', $1, '2026-01-01T00:00:00Z', 10)`, recent) db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obs2', 'Obs2', 'LAX', ?, '2026-01-01T00:00:00Z', 10)`, recent) + VALUES ('obs2', 'Obs2', 'LAX', $1, '2026-01-01T00:00:00Z', 10)`, recent) // Insert two copies of same CHAN message (same hash, different observers) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('CC01', 'dedup_chan_1', ?, 1, 5, '{"type":"CHAN","channel":"#general","text":"Alice: hello","sender":"Alice"}')`, recent) + VALUES ('CC01', 'dedup_chan_1', $1, 1, 5, '{"type":"CHAN","channel":"#general","text":"Alice: hello","sender":"Alice"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 12.0, -88, '["aa"]', ?)`, recentEpoch) + VALUES (1, 1, 12.0, -88, '["aa"]', $1)`, recentEpoch) // Same sender + hash → different observation (simulates dedup) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('CC02', 'dedup_chan_1', ?, 1, 5, '{"type":"CHAN","channel":"#general","text":"Alice: hello","sender":"Alice"}')`, recent) + VALUES ('CC02', 'dedup_chan_1', $1, 1, 5, '{"type":"CHAN","channel":"#general","text":"Alice: hello","sender":"Alice"}')`, recent) // Note: won't load due to UNIQUE constraint on hash → tests the code path with single tx having multiple obs // Second different message db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('CC03', 'dedup_chan_2', ?, 1, 5, '{"type":"CHAN","channel":"#general","text":"Bob: world","sender":"Bob"}')`, recent) + VALUES ('CC03', 'dedup_chan_2', $1, 1, 5, '{"type":"CHAN","channel":"#general","text":"Bob: world","sender":"Bob"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (2, 2, 10.0, -90, '["bb"]', ?)`, recentEpoch) + VALUES (2, 2, 10.0, -90, '["bb"]', $1)`, recentEpoch) // GRP_TXT (not CHAN) — should be skipped by GetChannelMessages db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('DD01', 'grp_msg_hash_1', ?, 1, 5, '{"type":"GRP_TXT","channelHash":"42","text":"encrypted"}')`, recent) + VALUES ('DD01', 'grp_msg_hash_1', $1, 1, 5, '{"type":"GRP_TXT","channelHash":"42","text":"encrypted"}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (3, 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (3, 1, 10.0, -90, '[]', $1)`, recentEpoch) store := NewPacketStore(db, nil) store.Load() @@ -3916,16 +3941,16 @@ func TestIngestNewFromDBDuplicateObs(t *testing.T) { // Insert new transmission with same hash as existing (should merge into existing tx) now := time.Now().UTC().Format(time.RFC3339) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('AABB', 'dedup_test_hash_01', ?, 1, 4, '{"pubKey":"aabbccdd11223344","type":"ADVERT"}')`, now) + VALUES ('AABB', 'dedup_test_hash_01', $1, 1, 4, '{"pubKey":"aabbccdd11223344","type":"ADVERT"}')`, now) newTxID := 0 db.conn.QueryRow("SELECT MAX(id) FROM transmissions").Scan(&newTxID) // Add observation db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (?, 1, 11.0, -89, '["dd"]', ?)`, newTxID, time.Now().Unix()) + VALUES ($1, 1, 11.0, -89, '["dd"]', $2)`, newTxID, time.Now().Unix()) // Add duplicate observation (same observer_id + path_json) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (?, 1, 11.0, -89, '["dd"]', ?)`, newTxID, time.Now().Unix()) + VALUES ($1, 1, 11.0, -89, '["dd"]', $2)`, newTxID, time.Now().Unix()) _, newMax := store.IngestNewFromDB(initialMax, 100) if newMax <= initialMax { @@ -3958,7 +3983,7 @@ func TestIngestNewObservations(t *testing.T) { // Simulate a new observation arriving for the existing transmission AFTER // the poller has already advanced past its transmission ID db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 2, 5.0, -100, '["aa","bb","cc"]', ?)`, time.Now().Unix()) + VALUES (1, 2, 5.0, -100, '["aa","bb","cc"]', $1)`, time.Now().Unix()) // Verify IngestNewFromDB does NOT pick up the new observation (tx id hasn't changed) txMax := store.MaxTransmissionID() @@ -3996,7 +4021,7 @@ func TestIngestNewObservations(t *testing.T) { t.Run("dedup by observer+path", func(t *testing.T) { // Insert duplicate observation (same observer + path as existing) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 12.5, -90, '["aa","bb"]', ?)`, time.Now().Unix()) + VALUES (1, 1, 12.5, -90, '["aa","bb"]', $1)`, time.Now().Unix()) prevCount := initialTx.ObservationCount maps := store.IngestNewObservations(db.GetMaxObservationID()-1, 500) if initialTx.ObservationCount != prevCount { @@ -4030,7 +4055,7 @@ func TestIngestNewObservationsV2(t *testing.T) { // Add new observation for existing transmission db.conn.Exec(`INSERT INTO observations (transmission_id, observer_id, observer_name, snr, rssi, path_json, timestamp) - VALUES (1, 'obs2', 'Obs Two', 6.0, -98, '["dd","ee"]', ?)`, time.Now().Unix()) + VALUES (1, 'obs2', 'Obs Two', 6.0, -98, '["dd","ee"]', $1)`, time.Now().Unix()) newMaps := store.IngestNewObservations(maxObsID, 500) if len(newMaps) != 1 { @@ -4082,7 +4107,7 @@ func TestHandleNodeAnalyticsNameless(t *testing.T) { // Insert a node without a name db.conn.Exec(`INSERT INTO nodes (public_key, role, lat, lon, last_seen, first_seen, advert_count) - VALUES ('nameless_node_pk_1', 'repeater', 37.5, -122.0, ?, '2026-01-01', 1)`, + VALUES ('nameless_node_pk_1', 'repeater', 37.5, -122.0, $1, '2026-01-01', 1)`, time.Now().UTC().Format(time.RFC3339)) cfg := &Config{Port: 3000} @@ -4206,8 +4231,8 @@ func TestGetDBSizeStatsMemory(t *testing.T) { seedTestData(t, db) stats := db.GetDBSizeStats() - if stats["dbSizeMB"] != float64(0) { - t.Errorf("expected 0 for in-memory, got %v", stats["dbSizeMB"]) + if stats["dbSizeMB"].(float64) < 0 || (testBackendValue() == "postgres" && stats["dbSizeMB"].(float64) == 0) { + t.Errorf("expected positive PostgreSQL database size, got %v", stats["dbSizeMB"]) } } @@ -4230,11 +4255,11 @@ func TestGetChannelMessagesAfterIngest(t *testing.T) { now := time.Now().UTC() nowStr := now.Format(time.RFC3339) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('FF01', 'newchannelmsg19800', ?, 1, 5, '{"type":"CHAN","channel":"#test","text":"NewUser: brand new message","sender":"NewUser"}')`, nowStr) + VALUES ('FF01', 'newchannelmsg19800', $1, 1, 5, '{"type":"CHAN","channel":"#test","text":"NewUser: brand new message","sender":"NewUser"}')`, nowStr) newTxID := 0 db.conn.QueryRow("SELECT MAX(id) FROM transmissions").Scan(&newTxID) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (?, 1, 12.0, -88, '[]', ?)`, newTxID, now.Unix()) + VALUES ($1, 1, 12.0, -88, '[]', $2)`, newTxID, now.Unix()) // Ingest the new data _, newMax := store.IngestNewFromDB(initialMax, 100) @@ -4528,8 +4553,8 @@ func TestBuildTransmissionWhereMultiObserver(t *testing.T) { t.Fatalf("expected 1 WHERE clause, got %d", len(where)) } clause := where[0] - if !strings.Contains(clause, "IN (?,?)") { - t.Errorf("expected IN (?,?) in clause, got: %s", clause) + if !strings.Contains(clause, "IN ("+db.parameter(1)+","+db.parameter(2)+")") { + t.Errorf("expected IN ($1,$2) in clause, got: %s", clause) } if len(args) != 2 { t.Fatalf("expected 2 args, got %d", len(args)) @@ -4556,8 +4581,8 @@ func TestBuildTransmissionWhereMultiObserver(t *testing.T) { if len(where) != 1 { t.Fatalf("expected 1 WHERE clause, got %d", len(where)) } - if !strings.Contains(where[0], "IN (?)") { - t.Errorf("expected IN (?) for single observer, got: %s", where[0]) + if !strings.Contains(where[0], "IN ("+db.parameter(1)+")") { + t.Errorf("expected IN ($1) for single observer, got: %s", where[0]) } if len(args) != 1 || args[0] != "obs1" { t.Errorf("expected [obs1], got %v", args) @@ -4581,7 +4606,7 @@ func TestDistanceIncrementalUpdate(t *testing.T) { // Insert a new observation with a different path to trigger an incremental update. maxObsID := db.GetMaxObservationID() db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 2, 5.0, -100, '["xx","yy","zz"]', ?)`, time.Now().Unix()) + VALUES (1, 2, 5.0, -100, '["xx","yy","zz"]', $1)`, time.Now().Unix()) store.IngestNewObservations(maxObsID, 500) @@ -4593,7 +4618,7 @@ func TestDistanceIncrementalUpdate(t *testing.T) { // Insert another observation with yet another path. maxObsID = db.GetMaxObservationID() db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 2, 7.0, -95, '["aa","bb","cc","dd"]', ?)`, time.Now().Unix()) + VALUES (1, 2, 7.0, -95, '["aa","bb","cc","dd"]', $1)`, time.Now().Unix()) store.IngestNewObservations(maxObsID, 500) @@ -4693,11 +4718,11 @@ func TestIngestTraceBroadcastIncludesPath(t *testing.T) { traceHex := "2502AABB010000000200000000DEADBEEF" now := time.Now().UTC().Format(time.RFC3339) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES (?, 'tracehash683test', ?, 1, 9, '')`, traceHex, now) + VALUES ($1, 'tracehash683test', $2, 1, 9, '')`, traceHex, now) newTxID := 0 db.conn.QueryRow("SELECT MAX(id) FROM transmissions").Scan(&newTxID) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (?, 1, 5.0, -100, '["aa"]', ?)`, newTxID, time.Now().Unix()) + VALUES ($1, 1, 5.0, -100, '["aa"]', $2)`, newTxID, time.Now().Unix()) broadcastMaps, _ := store.IngestNewFromDB(initialMax, 100) if len(broadcastMaps) < 1 { diff --git a/cmd/server/db.go b/cmd/server/db.go index 68ff61550..498061897 100644 --- a/cmd/server/db.go +++ b/cmd/server/db.go @@ -7,15 +7,13 @@ import ( "fmt" "log" "math" - "os" "sort" "strings" "sync" "sync/atomic" "time" - _ "github.com/mattn/go-sqlite3" - "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/dbconfig" "github.com/meshcore-analyzer/geofilter" "github.com/meshcore-analyzer/packetpath" "golang.org/x/sync/singleflight" @@ -36,13 +34,15 @@ const routeTypeTransportSQL = "route_type IN (0, 3)" // per MeshCore protocol (#1838). const routeTypeNonTransportSQL = "route_type IN (1, 2)" -// DB wraps a read-only connection to the MeshCore SQLite database. +// DB wraps a read-only connection to the selected telemetry database. type DB struct { + backend dbconfig.Backend advertEvidenceTable atomic.Bool txRewriteFeedTable atomic.Bool // tx_rewrite_feed present (#2107); re-probed while absent advertEvidenceReadHook func() // test-only: immediately before a bulk mask query conn *sql.DB - path string // filesystem path to the database file + path string // PostgreSQL URL; never log or use as a filesystem path + stateDir string // local queues and small sidecars isV3 bool // v3 schema: observer_idx in observations (vs observer_id in v2) hasResolvedPath bool // observations table has resolved_path column hasObsRawHex bool // observations table has raw_hex column (#881) @@ -59,6 +59,13 @@ type DB struct { // startup probe can lose the race. See declaredRegionsTablePresent. declaredRegionsTableLate atomic.Bool + pgStatsMu sync.Mutex + pgStatsCache postgresDatabaseSample + pgStatsExpires time.Time + pgStatsSF singleflight.Group + pgStatsMissHook func() // test-only: after an initial cache miss + pgStatsQueryHook func() // test-only: immediately before the shared sample query + // Channel list caches, keyed by region param — avoids repeated GROUP BY // scans (#762). Keyed per-region (not a single slot) so mixed-region // traffic doesn't evict and re-run the query on every request. @@ -103,10 +110,6 @@ type DB struct { stmtCountNodesByRoleAll *sql.Stmt // WHERE role = ? stmtMaxTxID *sql.Stmt // COALESCE(MAX(id), 0) FROM transmissions stmtMaxObsID *sql.Stmt // COALESCE(MAX(id), 0) FROM observations - // rowCounts caches the /api/perf table row counts (cachedRowCounts). - rowCountsMu sync.Mutex - rowCounts *SqliteRowCounts - rowCountsAt time.Time } // channelsCacheTTL is shared by the GetChannels and GetEncryptedChannels @@ -133,60 +136,14 @@ type channelMessagesCacheEntry struct { exp time.Time } -// OpenDB opens a read-only SQLite connection. -// -// The DSN used to pass _journal_mode=WAL and _busy_timeout=5000, which -// modernc.org/sqlite silently ignored: it understood only the -// _pragma=name(value) form. Under github.com/mattn/go-sqlite3 those parameters -// are honoured, and setting journal_mode on a read-only handle is a write — it -// would succeed only because the database is already WAL. Both are gone: -// mattn's own busy_timeout default is already 5000ms, so the read handle keeps -// the timeout (which it had silently lacked) without the pointless write. -// -// What remains is deliberate. mode=ro is the read-only invariant from -// #1283/#1289 and works because mattn's C wrapper ORs SQLITE_OPEN_URI into the -// open flags — see TestOpenDBRefusesMissingDatabase, which fails if that ever -// stops holding. _cache_size pins SQLite's C-allocated page cache at 2 MiB per -// connection; it sits outside GOMEMLIMIT, so it is bounded here rather than -// left to the driver's default (see memlimit.go). -func OpenDB(path string) (*DB, error) { - dsn := fmt.Sprintf("file:%s?mode=ro&_cache_size=-2000", path) - conn, err := sql.Open("sqlite3", dsn) - if err != nil { - return nil, err - } - conn.SetMaxOpenConns(4) - conn.SetMaxIdleConns(2) - if err := conn.Ping(); err != nil { - conn.Close() - return nil, fmt.Errorf("ping failed: %w", err) - } - d := &DB{conn: conn, path: path} - // Detect the on-disk schema on a single pinned connection and fail loudly if - // it cannot be probed. A swallowed detection failure would silently cache the - // wrong schema mode (isV3=false against a v3 DB) for the entire process - // lifetime, breaking every read path until a manual restart (#1901). Aborting - // here lets the supervisor restart us, which clears any transient cause (e.g. - // a WAL recovery racing the read-only open). - ctx := context.Background() - sc, err := conn.Conn(ctx) - if err != nil { - conn.Close() - return nil, fmt.Errorf("schema detection: acquire connection: %w", err) +// OpenDB preserves unambiguous legacy path/URL callers. Runtime main uses +// explicit Storage from the shared selection record. +func OpenDB(target string) (*DB, error) { + storage := dbconfig.Storage{Backend: dbconfig.SQLite, DBPath: target} + if strings.HasPrefix(target, "postgres://") || strings.HasPrefix(target, "postgresql://") { + storage = dbconfig.Storage{Backend: dbconfig.Postgres, ReaderDatabaseURL: target} } - derr := d.detectSchema(ctx, sc) - _ = sc.Close() - if derr != nil { - conn.Close() - return nil, fmt.Errorf("schema detection failed: %w", derr) - } - // Statements are prepared after schema detection so they can never be - // compiled against a schema mode that turned out to be wrong (#1901). - if err := d.prepareStatements(); err != nil { - conn.Close() - return nil, fmt.Errorf("prepare statements: %w", err) - } - return d, nil + return OpenStorage(storage) } // stmtQueryRow returns a QueryRow-like helper that uses the prepared statement @@ -200,7 +157,7 @@ func (db *DB) stmtQueryRow(stmt *sql.Stmt, fallbackSQL string, args ...interface } // prepareStatements creates prepared statements for frequently-called queries. -// SQLite compiles and caches the query plan once, avoiding re-parsing per request. +// PostgreSQL prepares these queries once per connection. func (db *DB) prepareStatements() error { var err error prepare := func(query string) *sql.Stmt { @@ -214,15 +171,15 @@ func (db *DB) prepareStatements() error { db.stmtCountTransmissions = prepare("SELECT COUNT(*) FROM transmissions") db.stmtCountObservations = prepare("SELECT COUNT(*) FROM observations") - db.stmtCountNodesActive = prepare("SELECT COUNT(*) FROM nodes WHERE last_seen > ?") + db.stmtCountNodesActive = prepare("SELECT COUNT(*) FROM nodes WHERE last_seen > " + db.parameter(1)) db.stmtCountNodesAll = prepare("SELECT COUNT(*) FROM nodes") db.stmtCountObservers = prepare("SELECT COUNT(*) FROM observers WHERE inactive IS NULL OR inactive = 0") - db.stmtCountObsLastHour = prepare("SELECT COUNT(*) FROM observations WHERE timestamp > ?") - db.stmtCountObsLastDay = prepare("SELECT COUNT(*) FROM observations WHERE timestamp > ?") - db.stmtNodeLookup = prepare("SELECT public_key FROM nodes WHERE public_key = ? OR name = ? LIMIT 1") - db.stmtTxByHash = prepare("SELECT id FROM transmissions WHERE hash = ?") - db.stmtCountNodesByRole = prepare("SELECT COUNT(*) FROM nodes WHERE role = ? AND last_seen > ?") - db.stmtCountNodesByRoleAll = prepare("SELECT COUNT(*) FROM nodes WHERE role = ?") + db.stmtCountObsLastHour = prepare("SELECT COUNT(*) FROM observations WHERE timestamp > " + db.parameter(1)) + db.stmtCountObsLastDay = prepare("SELECT COUNT(*) FROM observations WHERE timestamp > " + db.parameter(1)) + db.stmtNodeLookup = prepare("SELECT public_key FROM nodes WHERE public_key = " + db.parameter(1) + " OR name = " + db.parameter(2) + " LIMIT 1") + db.stmtTxByHash = prepare("SELECT id FROM transmissions WHERE hash = " + db.parameter(1)) + db.stmtCountNodesByRole = prepare("SELECT COUNT(*) FROM nodes WHERE role = " + db.parameter(1) + " AND last_seen > " + db.parameter(2)) + db.stmtCountNodesByRoleAll = prepare("SELECT COUNT(*) FROM nodes WHERE role = " + db.parameter(1)) db.stmtMaxTxID = prepare("SELECT COALESCE(MAX(id), 0) FROM transmissions") db.stmtMaxObsID = prepare("SELECT COALESCE(MAX(id), 0) FROM observations") return err @@ -243,11 +200,6 @@ func (db *DB) Close() error { s.Close() } } - // No WAL checkpoint here. The connection is opened read-only (mode=ro in - // OpenDB), so PRAGMA wal_checkpoint(TRUNCATE) always fails with "disk I/O - // error (778)" on it. Attempting it emitted a misleading storage-fault line - // on every shutdown that wasted incident investigation time (#1901); the - // ingestor (the writer) owns checkpointing. return db.conn.Close() } @@ -256,11 +208,11 @@ func (db *DB) Close() error { // It returns an error if any probe query fails. Previously these failures were // swallowed with a bare return, leaving isV3 (and the feature flags) at their // zero value for the whole process lifetime: a single transient failure of the -// first PRAGMA silently ran v2 SQL against a v3 database until the process was +// first schema probe silently ran v2 SQL against a v3 database until the process was // restarted (#1901). Detection is now all-or-nothing — on any probe error the // caller aborts startup so the supervisor can retry. func (db *DB) detectSchema(ctx context.Context, q rowQuerier) error { - obs, err := schemaColumns(ctx, q, "observations") + obs, err := schemaColumns(ctx, q, "observations", db.Backend()) if err != nil { return fmt.Errorf("probe observations: %w", err) } @@ -268,14 +220,14 @@ func (db *DB) detectSchema(ctx context.Context, q rowQuerier) error { db.hasResolvedPath = obs["resolved_path"] db.hasObsRawHex = obs["raw_hex"] - tx, err := schemaColumns(ctx, q, "transmissions") + tx, err := schemaColumns(ctx, q, "transmissions", db.Backend()) if err != nil { return fmt.Errorf("probe transmissions: %w", err) } db.hasScopeName = tx["scope_name"] db.hasLastSeen = tx["last_seen"] - nodes, err := schemaColumns(ctx, q, "nodes") + nodes, err := schemaColumns(ctx, q, "nodes", db.Backend()) if err != nil { return fmt.Errorf("probe nodes: %w", err) } @@ -287,7 +239,7 @@ func (db *DB) detectSchema(ctx context.Context, q rowQuerier) error { // install, so schemaColumns returns nothing and the flag stays false; // present on deployments that collect the same fact by another route. // A missing table is not an error here. - ndr, ndrErr := schemaColumns(ctx, q, "node_declared_regions") + ndr, ndrErr := schemaColumns(ctx, q, "node_declared_regions", db.Backend()) db.hasDeclaredRegionsTable = ndrErr == nil && len(ndr) > 0 if db.isV3 { @@ -305,24 +257,22 @@ type rowQuerier interface { QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) } -// schemaColumns returns the set of column names present on the given table via -// PRAGMA table_info. Unlike the previous inline scans, a query or scan error is -// returned rather than swallowed (#1901). The table name is a trusted literal -// supplied by the caller, not user input. -func schemaColumns(ctx context.Context, q rowQuerier, table string) (map[string]bool, error) { - rows, err := q.QueryContext(ctx, fmt.Sprintf("PRAGMA table_info(%s)", table)) +// schemaColumns reads visible columns through the selected native catalog. +func schemaColumns(ctx context.Context, q rowQuerier, table string, backend ...dbconfig.Backend) (map[string]bool, error) { + query := `SELECT column_name FROM information_schema.columns + WHERE table_schema = current_schema() AND table_name = $1` + if len(backend) > 0 && backend[0] == dbconfig.SQLite { + query = `SELECT name FROM pragma_table_info(?1)` + } + rows, err := q.QueryContext(ctx, query, table) if err != nil { return nil, err } defer rows.Close() cols := make(map[string]bool) for rows.Next() { - var cid int var name string - var ctype sql.NullString - var notnull, pk int - var dflt sql.NullString - if err := rows.Scan(&cid, &name, &ctype, ¬null, &dflt, &pk); err != nil { + if err := rows.Scan(&name); err != nil { return nil, err } cols[name] = true @@ -353,7 +303,7 @@ func (db *DB) transmissionBaseSQL() (selectCols, observerJoin string) { o.snr, o.rssi, o.path_json, o.direction` observerJoin = `LEFT JOIN observations o ON o.id = ( SELECT id FROM observations WHERE transmission_id = t.id - ORDER BY length(COALESCE(path_json,'')) DESC LIMIT 1 + ORDER BY length(COALESCE(path_json,'')) DESC NULLS LAST LIMIT 1 ) LEFT JOIN observers obs ON obs.rowid = o.observer_idx` } else { @@ -363,7 +313,7 @@ func (db *DB) transmissionBaseSQL() (selectCols, observerJoin string) { o.snr, o.rssi, o.path_json, o.direction` observerJoin = `LEFT JOIN observations o ON o.id = ( SELECT id FROM observations WHERE transmission_id = t.id - ORDER BY length(COALESCE(path_json,'')) DESC LIMIT 1 + ORDER BY length(COALESCE(path_json,'')) DESC NULLS LAST LIMIT 1 ) LEFT JOIN observers obs2 ON obs2.id = o.observer_id` } @@ -485,7 +435,7 @@ type Observation struct { Direction *string `json:"direction"` SNR *float64 `json:"snr"` RSSI *float64 `json:"rssi"` - Score *int `json:"score"` + Score *float64 `json:"score"` Hash *string `json:"hash"` RouteType *int `json:"route_type"` PayloadType *int `json:"payload_type"` @@ -513,7 +463,7 @@ func (db *DB) CountActiveNodes() (int, error) { // Node.js uses 7-day active nodes for totalNodes sevenDaysAgo := time.Now().Add(-7 * 24 * time.Hour).Format(time.RFC3339) var n int - err := db.stmtQueryRow(db.stmtCountNodesActive, "SELECT COUNT(*) FROM nodes WHERE last_seen > ?", sevenDaysAgo).Scan(&n) + err := db.stmtQueryRow(db.stmtCountNodesActive, "SELECT COUNT(*) FROM nodes WHERE last_seen > "+db.parameter(1), sevenDaysAgo).Scan(&n) return n, err } @@ -532,146 +482,60 @@ func (db *DB) GetStats() (*Stats, error) { db.stmtQueryRow(db.stmtCountObservers, "SELECT COUNT(*) FROM observers WHERE inactive IS NULL OR inactive = 0").Scan(&s.TotalObservers) oneHourAgo := time.Now().Add(-1 * time.Hour).Unix() - db.stmtQueryRow(db.stmtCountObsLastHour, "SELECT COUNT(*) FROM observations WHERE timestamp > ?", oneHourAgo).Scan(&s.PacketsLastHour) + db.stmtQueryRow(db.stmtCountObsLastHour, "SELECT COUNT(*) FROM observations WHERE timestamp > "+db.parameter(1), oneHourAgo).Scan(&s.PacketsLastHour) oneDayAgo := time.Now().Add(-24 * time.Hour).Unix() - db.stmtQueryRow(db.stmtCountObsLastDay, "SELECT COUNT(*) FROM observations WHERE timestamp > ?", oneDayAgo).Scan(&s.PacketsLast24h) + db.stmtQueryRow(db.stmtCountObsLastDay, "SELECT COUNT(*) FROM observations WHERE timestamp > "+db.parameter(1), oneDayAgo).Scan(&s.PacketsLast24h) return s, nil } -// GetDBSizeStats returns SQLite file sizes and row counts (matching Node.js /api/perf sqlite shape). +// GetDBSizeStats retains the legacy map caller with native database metrics. func (db *DB) GetDBSizeStats() map[string]interface{} { - result := map[string]interface{}{} - - // DB file size - var dbSizeMB float64 - if db.path != "" && db.path != ":memory:" { - if info, err := os.Stat(db.path); err == nil { - dbSizeMB = math.Round(float64(info.Size())/1048576*10) / 10 - } + stats := db.GetDBSizeStatsTyped() + var size any + var rows map[string]int + if stats.DbSizeMB != nil { + size = *stats.DbSizeMB } - result["dbSizeMB"] = dbSizeMB - - // WAL file size - var walSizeMB float64 - if db.path != "" && db.path != ":memory:" { - if info, err := os.Stat(db.path + "-wal"); err == nil { - walSizeMB = math.Round(float64(info.Size())/1048576*10) / 10 - } + if stats.Rows != nil { + rows = map[string]int{"transmissions": stats.Rows.Transmissions, + "observations": stats.Rows.Observations, "nodes": stats.Rows.Nodes, + "observers": stats.Rows.Observers} } - result["walSizeMB"] = walSizeMB - - // Freelist size via PRAGMA (matches Node.js: page_size * freelist_count) - var pageSize, freelistCount int64 - db.conn.QueryRow("PRAGMA page_size").Scan(&pageSize) - db.conn.QueryRow("PRAGMA freelist_count").Scan(&freelistCount) - freelistMB := math.Round(float64(pageSize*freelistCount)/1048576*10) / 10 - result["freelistMB"] = freelistMB - - // WAL checkpoint info (matches Node.js: PRAGMA wal_checkpoint(PASSIVE)) - var walBusy, walLog, walCheckpointed int - err := db.conn.QueryRow("PRAGMA wal_checkpoint(PASSIVE)").Scan(&walBusy, &walLog, &walCheckpointed) - if err == nil { - result["walPages"] = map[string]interface{}{ - "total": walLog, - "checkpointed": walCheckpointed, - "busy": walBusy, - } - } else { - result["walPages"] = map[string]interface{}{ - "total": 0, - "checkpointed": 0, - "busy": 0, - } + result := map[string]interface{}{ + "engine": stats.Engine, "dbSizeMB": size, "rows": rows, "error": stats.Error, } - - // Row counts per table - rows := map[string]int{} - for _, table := range []string{"transmissions", "observations", "nodes", "observers"} { - var count int - db.conn.QueryRow("SELECT COUNT(*) FROM " + table).Scan(&count) - rows[table] = count + if stats.Engine == "sqlite" { + result["walSizeMB"] = stats.WalSizeMB + result["freelistMB"] = stats.FreelistMB } - result["rows"] = rows - return result } -// GetDBSizeStatsTyped returns SQLite file sizes and row counts as a typed struct. -func (db *DB) GetDBSizeStatsTyped() SqliteStats { - result := SqliteStats{} - - if db.path != "" && db.path != ":memory:" { - if info, err := os.Stat(db.path); err == nil { - result.DbSizeMB = math.Round(float64(info.Size())/1048576*10) / 10 - } - } - - if db.path != "" && db.path != ":memory:" { - if info, err := os.Stat(db.path + "-wal"); err == nil { - result.WalSizeMB = math.Round(float64(info.Size())/1048576*10) / 10 - } +// GetDBSizeStatsTyped reports native database size and table counts. +// WAL is cluster-wide and cannot be attributed to this database by a reader. +func (db *DB) GetDBSizeStatsTyped() PostgresStats { + sample, err := db.postgresStats(context.Background()) + result := PostgresStats{ + Engine: "postgresql", SampledAt: postgresSampleStamp(sample), + SampleIntervalSeconds: int(postgresStatsTTL / time.Second), Stale: err != nil, } - - var pageSize, freelistCount int64 - db.conn.QueryRow("PRAGMA page_size").Scan(&pageSize) - db.conn.QueryRow("PRAGMA freelist_count").Scan(&freelistCount) - result.FreelistMB = math.Round(float64(pageSize*freelistCount)/1048576*10) / 10 - - var walBusy, walLog, walCheckpointed int - err := db.conn.QueryRow("PRAGMA wal_checkpoint(PASSIVE)").Scan(&walBusy, &walLog, &walCheckpointed) - if err == nil { - result.WalPages = &WalPages{ - Total: walLog, - Checkpointed: walCheckpointed, - Busy: walBusy, - } - } else { - result.WalPages = &WalPages{} + if db.Backend() == dbconfig.SQLite { + result.Engine = "sqlite" } - - result.Rows = db.cachedRowCounts() - - return result -} - -// perfRowCountsTTL is how long /api/perf serves its table row counts before -// counting again. -const perfRowCountsTTL = 60 * time.Second - -// cachedRowCounts returns the row counts of the four main tables, counted -// at most once per perfRowCountsTTL: a COUNT(*) reads a whole index, 116 ms -// for observations on an 18M-row database (#2146). -func (db *DB) cachedRowCounts() *SqliteRowCounts { - db.rowCountsMu.Lock() - defer db.rowCountsMu.Unlock() - if db.rowCounts != nil && time.Since(db.rowCountsAt) < perfRowCountsTTL { - return db.rowCounts - } - rows := &SqliteRowCounts{} - failed := false - for _, table := range []string{"transmissions", "observations", "nodes", "observers"} { - var count int - if err := db.conn.QueryRow("SELECT COUNT(*) FROM " + table).Scan(&count); err != nil { - failed = true - } - switch table { - case "transmissions": - rows.Transmissions = count - case "observations": - rows.Observations = count - case "nodes": - rows.Nodes = count - case "observers": - rows.Observers = count - } + if sample.SampledAt.IsZero() { + result.Error = "database diagnostics unavailable" + return result } - // A failed count is served once, as before, but not cached. - if !failed { - db.rowCounts, db.rowCountsAt = rows, time.Now() + mb := math.Round(float64(sample.DatabaseBytes)/1048576*10) / 10 + result.DbSizeMB, result.Rows = &mb, &sample.Rows + if sample.SQLite != nil { + walMB := float64(sample.SQLite.WalSize) / 1048576 + freeMB := float64(sample.SQLite.FreelistCount*sample.SQLite.PageSize) / 1048576 + result.WalSizeMB, result.FreelistMB = &walMB, &freeMB } - return rows + return result } // GetRoleCounts returns count per role (7-day active, matching Node.js /api/stats). @@ -680,7 +544,7 @@ func (db *DB) GetRoleCounts() map[string]int { counts := map[string]int{} for _, role := range []string{"repeater", "room", "companion", "sensor"} { var c int - db.stmtQueryRow(db.stmtCountNodesByRole, "SELECT COUNT(*) FROM nodes WHERE role = ? AND last_seen > ?", role, sevenDaysAgo).Scan(&c) + db.stmtQueryRow(db.stmtCountNodesByRole, "SELECT COUNT(*) FROM nodes WHERE role = "+db.parameter(1)+" AND last_seen > "+db.parameter(2), role, sevenDaysAgo).Scan(&c) counts[role+"s"] = c } return counts @@ -691,7 +555,7 @@ func (db *DB) GetAllRoleCounts() map[string]int { counts := map[string]int{} for _, role := range []string{"repeater", "room", "companion", "sensor"} { var c int - db.stmtQueryRow(db.stmtCountNodesByRoleAll, "SELECT COUNT(*) FROM nodes WHERE role = ?", role).Scan(&c) + db.stmtQueryRow(db.stmtCountNodesByRoleAll, "SELECT COUNT(*) FROM nodes WHERE role = "+db.parameter(1), role).Scan(&c) counts[role+"s"] = c } return counts @@ -725,6 +589,7 @@ type PacketResult struct { // QueryPackets returns paginated, filtered packets as transmissions (matching Node.js shape). func (db *DB) QueryPackets(q PacketQuery) (*PacketResult, error) { + q.Offset = max(0, q.Offset) if q.Limit <= 0 { q.Limit = 50 } @@ -754,8 +619,8 @@ func (db *DB) QueryPackets(q PacketQuery) (*PacketResult, error) { // is what the packets page is showing. The `since=` filter still uses // first_seen / observation timestamp, preserving "received-by-radio since X." selectCols, observerJoin := db.transmissionBaseSQL() - querySQL := fmt.Sprintf("SELECT %s FROM transmissions t %s %s ORDER BY t.id %s LIMIT ? OFFSET ?", - selectCols, observerJoin, w, q.Order) + querySQL := fmt.Sprintf("SELECT %s FROM transmissions t %s %s ORDER BY t.id %s LIMIT %s OFFSET %s", + selectCols, observerJoin, w, q.Order, db.parameter(len(args)+1), db.parameter(len(args)+2)) qArgs := make([]interface{}, len(args)) copy(qArgs, args) @@ -780,6 +645,7 @@ func (db *DB) QueryPackets(q PacketQuery) (*PacketResult, error) { // QueryGroupedPackets groups by hash (transmissions) — queries transmissions table directly for performance. func (db *DB) QueryGroupedPackets(q PacketQuery) (*PacketResult, error) { + q.Offset = max(0, q.Offset) if q.Limit <= 0 { q.Limit = 50 } @@ -798,50 +664,7 @@ func (db *DB) QueryGroupedPackets(q PacketQuery) (*PacketResult, error) { db.conn.QueryRow(fmt.Sprintf("SELECT COUNT(*) FROM transmissions t %s", w), args...).Scan(&total) } - // Build grouped query using transmissions table with correlated subqueries. - // #1189 R2: distinct_iatas is a NEW column — comma-separated DISTINCT IATA - // codes across all observers of the transmission, with empty/NULL IATAs - // excluded. Frontend needs this on the DEFAULT COLLAPSED VIEW (where - // p._children is empty), so we compute it server-side. - // - // scope_name lives on the transmission row, so appending it as the last - // selected column is safe for both query shapes. - scopeNameCol := "" - if db.hasScopeName { - scopeNameCol = ", t.scope_name" - } - var querySQL string - if db.isV3 { - querySQL = fmt.Sprintf(`SELECT t.hash, t.first_seen, t.raw_hex, t.decoded_json, t.payload_type, t.route_type, - COALESCE((SELECT COUNT(*) FROM observations oi WHERE oi.transmission_id = t.id), 0) AS count, - COALESCE((SELECT COUNT(DISTINCT oi.observer_idx) FROM observations oi WHERE oi.transmission_id = t.id), 0) AS observer_count, - COALESCE((SELECT MAX(strftime('%%Y-%%m-%%dT%%H:%%M:%%fZ', oi.timestamp, 'unixepoch')) FROM observations oi WHERE oi.transmission_id = t.id), t.first_seen) AS latest, - obs.id AS observer_id, obs.name AS observer_name, COALESCE(obs.iata, '') AS observer_iata, - o.snr, o.rssi, o.path_json, - COALESCE((SELECT GROUP_CONCAT(DISTINCT obi.iata) FROM observations oi JOIN observers obi ON obi.rowid = oi.observer_idx WHERE oi.transmission_id = t.id AND obi.iata IS NOT NULL AND obi.iata != ''), '') AS distinct_iatas`+scopeNameCol+` - FROM transmissions t - LEFT JOIN observations o ON o.id = ( - SELECT id FROM observations WHERE transmission_id = t.id - ORDER BY length(COALESCE(path_json,'')) DESC LIMIT 1 - ) - LEFT JOIN observers obs ON obs.rowid = o.observer_idx - %s ORDER BY latest DESC LIMIT ? OFFSET ?`, w) - } else { - querySQL = fmt.Sprintf(`SELECT t.hash, t.first_seen, t.raw_hex, t.decoded_json, t.payload_type, t.route_type, - COALESCE((SELECT COUNT(*) FROM observations oi WHERE oi.transmission_id = t.id), 0) AS count, - COALESCE((SELECT COUNT(DISTINCT oi.observer_id) FROM observations oi WHERE oi.transmission_id = t.id), 0) AS observer_count, - COALESCE((SELECT MAX(oi.timestamp) FROM observations oi WHERE oi.transmission_id = t.id), t.first_seen) AS latest, - o.observer_id, o.observer_name, COALESCE(obs2.iata, '') AS observer_iata, - o.snr, o.rssi, o.path_json, - COALESCE((SELECT GROUP_CONCAT(DISTINCT obi.iata) FROM observations oi JOIN observers obi ON obi.id = oi.observer_id WHERE oi.transmission_id = t.id AND obi.iata IS NOT NULL AND obi.iata != ''), '') AS distinct_iatas`+scopeNameCol+` - FROM transmissions t - LEFT JOIN observations o ON o.id = ( - SELECT id FROM observations WHERE transmission_id = t.id - ORDER BY length(COALESCE(path_json,'')) DESC LIMIT 1 - ) - LEFT JOIN observers obs2 ON obs2.id = o.observer_id - %s ORDER BY latest DESC LIMIT ? OFFSET ?`, w) - } + querySQL := db.groupedPacketSelectSQL(w, len(args)) qArgs := make([]interface{}, len(args)) copy(qArgs, args) @@ -896,7 +719,55 @@ func (db *DB) QueryGroupedPackets(q PacketQuery) (*PacketResult, error) { return &PacketResult{Packets: packets, Total: total}, nil } -// parseDistinctIatasCSV turns SQLite GROUP_CONCAT output ("SJC,SFO,OAK") into +// groupedPacketSelectSQL ranks and pages matching transmissions before loading +// their observation details. The timestamp index can answer numeric MAX with +// one backwards lookup; formatting every observation before MAX prevents that. +func (db *DB) groupedPacketSelectSQL(w string, argumentCount int) string { + observerKey := "observer_id" + observerID, observerName := "o.observer_id", "o.observer_name" + observerJoin := "obs.id = o.observer_id" + iataJoin := "obi.id = oi.observer_id" + latest := `COALESCE((SELECT CAST(MAX(oi.timestamp) AS TEXT) FROM observations oi WHERE oi.transmission_id = t.id), t.first_seen)` + if db.isV3 { + observerKey = "observer_idx" + observerID, observerName = "obs.id", "obs.name" + observerJoin = "obs.rowid = o.observer_idx" + iataJoin = "obi.rowid = oi.observer_idx" + latest = db.nativeSQL( + `COALESCE((SELECT strftime('%Y-%m-%dT%H:%M:%fZ', MAX(oi.timestamp), 'unixepoch') FROM observations oi WHERE oi.transmission_id = t.id), t.first_seen)`, + `COALESCE((SELECT to_char(to_timestamp(MAX(oi.timestamp)) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"') FROM observations oi WHERE oi.transmission_id = t.id), t.first_seen)`) + } + // Keep latest independent of the observation filters: those decide which + // transmissions match, while the returned group describes all observations. + // LIMIT inside the derived table prevents both engines from joining/sorting + // best-path observations for transmissions outside the requested page. + page := fmt.Sprintf(`SELECT t.id, %s AS latest FROM transmissions t %s + ORDER BY latest DESC NULLS LAST, t.id DESC LIMIT %s OFFSET %s`, + latest, w, db.parameter(argumentCount+1), db.parameter(argumentCount+2)) + iatas := db.nativeSQL("group_concat(DISTINCT obi.iata)", "string_agg(DISTINCT obi.iata, ',')") + scopeNameCol := "" + if db.hasScopeName { + scopeNameCol = ", t.scope_name" + } + return fmt.Sprintf(`SELECT t.hash, t.first_seen, t.raw_hex, t.decoded_json, t.payload_type, t.route_type, + COALESCE((SELECT COUNT(*) FROM observations oi WHERE oi.transmission_id = t.id), 0) AS count, + COALESCE((SELECT COUNT(DISTINCT oi.%s) FROM observations oi WHERE oi.transmission_id = t.id), 0) AS observer_count, + page.latest, %s AS observer_id, %s AS observer_name, COALESCE(obs.iata, '') AS observer_iata, + o.snr, o.rssi, o.path_json, + COALESCE((SELECT %s FROM observations oi JOIN observers obi ON %s + WHERE oi.transmission_id = t.id AND obi.iata IS NOT NULL AND obi.iata != ''), '') AS distinct_iatas%s + FROM (%s) AS page + JOIN transmissions t ON t.id = page.id + LEFT JOIN observations o ON o.id = ( + SELECT id FROM observations WHERE transmission_id = t.id + ORDER BY length(COALESCE(path_json,'')) DESC NULLS LAST LIMIT 1 + ) + LEFT JOIN observers obs ON %s + ORDER BY page.latest DESC NULLS LAST, t.id DESC`, + observerKey, observerID, observerName, iatas, iataJoin, scopeNameCol, page, observerJoin) +} + +// parseDistinctIatasCSV turns PostgreSQL string_agg output ("SJC,SFO,OAK") into // a sorted, deduped []string. Returns an empty (non-nil) slice when the input // is empty/nil so JSON serialization stays consistent (`[]` not `null`). func parseDistinctIatasCSV(v interface{}) []string { @@ -924,38 +795,38 @@ func (db *DB) buildPacketWhere(q PacketQuery) ([]string, []interface{}) { var args []interface{} if q.Type != nil { - where = append(where, "payload_type = ?") + where = append(where, fmt.Sprintf("payload_type = %s", db.parameter(len(args)+1))) args = append(args, *q.Type) } if q.Route != nil { - where = append(where, "route_type = ?") + where = append(where, fmt.Sprintf("route_type = %s", db.parameter(len(args)+1))) args = append(args, *q.Route) } if q.Observer != "" { - where = append(where, "observer_id = ?") + where = append(where, fmt.Sprintf("observer_id = %s", db.parameter(len(args)+1))) args = append(args, q.Observer) } if q.Hash != "" { - where = append(where, "hash = ?") + where = append(where, fmt.Sprintf("hash = %s", db.parameter(len(args)+1))) args = append(args, strings.ToLower(q.Hash)) } if q.Since != "" { - where = append(where, "timestamp > ?") + where = append(where, fmt.Sprintf("timestamp > %s", db.parameter(len(args)+1))) args = append(args, q.Since) } if q.Until != "" { - where = append(where, "timestamp < ?") + where = append(where, fmt.Sprintf("timestamp < %s", db.parameter(len(args)+1))) args = append(args, q.Until) } if q.Region != "" { - where = append(where, "observer_id IN (SELECT id FROM observers WHERE iata = ?)") + where = append(where, fmt.Sprintf("observer_id IN (SELECT id FROM observers WHERE iata = %s)", db.parameter(len(args)+1))) args = append(args, q.Region) } if q.Node != "" { pk := db.resolveNodePubkey(q.Node) // #1143: exact-match on the dedicated from_pubkey column instead of // LIKE-on-JSON substring (adversarial spoof + same-name false positives). - where = append(where, "from_pubkey = ?") + where = append(where, fmt.Sprintf("from_pubkey = %s", db.parameter(len(args)+1))) args = append(args, pk) } return where, args @@ -968,15 +839,15 @@ func (db *DB) buildTransmissionWhere(q PacketQuery) ([]string, []interface{}) { var args []interface{} if q.Type != nil { - where = append(where, "t.payload_type = ?") + where = append(where, fmt.Sprintf("t.payload_type = %s", db.parameter(len(args)+1))) args = append(args, *q.Type) } if q.Route != nil { - where = append(where, "t.route_type = ?") + where = append(where, fmt.Sprintf("t.route_type = %s", db.parameter(len(args)+1))) args = append(args, *q.Route) } if q.Hash != "" { - where = append(where, "t.hash = ?") + where = append(where, fmt.Sprintf("t.hash = %s", db.parameter(len(args)+1))) args = append(args, strings.ToLower(q.Hash)) } if q.Since != "" { @@ -985,37 +856,36 @@ func (db *DB) buildTransmissionWhere(q PacketQuery) ([]string, []interface{}) { // but which have observations inside the window) are still included. // Non-RFC3339 falls back to t.first_seen string compare. if ts, err := time.Parse(time.RFC3339Nano, q.Since); err == nil { - where = append(where, "t.id IN (SELECT DISTINCT transmission_id FROM observations WHERE timestamp >= ?)") + where = append(where, fmt.Sprintf("t.id IN (SELECT DISTINCT transmission_id FROM observations WHERE timestamp >= %s)", db.parameter(len(args)+1))) args = append(args, ts.Unix()) } else { - where = append(where, "t.first_seen > ?") + where = append(where, fmt.Sprintf("t.first_seen > %s", db.parameter(len(args)+1))) args = append(args, q.Since) } } if q.Until != "" { if ts, err := time.Parse(time.RFC3339Nano, q.Until); err == nil { - where = append(where, "t.id IN (SELECT DISTINCT transmission_id FROM observations WHERE timestamp <= ?)") + where = append(where, fmt.Sprintf("t.id IN (SELECT DISTINCT transmission_id FROM observations WHERE timestamp <= %s)", db.parameter(len(args)+1))) args = append(args, ts.Unix()) } else { - where = append(where, "t.first_seen < ?") + where = append(where, fmt.Sprintf("t.first_seen < %s", db.parameter(len(args)+1))) args = append(args, q.Until) } } if q.Node != "" { pk := db.resolveNodePubkey(q.Node) // #1143: exact-match on dedicated from_pubkey column. - where = append(where, "t.from_pubkey = ?") + where = append(where, fmt.Sprintf("t.from_pubkey = %s", db.parameter(len(args)+1))) args = append(args, pk) } if q.Channel != "" { // channel_hash column is indexed for payload_type = 5; filter is exact match. - where = append(where, "t.channel_hash = ?") + where = append(where, fmt.Sprintf("t.channel_hash = %s", db.parameter(len(args)+1))) args = append(args, q.Channel) } if q.Observer != "" { ids := strings.Split(q.Observer, ",") - placeholders := strings.Repeat("?,", len(ids)) - placeholders = placeholders[:len(placeholders)-1] + placeholders := db.sqlPlaceholders(len(ids), len(args)+1) if db.isV3 { where = append(where, "EXISTS (SELECT 1 FROM observations oi JOIN observers obi ON obi.rowid = oi.observer_idx WHERE oi.transmission_id = t.id AND obi.id IN ("+placeholders+"))") } else { @@ -1027,9 +897,9 @@ func (db *DB) buildTransmissionWhere(q PacketQuery) ([]string, []interface{}) { } if q.Region != "" { if db.isV3 { - where = append(where, "EXISTS (SELECT 1 FROM observations oi JOIN observers obi ON obi.rowid = oi.observer_idx WHERE oi.transmission_id = t.id AND obi.iata = ?)") + where = append(where, fmt.Sprintf("EXISTS (SELECT 1 FROM observations oi JOIN observers obi ON obi.rowid = oi.observer_idx WHERE oi.transmission_id = t.id AND obi.iata = %s)", db.parameter(len(args)+1))) } else { - where = append(where, "EXISTS (SELECT 1 FROM observations oi JOIN observers obi ON obi.id = oi.observer_id WHERE oi.transmission_id = t.id AND obi.iata = ?)") + where = append(where, fmt.Sprintf("EXISTS (SELECT 1 FROM observations oi JOIN observers obi ON obi.id = oi.observer_id WHERE oi.transmission_id = t.id AND obi.iata = %s)", db.parameter(len(args)+1))) } args = append(args, q.Region) } @@ -1038,7 +908,7 @@ func (db *DB) buildTransmissionWhere(q PacketQuery) ([]string, []interface{}) { func (db *DB) resolveNodePubkey(nodeIDOrName string) string { var pk string - err := db.stmtQueryRow(db.stmtNodeLookup, "SELECT public_key FROM nodes WHERE public_key = ? OR name = ? LIMIT 1", nodeIDOrName, nodeIDOrName).Scan(&pk) + err := db.stmtQueryRow(db.stmtNodeLookup, "SELECT public_key FROM nodes WHERE public_key = "+db.parameter(1)+" OR name = "+db.parameter(2)+" LIMIT 1", nodeIDOrName, nodeIDOrName).Scan(&pk) if err != nil { return nodeIDOrName } @@ -1048,7 +918,7 @@ func (db *DB) resolveNodePubkey(nodeIDOrName string) string { // GetTransmissionByID fetches from transmissions table with observer data. func (db *DB) GetTransmissionByID(id int) (map[string]interface{}, error) { selectCols, observerJoin := db.transmissionBaseSQL() - querySQL := fmt.Sprintf("SELECT %s FROM transmissions t %s WHERE t.id = ?", selectCols, observerJoin) + querySQL := fmt.Sprintf("SELECT %s FROM transmissions t %s WHERE t.id = "+db.parameter(1), selectCols, observerJoin) rows, err := db.conn.Query(querySQL, id) if err != nil { @@ -1064,7 +934,7 @@ func (db *DB) GetTransmissionByID(id int) (map[string]interface{}, error) { // GetPacketByHash fetches a transmission by content hash with observer data. func (db *DB) GetPacketByHash(hash string) (map[string]interface{}, error) { selectCols, observerJoin := db.transmissionBaseSQL() - querySQL := fmt.Sprintf("SELECT %s FROM transmissions t %s WHERE t.hash = ?", selectCols, observerJoin) + querySQL := fmt.Sprintf("SELECT %s FROM transmissions t %s WHERE t.hash = "+db.parameter(1), selectCols, observerJoin) rows, err := db.conn.Query(querySQL, strings.ToLower(hash)) if err != nil { @@ -1082,7 +952,7 @@ func (db *DB) GetPacketByHash(hash string) (map[string]interface{}, error) { // when the in-memory PacketStore has pruned the entry but the DB still has it. func (db *DB) GetObservationsForHash(hash string) []map[string]interface{} { var txID int - err := db.stmtQueryRow(db.stmtTxByHash, "SELECT id FROM transmissions WHERE hash = ?", strings.ToLower(hash)).Scan(&txID) + err := db.stmtQueryRow(db.stmtTxByHash, "SELECT id FROM transmissions WHERE hash = "+db.parameter(1), strings.ToLower(hash)).Scan(&txID) if err != nil { return nil } @@ -1112,13 +982,13 @@ func (db *DB) ObservationRawHexForHash(hash string) map[int]string { return nil } var txID int - if err := db.stmtQueryRow(db.stmtTxByHash, "SELECT id FROM transmissions WHERE hash = ?", + if err := db.stmtQueryRow(db.stmtTxByHash, "SELECT id FROM transmissions WHERE hash = "+db.parameter(1), strings.ToLower(hash)).Scan(&txID); err != nil { return nil } rows, err := db.conn.Query( `SELECT id, raw_hex FROM observations - WHERE transmission_id = ? AND raw_hex IS NOT NULL AND raw_hex <> ''`, txID) + WHERE transmission_id = `+db.parameter(1)+` AND raw_hex IS NOT NULL AND raw_hex <> ''`, txID) if err != nil { return nil } @@ -1158,7 +1028,7 @@ type NodeQuery struct { // GetNodes returns filtered, paginated node list. func (db *DB) GetNodes(nq NodeQuery) ([]map[string]interface{}, int, map[string]int, error) { - limit, offset := nq.Limit, nq.Offset + limit, offset := nq.Limit, max(0, nq.Offset) role, search, before, lastHeard, sortBy, region := nq.Role, nq.Search, nq.Before, nq.LastHeard, nq.SortBy, nq.Region if nq.RegionPubkeys != nil { region = "" @@ -1167,15 +1037,15 @@ func (db *DB) GetNodes(nq NodeQuery) ([]map[string]interface{}, int, map[string] var args []interface{} if role != "" { - where = append(where, "role = ?") + where = append(where, fmt.Sprintf("role = %s", db.parameter(len(args)+1))) args = append(args, role) } if search != "" { - where = append(where, "name LIKE ?") + where = append(where, fmt.Sprintf(db.nativeSQL("name LIKE %s", "name ILIKE %s ESCAPE ''"), db.parameter(len(args)+1))) args = append(args, "%"+search+"%") } if before != "" { - where = append(where, "first_seen <= ?") + where = append(where, fmt.Sprintf("first_seen <= %s", db.parameter(len(args)+1))) args = append(args, before) } if lastHeard != "" { @@ -1185,7 +1055,7 @@ func (db *DB) GetNodes(nq NodeQuery) ([]map[string]interface{}, int, map[string] } if ms, ok := durations[lastHeard]; ok { since := time.Now().Add(-time.Duration(ms) * time.Millisecond).Format(time.RFC3339) - where = append(where, "last_seen > ?") + where = append(where, fmt.Sprintf("last_seen > %s", db.parameter(len(args)+1))) args = append(args, since) } } @@ -1196,7 +1066,7 @@ func (db *DB) GetNodes(nq NodeQuery) ([]map[string]interface{}, int, map[string] placeholders := make([]string, len(codes)) regionArgs := make([]interface{}, len(codes)) for i, c := range codes { - placeholders[i] = "?" + placeholders[i] = fmt.Sprintf("%s", db.parameter(len(args)+i+1)) regionArgs[i] = c } joinCond := "obs.rowid = o.observer_idx" @@ -1220,7 +1090,7 @@ func (db *DB) GetNodes(nq NodeQuery) ([]map[string]interface{}, int, map[string] } if nq.RegionPubkeys != nil { if len(nq.RegionPubkeys) == 0 { - where = append(where, "0") + where = append(where, "FALSE") } else { // One JSON parameter rather than one placeholder per key: a region // can hold thousands of nodes, and the IN list stays a primary-key @@ -1229,7 +1099,7 @@ func (db *DB) GetNodes(nq NodeQuery) ([]map[string]interface{}, int, map[string] if err != nil { return nil, 0, nil, err } - where = append(where, "public_key IN (SELECT value FROM json_each(?))") + where = append(where, fmt.Sprintf(db.nativeSQL("public_key IN (SELECT value FROM json_each(%s))", "public_key IN (SELECT value FROM json_array_elements_text(%s::json))"), db.parameter(len(args)+1))) args = append(args, string(keysJSON)) } } @@ -1240,9 +1110,9 @@ func (db *DB) GetNodes(nq NodeQuery) ([]map[string]interface{}, int, map[string] } sortMap := map[string]string{ - "name": "name ASC", "lastSeen": "last_seen DESC", "packetCount": "advert_count DESC", + "name": "name ASC NULLS FIRST", "lastSeen": "last_seen DESC NULLS LAST", "packetCount": "advert_count DESC NULLS LAST", } - order := "last_seen DESC" + order := "last_seen DESC NULLS LAST" if s, ok := sortMap[sortBy]; ok { order = s } @@ -1254,7 +1124,7 @@ func (db *DB) GetNodes(nq NodeQuery) ([]map[string]interface{}, int, map[string] var total int db.conn.QueryRow(fmt.Sprintf("SELECT COUNT(*) FROM nodes %s", w), args...).Scan(&total) - querySQL := fmt.Sprintf("SELECT %s FROM nodes %s ORDER BY %s LIMIT ? OFFSET ?", db.nodeSelectCols(), w, order) + querySQL := fmt.Sprintf("SELECT %s FROM nodes %s ORDER BY %s LIMIT %s OFFSET %s", db.nodeSelectCols(), w, order, db.parameter(len(args)+1), db.parameter(len(args)+2)) qArgs := append(args, limit, offset) rows, err := db.conn.Query(querySQL, qArgs...) @@ -1280,7 +1150,7 @@ func (db *DB) SearchNodes(query string, limit int) ([]map[string]interface{}, er if limit <= 0 { limit = 10 } - rows, err := db.conn.Query(fmt.Sprintf("SELECT %s FROM nodes WHERE name LIKE ? OR public_key LIKE ? ORDER BY last_seen DESC LIMIT ?", db.nodeSelectCols()), + rows, err := db.conn.Query(fmt.Sprintf(db.nativeSQL("SELECT %s FROM nodes WHERE name LIKE ?1 OR public_key LIKE ?2 ORDER BY last_seen DESC NULLS LAST LIMIT ?3", "SELECT %s FROM nodes WHERE name ILIKE "+db.parameter(1)+" ESCAPE '' OR public_key ILIKE "+db.parameter(2)+" ESCAPE '' ORDER BY last_seen DESC NULLS LAST LIMIT "+db.parameter(3)), db.nodeSelectCols()), "%"+query+"%", query+"%", limit) if err != nil { return nil, err @@ -1318,7 +1188,7 @@ func (db *DB) GetNodeByPrefix(prefix string) (map[string]interface{}, bool, erro } } rows, err := db.conn.Query( - fmt.Sprintf("SELECT %s FROM nodes WHERE public_key LIKE ? LIMIT 2", db.nodeSelectCols()), + fmt.Sprintf(db.nativeSQL("SELECT %s FROM nodes WHERE public_key LIKE ?1 LIMIT 2", "SELECT %s FROM nodes WHERE public_key ILIKE "+db.parameter(1)+" ESCAPE '' LIMIT 2"), db.nodeSelectCols()), prefix+"%", ) if err != nil { @@ -1347,7 +1217,7 @@ func (db *DB) GetNodeByPrefix(prefix string) (map[string]interface{}, bool, erro // GetNodeByPubkey returns a single node. func (db *DB) GetNodeByPubkey(pubkey string) (map[string]interface{}, error) { - rows, err := db.conn.Query(fmt.Sprintf("SELECT %s FROM nodes WHERE public_key = ?", db.nodeSelectCols()), pubkey) + rows, err := db.conn.Query(fmt.Sprintf("SELECT %s FROM nodes WHERE public_key = "+db.parameter(1), db.nodeSelectCols()), pubkey) if err != nil { return nil, err } @@ -1374,7 +1244,7 @@ func (db *DB) GetRecentTransmissionsForNode(pubkey string, limit int) ([]map[str // #1345: order by ingest id, not first_seen (=rxTime). Buffered observer // uploads with old rxTime would otherwise displace fresh activity from // the "recent transmissions for node" list. - querySQL := fmt.Sprintf("SELECT %s FROM transmissions t %s WHERE t.from_pubkey = ? ORDER BY t.id DESC LIMIT ?", + querySQL := fmt.Sprintf("SELECT %s FROM transmissions t %s WHERE t.from_pubkey = "+db.parameter(1)+" ORDER BY t.id DESC LIMIT "+db.parameter(2), selectCols, observerJoin) args := []interface{}{pubkey, limit} @@ -1439,18 +1309,23 @@ func (db *DB) getObservationsForTransmissions(txIDs []int) map[int][]map[string] placeholders := make([]string, len(txIDs)) args := make([]interface{}, len(txIDs)) for i, id := range txIDs { - placeholders[i] = "?" + placeholders[i] = fmt.Sprintf("%s", db.parameter(i+1)) args[i] = id } var querySQL string if db.isV3 { - querySQL = fmt.Sprintf(`SELECT o.transmission_id, o.id, obs.id AS observer_id, obs.name AS observer_name, COALESCE(obs.iata, '') AS observer_iata, + querySQL = fmt.Sprintf(db.nativeSQL(`SELECT o.transmission_id, o.id, obs.id AS observer_id, obs.name AS observer_name, COALESCE(obs.iata, '') AS observer_iata, o.direction, o.snr, o.rssi, o.path_json, strftime('%%Y-%%m-%%dT%%H:%%M:%%fZ', o.timestamp, 'unixepoch') AS obs_timestamp FROM observations o LEFT JOIN observers obs ON obs.rowid = o.observer_idx WHERE o.transmission_id IN (%s) - ORDER BY o.timestamp DESC`, strings.Join(placeholders, ",")) + ORDER BY o.timestamp DESC`, `SELECT o.transmission_id, o.id, obs.id AS observer_id, obs.name AS observer_name, COALESCE(obs.iata, '') AS observer_iata, + o.direction, o.snr, o.rssi, o.path_json, to_char(to_timestamp(o.timestamp) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"') AS obs_timestamp + FROM observations o + LEFT JOIN observers obs ON obs.rowid = o.observer_idx + WHERE o.transmission_id IN (%s) + ORDER BY o.timestamp DESC`), strings.Join(placeholders, ",")) } else { querySQL = fmt.Sprintf(`SELECT o.transmission_id, o.id, o.observer_id, o.observer_name, COALESCE(obs.iata, '') AS observer_iata, o.direction, o.snr, o.rssi, o.path_json, o.timestamp AS obs_timestamp @@ -1508,17 +1383,17 @@ func (db *DB) GetObservers() ([]Observer, error) { // and the server returns CanRelay=nil so the UI shows no badge. canRelayClause := "COALESCE(can_relay, 1)" canRelaySeenClause := "0" - if hasCol, _ := dbschema.TableHasColumn(db.conn, "observers", "can_relay"); !hasCol { + if hasCol, _ := db.tableHasColumn("observers", "can_relay"); !hasCol { canRelayClause = "1" } - if hasCol, _ := dbschema.TableHasColumn(db.conn, "observers", "can_relay_seen"); hasCol { + if hasCol, _ := db.tableHasColumn("observers", "can_relay_seen"); hasCol { canRelaySeenClause = "COALESCE(can_relay_seen, 0)" } rows, err := db.conn.Query(`SELECT id, name, iata, last_seen, first_seen, packet_count, model, firmware, client_version, radio, battery_mv, uptime_secs, noise_floor, last_packet_at, clock_skew_seconds, clock_skew_count_24h, clock_last_naive_at, ` + canRelayClause + `, ` + canRelaySeenClause + ` - FROM observers WHERE inactive IS NULL OR inactive = 0 ORDER BY last_seen DESC`) + FROM observers WHERE inactive IS NULL OR inactive = 0 ORDER BY last_seen DESC NULLS LAST`) if err != nil { return nil, err } @@ -1571,7 +1446,7 @@ func (db *DB) GetObservers() ([]Observer, error) { func (db *DB) GetNonRelayObserverPubkeys() ([]string, error) { // Graceful no-op when can_relay column is absent (legacy DB / older // test fixture). Avoids noisy schema-degradation log spam. - if hasCol, _ := dbschema.TableHasColumn(db.conn, "observers", "can_relay"); !hasCol { + if hasCol, _ := db.tableHasColumn("observers", "can_relay"); !hasCol { return nil, nil } rows, err := db.conn.Query(`SELECT LOWER(id) FROM observers @@ -1597,7 +1472,7 @@ func (db *DB) GetNonRelayObserverPubkeys() ([]string, error) { // this to render tri-state — observers NOT in this set are "unknown" // and the UI shows no badge. func (db *DB) GetCanRelaySeenObserverPubkeys() ([]string, error) { - if hasCol, _ := dbschema.TableHasColumn(db.conn, "observers", "can_relay_seen"); !hasCol { + if hasCol, _ := db.tableHasColumn("observers", "can_relay_seen"); !hasCol { return nil, nil } rows, err := db.conn.Query(`SELECT LOWER(id) FROM observers @@ -1626,17 +1501,17 @@ func (db *DB) GetObserverByID(id string) (*Observer, error) { var canRelay, canRelaySeen int canRelayClause := "COALESCE(can_relay, 1)" canRelaySeenClause := "0" - if hasCol, _ := dbschema.TableHasColumn(db.conn, "observers", "can_relay"); !hasCol { + if hasCol, _ := db.tableHasColumn("observers", "can_relay"); !hasCol { canRelayClause = "1" } - if hasCol, _ := dbschema.TableHasColumn(db.conn, "observers", "can_relay_seen"); hasCol { + if hasCol, _ := db.tableHasColumn("observers", "can_relay_seen"); hasCol { canRelaySeenClause = "COALESCE(can_relay_seen, 0)" } err := db.conn.QueryRow(`SELECT id, name, iata, last_seen, first_seen, packet_count, model, firmware, client_version, radio, battery_mv, uptime_secs, noise_floor, last_packet_at, clock_skew_seconds, clock_skew_count_24h, clock_last_naive_at, `+canRelayClause+`, `+canRelaySeenClause+` - FROM observers WHERE id = ?`, id). + FROM observers WHERE id = `+db.parameter(1), id). Scan(&o.ID, &o.Name, &o.IATA, &o.LastSeen, &o.FirstSeen, &o.PacketCount, &o.Model, &o.Firmware, &o.ClientVersion, &o.Radio, &batteryMv, &uptimeSecs, &noiseFloor, &o.LastPacketAt, &clockSkewSec, &clockSkewCount, &o.ClockLastNaiveAt, &canRelay, &canRelaySeen) @@ -1676,7 +1551,7 @@ func (db *DB) GetObserverIdsForRegion(regionParam string) ([]string, error) { placeholders := make([]string, len(codes)) args := make([]interface{}, len(codes)) for i, c := range codes { - placeholders[i] = "?" + placeholders[i] = fmt.Sprintf("%s", db.parameter(i+1)) args[i] = c } rows, err := db.conn.Query(fmt.Sprintf("SELECT id FROM observers WHERE UPPER(TRIM(iata)) IN (%s)", strings.Join(placeholders, ",")), args...) @@ -1795,22 +1670,35 @@ func (db *DB) GetNetworkStatus(healthThresholds HealthThresholds) (map[string]in func (db *DB) GetTraces(hash string) ([]map[string]interface{}, error) { var querySQL string if db.isV3 { - querySQL = `SELECT obs.id AS observer_id, obs.name AS observer_name, + querySQL = db.nativeSQL(`SELECT obs.id AS observer_id, obs.name AS observer_name, strftime('%Y-%m-%dT%H:%M:%fZ', o.timestamp, 'unixepoch') AS timestamp, o.snr, o.rssi, o.path_json FROM observations o JOIN transmissions t ON t.id = o.transmission_id LEFT JOIN observers obs ON obs.rowid = o.observer_idx - WHERE t.hash = ? - ORDER BY o.timestamp ASC` + WHERE t.hash = ?1 + ORDER BY o.timestamp ASC`, `SELECT obs.id AS observer_id, obs.name AS observer_name, + to_char(to_timestamp(o.timestamp) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"') AS timestamp, + o.snr, o.rssi, o.path_json + FROM observations o + JOIN transmissions t ON t.id = o.transmission_id + LEFT JOIN observers obs ON obs.rowid = o.observer_idx + WHERE t.hash = `+db.parameter(1)+` + ORDER BY o.timestamp ASC`) } else { - querySQL = `SELECT o.observer_id, o.observer_name, + querySQL = db.nativeSQL(`SELECT o.observer_id, o.observer_name, strftime('%Y-%m-%dT%H:%M:%fZ', o.timestamp, 'unixepoch') AS timestamp, o.snr, o.rssi, o.path_json FROM observations o JOIN transmissions t ON t.id = o.transmission_id - WHERE t.hash = ? - ORDER BY o.timestamp ASC` + WHERE t.hash = ?1 + ORDER BY o.timestamp ASC`, `SELECT o.observer_id, o.observer_name, + to_char(to_timestamp(o.timestamp) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"') AS timestamp, + o.snr, o.rssi, o.path_json + FROM observations o + JOIN transmissions t ON t.id = o.transmission_id + WHERE t.hash = `+db.parameter(1)+` + ORDER BY o.timestamp ASC`) } rows, err := db.conn.Query(querySQL, strings.ToLower(hash)) if err != nil { @@ -1912,14 +1800,11 @@ func (db *DB) GetChannels(region ...string) ([]map[string]interface{}, error) { if len(regionCodes) > 0 { placeholders := make([]string, len(regionCodes)) for i, code := range regionCodes { - placeholders[i] = "?" + placeholders[i] = fmt.Sprintf("%s", db.parameter(i+1)) args = append(args, code) } regionPlaceholder := strings.Join(placeholders, ",") - // #1899: the sample_json subquery is region-scoped too, so its placeholders - // appear FIRST in the statement (it sits in the SELECT list, ahead of the - // WHERE). Bind the codes twice, subquery set first. - args = append(append(make([]interface{}, 0, len(regionCodes)*2), args...), args...) + // Reuse the same native parameters for sample and channel region filters. if db.isV3 { querySQL = fmt.Sprintf(`SELECT t.channel_hash, COUNT(*) AS msg_count, @@ -1938,7 +1823,7 @@ func (db *DB) GetChannels(region ...string) ([]map[string]interface{}, error) { AND t.channel_hash NOT LIKE 'enc_%%' AND obs.rowid IS NOT NULL AND UPPER(TRIM(obs.iata)) IN (%s) GROUP BY t.channel_hash - ORDER BY last_activity DESC`, regionPlaceholder, regionPlaceholder) + ORDER BY last_activity DESC NULLS LAST`, regionPlaceholder, regionPlaceholder) } else { querySQL = fmt.Sprintf(`SELECT t.channel_hash, COUNT(*) AS msg_count, @@ -1963,7 +1848,7 @@ func (db *DB) GetChannels(region ...string) ([]map[string]interface{}, error) { AND UPPER(TRIM(obs.iata)) IN (%s) ) GROUP BY t.channel_hash - ORDER BY last_activity DESC`, regionPlaceholder, regionPlaceholder) + ORDER BY last_activity DESC NULLS LAST`, regionPlaceholder, regionPlaceholder) } } else { querySQL = `SELECT channel_hash, @@ -1977,7 +1862,7 @@ func (db *DB) GetChannels(region ...string) ([]map[string]interface{}, error) { AND channel_hash IS NOT NULL AND channel_hash NOT LIKE 'enc_%%' GROUP BY channel_hash - ORDER BY last_activity DESC` + ORDER BY last_activity DESC NULLS LAST` } rows, err := db.conn.Query(querySQL, args...) @@ -2064,7 +1949,7 @@ func (db *DB) GetEncryptedChannels(region ...string) ([]map[string]interface{}, if len(regionCodes) > 0 { placeholders := make([]string, len(regionCodes)) for i, code := range regionCodes { - placeholders[i] = "?" + placeholders[i] = fmt.Sprintf("%s", db.parameter(i+1)) args = append(args, code) } regionPlaceholder := strings.Join(placeholders, ",") @@ -2079,7 +1964,7 @@ func (db *DB) GetEncryptedChannels(region ...string) ([]map[string]interface{}, AND t.channel_hash LIKE 'enc_%%' AND obs.rowid IS NOT NULL AND UPPER(TRIM(obs.iata)) IN (%s) GROUP BY t.channel_hash - ORDER BY last_activity DESC`, regionPlaceholder) + ORDER BY last_activity DESC NULLS LAST`, regionPlaceholder) } else { querySQL = fmt.Sprintf(`SELECT t.channel_hash, COUNT(*) AS msg_count, @@ -2094,7 +1979,7 @@ func (db *DB) GetEncryptedChannels(region ...string) ([]map[string]interface{}, AND UPPER(TRIM(obs.iata)) IN (%s) ) GROUP BY t.channel_hash - ORDER BY last_activity DESC`, regionPlaceholder) + ORDER BY last_activity DESC NULLS LAST`, regionPlaceholder) } } else { querySQL = `SELECT channel_hash, @@ -2104,7 +1989,7 @@ func (db *DB) GetEncryptedChannels(region ...string) ([]map[string]interface{}, WHERE payload_type = 5 AND channel_hash LIKE 'enc_%%' GROUP BY channel_hash - ORDER BY last_activity DESC` + ORDER BY last_activity DESC NULLS LAST` } rows, err := db.conn.Query(querySQL, args...) @@ -2201,7 +2086,7 @@ func (db *DB) GetChannelMessages(channelHash string, limit, offset int, region . if len(regionCodes) > 0 { placeholders := make([]string, len(regionCodes)) for i, code := range regionCodes { - placeholders[i] = "?" + placeholders[i] = fmt.Sprintf("%s", db.parameter(i+2)) regionArgs = append(regionArgs, code) } regionPlaceholders = strings.Join(placeholders, ",") @@ -2228,7 +2113,7 @@ func (db *DB) GetChannelMessages(channelHash string, limit, offset int, region . // 1) Total count (after region filter, before pagination). countSQL := `SELECT COUNT(*) FROM transmissions t - WHERE t.channel_hash = ? AND t.payload_type = 5` + regionFilter + WHERE t.channel_hash = ` + db.parameter(1) + ` AND t.payload_type = 5` + regionFilter countArgs := []interface{}{channelHash} countArgs = append(countArgs, regionArgs...) var total int @@ -2258,16 +2143,16 @@ func (db *DB) GetChannelMessages(channelHash string, limit, offset int, region . pageSQL := `SELECT t.id, COALESCE((SELECT MAX(timestamp) FROM observations WHERE transmission_id = t.id), 0) AS latest_obs_epoch FROM transmissions t - WHERE t.channel_hash = ? AND t.payload_type = 5 - ORDER BY latest_obs_epoch DESC, t.id DESC - LIMIT ? OFFSET ?` + WHERE t.channel_hash = ` + db.parameter(1) + ` AND t.payload_type = 5 + ORDER BY latest_obs_epoch DESC NULLS LAST, t.id DESC + LIMIT ` + fmt.Sprintf("%s OFFSET %s", db.parameter(len(regionArgs)+2), db.parameter(len(regionArgs)+3)) if len(regionCodes) > 0 { pageSQL = `SELECT t.id, COALESCE((SELECT MAX(timestamp) FROM observations WHERE transmission_id = t.id), 0) AS latest_obs_epoch FROM transmissions t - WHERE t.channel_hash = ? AND t.payload_type = 5` + regionFilter + ` - ORDER BY latest_obs_epoch DESC, t.id DESC - LIMIT ? OFFSET ?` + WHERE t.channel_hash = ` + db.parameter(1) + ` AND t.payload_type = 5` + regionFilter + ` + ORDER BY latest_obs_epoch DESC NULLS LAST, t.id DESC + LIMIT ` + fmt.Sprintf("%s OFFSET %s", db.parameter(len(regionArgs)+2), db.parameter(len(regionArgs)+3)) } pageArgs := []interface{}{channelHash} pageArgs = append(pageArgs, regionArgs...) @@ -2299,7 +2184,7 @@ func (db *DB) GetChannelMessages(channelHash string, limit, offset int, region . idPlaceholders := make([]string, len(pageIDs)) obsArgs := make([]interface{}, len(pageIDs)) for i, id := range pageIDs { - idPlaceholders[i] = "?" + idPlaceholders[i] = fmt.Sprintf("%s", db.parameter(i+1)) obsArgs[i] = id } // #1851: scope_name lives on the transmission row, so appending it as the @@ -2460,13 +2345,17 @@ func (db *DB) GetChannelMessages(channelHash string, limit, offset int, region . return messages, total, nil } +// IDs are NOT NULL. Keep native index ordering so LIMIT stops the cursor +// after its page, without sorting every row beyond the cursor. +const newTransmissionsSinceSQL = `SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type, t.payload_type, t.payload_version, t.decoded_json + FROM transmissions t WHERE t.id > $1 ORDER BY t.id ASC LIMIT $2` + // GetNewTransmissionsSince returns new transmissions after a given ID for WebSocket polling. func (db *DB) GetNewTransmissionsSince(lastID int, limit int) ([]map[string]interface{}, error) { if limit <= 0 { limit = 100 } - rows, err := db.conn.Query(`SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type, t.payload_type, t.payload_version, t.decoded_json - FROM transmissions t WHERE t.id > ? ORDER BY t.id ASC LIMIT ?`, lastID, limit) + rows, err := db.conn.Query(newTransmissionsSinceSQL, lastID, limit) if err != nil { return nil, err } @@ -2483,6 +2372,7 @@ func (db *DB) GetNewTransmissionsSince(lastID int, limit int) ([]map[string]inte "raw_hex": nullStr(rawHex), "hash": nullStr(hash), "first_seen": nullStr(firstSeen), + "timestamp": nullStr(firstSeen), "route_type": nullInt(routeType), "payload_type": nullInt(payloadType), "payload_version": nullInt(payloadVersion), @@ -2506,23 +2396,25 @@ func (db *DB) GetMaxObservationID() int { return maxID } -// GetObserverPacketCounts returns packetsLastHour for all observers (batch query). -func (db *DB) GetObserverPacketCounts(sinceEpoch int64) map[string]int { - counts := make(map[string]int) - var rows *sql.Rows - var err error +// observerPacketCountsSQL is shared with the native planner regression. +func (db *DB) observerPacketCountsSQL() string { if db.isV3 { - rows, err = db.conn.Query(`SELECT obs.id, COUNT(*) as cnt + return `SELECT obs.id, COUNT(*) as cnt FROM observations o JOIN observers obs ON obs.rowid = o.observer_idx - WHERE o.timestamp > ? - GROUP BY obs.id`, sinceEpoch) - } else { - rows, err = db.conn.Query(`SELECT o.observer_id, COUNT(*) as cnt - FROM observations o - WHERE o.observer_id IS NOT NULL AND o.timestamp > ? - GROUP BY o.observer_id`, sinceEpoch) + WHERE o.timestamp > ` + db.parameter(1) + ` + GROUP BY obs.id` } + return `SELECT o.observer_id, COUNT(*) as cnt + FROM observations o + WHERE o.observer_id IS NOT NULL AND o.timestamp > ` + db.parameter(1) + ` + GROUP BY o.observer_id` +} + +// GetObserverPacketCounts returns packetsLastHour for all observers (batch query). +func (db *DB) GetObserverPacketCounts(sinceEpoch int64) map[string]int { + counts := make(map[string]int) + rows, err := db.conn.Query(db.observerPacketCountsSQL(), sinceEpoch) if err != nil { return counts } @@ -2568,7 +2460,7 @@ func (db *DB) GetNodeLocationsByKeys(keys []string) map[string]map[string]interf placeholders := make([]string, len(keys)) args := make([]interface{}, len(keys)) for i, k := range keys { - placeholders[i] = "?" + placeholders[i] = fmt.Sprintf("%s", db.parameter(i+1)) args[i] = strings.ToLower(k) } // #1481 P0-3: drop LOWER(public_key) — that wrap is non-sargable and @@ -2596,6 +2488,7 @@ func (db *DB) GetNodeLocationsByKeys(keys []string) map[string]map[string]interf // QueryMultiNodePackets returns transmissions referencing any of the given pubkeys. func (db *DB) QueryMultiNodePackets(pubkeys []string, limit, offset int, order, since, until string) (*PacketResult, error) { + offset = max(0, offset) if len(pubkeys) == 0 { return &PacketResult{Packets: []map[string]interface{}{}, Total: 0}, nil } @@ -2613,17 +2506,17 @@ func (db *DB) QueryMultiNodePackets(pubkeys []string, limit, offset int, order, for _, pk := range pubkeys { resolved := db.resolveNodePubkey(pk) args = append(args, resolved) - placeholders = append(placeholders, "?") + placeholders = append(placeholders, fmt.Sprintf("%s", db.parameter(len(args)))) } pkWhere := "t.from_pubkey IN (" + strings.Join(placeholders, ",") + ")" var timeFilters []string if since != "" { - timeFilters = append(timeFilters, "t.first_seen >= ?") + timeFilters = append(timeFilters, fmt.Sprintf("t.first_seen >= %s", db.parameter(len(args)+1))) args = append(args, since) } if until != "" { - timeFilters = append(timeFilters, "t.first_seen <= ?") + timeFilters = append(timeFilters, fmt.Sprintf("t.first_seen <= %s", db.parameter(len(args)+1))) args = append(args, until) } @@ -2637,8 +2530,8 @@ func (db *DB) QueryMultiNodePackets(pubkeys []string, limit, offset int, order, selectCols, observerJoin := db.transmissionBaseSQL() // #1345: order by ingest id (see QueryPackets comment above). - querySQL := fmt.Sprintf("SELECT %s FROM transmissions t %s %s ORDER BY t.id %s LIMIT ? OFFSET ?", - selectCols, observerJoin, w, order) + querySQL := fmt.Sprintf("SELECT %s FROM transmissions t %s %s ORDER BY t.id %s LIMIT %s OFFSET %s", + selectCols, observerJoin, w, order, db.parameter(len(args)+1), db.parameter(len(args)+2)) qArgs := make([]interface{}, len(args)) copy(qArgs, args) @@ -2665,8 +2558,8 @@ func (db *DB) QueryMultiNodePackets(pubkeys []string, limit, offset int, order, func scanPacketRow(rows *sql.Rows) map[string]interface{} { var id int var rawHex, ts, obsID, obsName, direction, hash, pathJSON, decodedJSON, createdAt sql.NullString - var snr, rssi sql.NullFloat64 - var score, routeType, payloadType, payloadVersion sql.NullInt64 + var snr, rssi, score sql.NullFloat64 + var routeType, payloadType, payloadVersion sql.NullInt64 if err := rows.Scan(&id, &rawHex, &ts, &obsID, &obsName, &direction, &snr, &rssi, &score, &hash, &routeType, &payloadType, &payloadVersion, &pathJSON, &decodedJSON, &createdAt); err != nil { return nil @@ -2680,7 +2573,7 @@ func scanPacketRow(rows *sql.Rows) map[string]interface{} { "direction": nullStr(direction), "snr": nullFloat(snr), "rssi": nullFloat(rssi), - "score": nullInt(score), + "score": nullFloat(score), "hash": nullStr(hash), "route_type": nullInt(routeType), "payload_type": nullInt(payloadType), @@ -2797,7 +2690,7 @@ func nullInt(ni sql.NullInt64) interface{} { // PruneOldPackets, PruneOldMetrics, and RemoveStaleObservers were // removed in #1283 — they are write operations and now live on the // ingestor's *Store (cmd/ingestor/maintenance.go and cmd/ingestor/db.go). -// The server is the read path; it must not hold the SQLite write lock. +// The server is the read path; it must not write telemetry. // MetricsSample represents a single row from observer_metrics with computed deltas. type MetricsSample struct { @@ -2849,37 +2742,47 @@ func (db *DB) GetObserverMetrics(observerID, since, until, resolution string, sa // Use LAST value per bucket (latest timestamp) instead of MAX to preserve // reboot semantics: if a device reboots mid-bucket, the last sample is the // post-reboot baseline, not the pre-reboot high-water mark. - query = `SELECT ts, noise_floor, tx_air_secs, rx_air_secs, recv_errors, battery_mv, packets_sent, packets_recv FROM ( + query = db.nativeSQL(`SELECT ts, noise_floor, tx_air_secs, rx_air_secs, recv_errors, battery_mv, packets_sent, packets_recv FROM ( SELECT strftime('%Y-%m-%dT%H:00:00Z', timestamp) as ts, noise_floor, tx_air_secs, rx_air_secs, recv_errors, battery_mv, packets_sent, packets_recv, ROW_NUMBER() OVER (PARTITION BY observer_id, strftime('%Y-%m-%dT%H:00:00Z', timestamp) ORDER BY timestamp DESC) as rn - FROM observer_metrics WHERE observer_id = ?` + FROM observer_metrics WHERE observer_id = ?1`, `SELECT ts, noise_floor, tx_air_secs, rx_air_secs, recv_errors, battery_mv, packets_sent, packets_recv FROM ( + SELECT + to_char(timestamp::timestamptz AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:00:00"Z"') as ts, + noise_floor, tx_air_secs, rx_air_secs, recv_errors, battery_mv, packets_sent, packets_recv, + ROW_NUMBER() OVER (PARTITION BY observer_id, to_char(timestamp::timestamptz AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:00:00"Z"') ORDER BY timestamp DESC) as rn + FROM observer_metrics WHERE observer_id = `+db.parameter(1)) case "1d": bucketSizeSec = 86400 - query = `SELECT ts, noise_floor, tx_air_secs, rx_air_secs, recv_errors, battery_mv, packets_sent, packets_recv FROM ( + query = db.nativeSQL(`SELECT ts, noise_floor, tx_air_secs, rx_air_secs, recv_errors, battery_mv, packets_sent, packets_recv FROM ( SELECT strftime('%Y-%m-%dT00:00:00Z', timestamp) as ts, noise_floor, tx_air_secs, rx_air_secs, recv_errors, battery_mv, packets_sent, packets_recv, ROW_NUMBER() OVER (PARTITION BY observer_id, strftime('%Y-%m-%dT00:00:00Z', timestamp) ORDER BY timestamp DESC) as rn - FROM observer_metrics WHERE observer_id = ?` + FROM observer_metrics WHERE observer_id = ?1`, `SELECT ts, noise_floor, tx_air_secs, rx_air_secs, recv_errors, battery_mv, packets_sent, packets_recv FROM ( + SELECT + to_char(timestamp::timestamptz AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"00:00:00"Z"') as ts, + noise_floor, tx_air_secs, rx_air_secs, recv_errors, battery_mv, packets_sent, packets_recv, + ROW_NUMBER() OVER (PARTITION BY observer_id, to_char(timestamp::timestamptz AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"00:00:00"Z"') ORDER BY timestamp DESC) as rn + FROM observer_metrics WHERE observer_id = `+db.parameter(1)) default: // "5m" or raw query = `SELECT timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors, battery_mv, packets_sent, packets_recv - FROM observer_metrics WHERE observer_id = ?` + FROM observer_metrics WHERE observer_id = ` + db.parameter(1) } if since != "" { - query += " AND timestamp >= ?" + query += fmt.Sprintf(" AND timestamp >= %s", db.parameter(len(args)+1)) args = append(args, since) } if until != "" { - query += " AND timestamp <= ?" + query += fmt.Sprintf(" AND timestamp <= %s", db.parameter(len(args)+1)) args = append(args, until) } switch resolution { case "1h", "1d": - query += ") WHERE rn = 1 ORDER BY ts ASC" + query += ") WHERE rn = 1 ORDER BY ts ASC NULLS FIRST" default: query += " ORDER BY timestamp ASC" } @@ -3053,7 +2956,7 @@ func (db *DB) GetMetricsSummary(since string, now time.Time) ([]MetricsSummaryRo SELECT observer_id, noise_floor, battery_mv, ROW_NUMBER() OVER (PARTITION BY observer_id ORDER BY timestamp DESC) as rn FROM observer_metrics - WHERE timestamp >= ? + WHERE timestamp >= ` + db.parameter(1) + ` ) SELECT m.observer_id, o.name, COALESCE(o.iata, '') as iata, r.noise_floor as current_nf, @@ -3064,9 +2967,9 @@ func (db *DB) GetMetricsSummary(since string, now time.Time) ([]MetricsSummaryRo FROM observer_metrics m LEFT JOIN observers o ON o.id = m.observer_id LEFT JOIN ranked r ON r.observer_id = m.observer_id AND r.rn = 1 - WHERE m.timestamp >= ? - GROUP BY m.observer_id - ORDER BY max_nf DESC + WHERE m.timestamp >= ` + db.parameter(2) + ` + GROUP BY m.observer_id, o.name, o.iata, r.noise_floor, r.battery_mv + ORDER BY max_nf DESC NULLS LAST ` rows, err := db.conn.Query(query, since, since) if err != nil { @@ -3089,7 +2992,7 @@ func (db *DB) GetMetricsSummary(since string, now time.Time) ([]MetricsSummaryRo // Fetch sparkline data (noise_floor series) for all observers in one query if len(result) > 0 { sparkQuery := `SELECT observer_id, noise_floor FROM observer_metrics - WHERE timestamp >= ? ORDER BY observer_id, timestamp ASC` + WHERE timestamp >= ` + db.parameter(1) + ` ORDER BY observer_id, timestamp ASC` sparkRows, err := db.conn.Query(sparkQuery, since) if err != nil { return nil, err @@ -3128,7 +3031,7 @@ func (db *DB) GetMetricsSummary(since string, now time.Time) ([]MetricsSummaryRo func (db *DB) fillAirtime1h(rows []MetricsSummaryRow, now time.Time) error { since := now.Add(-airtimeWindow).Format(time.RFC3339) q, err := db.conn.Query(`SELECT observer_id, timestamp, tx_air_secs, rx_air_secs - FROM observer_metrics WHERE timestamp >= ? ORDER BY observer_id, timestamp ASC`, since) + FROM observer_metrics WHERE timestamp >= `+db.parameter(1)+` ORDER BY observer_id, timestamp ASC`, since) if err != nil { return err } @@ -3171,17 +3074,17 @@ func (db *DB) GetDroppedPackets(limit int, observerID, nodePubkey string) ([]map var conditions []string var args []interface{} if observerID != "" { - conditions = append(conditions, "observer_id = ?") + conditions = append(conditions, fmt.Sprintf("observer_id = %s", db.parameter(len(args)+1))) args = append(args, observerID) } if nodePubkey != "" { - conditions = append(conditions, "node_pubkey = ?") + conditions = append(conditions, fmt.Sprintf("node_pubkey = %s", db.parameter(len(args)+1))) args = append(args, nodePubkey) } if len(conditions) > 0 { query += " WHERE " + strings.Join(conditions, " AND ") } - query += " ORDER BY dropped_at DESC LIMIT ?" + query += fmt.Sprintf(" ORDER BY dropped_at DESC NULLS LAST LIMIT %s", db.parameter(len(args)+1)) args = append(args, limit) rows, err := db.conn.Query(query, args...) @@ -3285,6 +3188,14 @@ var scopeStatsByRegionQuery = ` ORDER BY cnt DESC ` +var postgresScopeStatsByRegionQuery = ` + SELECT scope_name, COUNT(*) AS cnt + FROM transmissions + WHERE ` + routeTypeTransportSQL + ` AND scope_name IS NOT NULL AND scope_name != '' AND first_seen >= $1 + GROUP BY scope_name + ORDER BY cnt DESC NULLS LAST + ` + func (db *DB) GetScopeStats(window string) (*ScopeStatsResponse, error) { if !db.hasScopeName { return nil, fmt.Errorf("scope_name column not present — run ingestor to apply migrations") @@ -3296,16 +3207,16 @@ func (db *DB) GetScopeStats(window string) (*ScopeStatsResponse, error) { case "1h": since = time.Now().Add(-1 * time.Hour).UTC().Format(time.RFC3339) // 5-minute buckets - bucketExpr = `strftime('%Y-%m-%dT%H:', first_seen) || printf('%02d', (CAST(strftime('%M', first_seen) AS INTEGER) / 5) * 5) || ':00Z'` + bucketExpr = db.nativeSQL(`strftime('%Y-%m-%dT%H:', first_seen) || printf('%02d', (CAST(strftime('%M', first_seen) AS INTEGER) / 5) * 5) || ':00Z'`, `to_char(date_bin('5 minutes', (CASE WHEN pg_input_is_valid(first_seen, 'timestamp with time zone') THEN first_seen::timestamptz END), '2000-01-01T00:00:00Z'::timestamptz) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"')`) case "7d": since = time.Now().Add(-7 * 24 * time.Hour).UTC().Format(time.RFC3339) // 6-hour buckets - bucketExpr = `strftime('%Y-%m-%dT', first_seen) || printf('%02d', (CAST(strftime('%H', first_seen) AS INTEGER) / 6) * 6) || ':00:00Z'` + bucketExpr = db.nativeSQL(`strftime('%Y-%m-%dT', first_seen) || printf('%02d', (CAST(strftime('%H', first_seen) AS INTEGER) / 6) * 6) || ':00:00Z'`, `to_char(date_bin('6 hours', (CASE WHEN pg_input_is_valid(first_seen, 'timestamp with time zone') THEN first_seen::timestamptz END), '2000-01-01T00:00:00Z'::timestamptz) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"')`) default: // "24h" window = "24h" since = time.Now().Add(-24 * time.Hour).UTC().Format(time.RFC3339) // 1-hour buckets - bucketExpr = `strftime('%Y-%m-%dT%H:00:00Z', first_seen)` + bucketExpr = db.nativeSQL(`strftime('%Y-%m-%dT%H:00:00Z', first_seen)`, `to_char((CASE WHEN pg_input_is_valid(first_seen, 'timestamp with time zone') THEN first_seen::timestamptz END) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:00:00"Z"')`) } resp := &ScopeStatsResponse{Window: window} @@ -3318,7 +3229,7 @@ func (db *DB) GetScopeStats(window string) (*ScopeStatsResponse, error) { COALESCE(SUM(CASE WHEN scope_name IS NULL THEN 1 ELSE 0 END), 0) AS unscoped, COALESCE(SUM(CASE WHEN scope_name = '' THEN 1 ELSE 0 END), 0) AS unknown_scope FROM transmissions - WHERE `+routeTypeTransportSQL+` AND first_seen >= ? + WHERE `+routeTypeTransportSQL+` AND first_seen >= `+db.parameter(1)+` `, since) if err := row.Scan( &resp.Summary.TransportTotal, @@ -3336,14 +3247,14 @@ func (db *DB) GetScopeStats(window string) (*ScopeStatsResponse, error) { var nonTransportUnscoped int if err := db.conn.QueryRow(` SELECT COUNT(*) FROM transmissions - WHERE `+routeTypeNonTransportSQL+` AND first_seen >= ? + WHERE `+routeTypeNonTransportSQL+` AND first_seen >= `+db.parameter(1)+` `, since).Scan(&nonTransportUnscoped); err != nil { return nil, fmt.Errorf("scope non-transport count query: %w", err) } resp.Summary.Unscoped += nonTransportUnscoped // Per-region counts (named regions only) - rows, err := db.conn.Query(scopeStatsByRegionQuery, since) + rows, err := db.conn.Query(db.nativeSQL(scopeStatsByRegionQuery, postgresScopeStatsByRegionQuery), since) if err != nil { return nil, fmt.Errorf("scope byRegion query: %w", err) } @@ -3367,7 +3278,7 @@ func (db *DB) GetScopeStats(window string) (*ScopeStatsResponse, error) { COUNT(scope_name) AS scoped, SUM(CASE WHEN scope_name IS NULL THEN 1 ELSE 0 END) AS unscoped FROM transmissions - WHERE `+routeTypeTransportSQL+` AND first_seen >= ? + WHERE `+routeTypeTransportSQL+` AND first_seen >= `+db.parameter(1)+` GROUP BY bucket ORDER BY bucket `, bucketExpr) @@ -3402,9 +3313,9 @@ func (db *DB) GetScopeStats(window string) (*ScopeStatsResponse, error) { SUM(CASE WHEN t.scope_name != '' THEN 1 ELSE 0 END) AS named FROM transmissions t LEFT JOIN nodes n ON n.public_key = t.from_pubkey - WHERE +t.payload_type = ? AND t.route_type IN (0, 1) AND t.first_seen >= ? + WHERE +t.payload_type = `+db.parameter(1)+` AND t.route_type IN (0, 1) AND t.first_seen >= `+db.parameter(2)+` GROUP BY 1 - ORDER BY COUNT(*) DESC, 1 + ORDER BY COUNT(*) DESC NULLS LAST, 1 `, payloadTypeAdvert, since) if err != nil { return nil, fmt.Errorf("scope advertsByRole query: %w", err) diff --git a/cmd/server/db_channel_messages_perf_test.go b/cmd/server/db_channel_messages_perf_test.go index b96fc8d42..9a7a0d1dd 100644 --- a/cmd/server/db_channel_messages_perf_test.go +++ b/cmd/server/db_channel_messages_perf_test.go @@ -37,39 +37,15 @@ func TestGetChannelMessagesPerfLargeChannel(t *testing.T) { const numTx = 1500 const obsPerTx = 50 - tx, err := db.conn.Begin() - if err != nil { - t.Fatal(err) - } - txStmt, err := tx.Prepare(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash) - VALUES (?, ?, ?, 1, 5, ?, '#perf')`) - if err != nil { - t.Fatal(err) - } - obsStmt, err := tx.Prepare(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (?, 1, 10.0, -90, '[]', ?)`) - if err != nil { - t.Fatal(err) - } base := time.Now().UTC().Add(-24 * time.Hour) - for i := 0; i < numTx; i++ { - ts := base.Add(time.Duration(i) * time.Second).Format(time.RFC3339) - hash := fmt.Sprintf("perfhash%08d", i) + copyTestRows(t, db.conn, "transmissions", []string{"id", "raw_hex", "hash", "first_seen", "route_type", "payload_type", "decoded_json", "channel_hash"}, numTx, func(i int) []any { body := fmt.Sprintf(`{"type":"CHAN","channel":"#perf","text":"Sender%d: msg %d","sender":"Sender%d"}`, i%10, i, i%10) - res, err := txStmt.Exec(fmt.Sprintf("%04X", i), hash, ts, body) - if err != nil { - t.Fatal(err) - } - txID, _ := res.LastInsertId() - for o := 0; o < obsPerTx; o++ { - if _, err := obsStmt.Exec(txID, base.Unix()+int64(i*100+o)); err != nil { - t.Fatal(err) - } - } - } - if err := tx.Commit(); err != nil { - t.Fatal(err) - } + return []any{i + 1, fmt.Sprintf("%04X", i), fmt.Sprintf("perfhash%08d", i), base.Add(time.Duration(i) * time.Second).Format(time.RFC3339), 1, 5, body, "#perf"} + }) + copyTestRows(t, db.conn, "observations", []string{"id", "transmission_id", "observer_idx", "snr", "rssi", "path_json", "timestamp"}, numTx*obsPerTx, func(n int) []any { + i, o := n/obsPerTx, n%obsPerTx + return []any{n + 1, i + 1, 1, 10.0, -90.0, "[]", base.Unix() + int64(i*100+o)} + }) // Warm-up call to amortize first-run prepare cost. if _, _, err := db.GetChannelMessages("#perf", 50, 0); err != nil { diff --git a/cmd/server/db_test.go b/cmd/server/db_test.go index 1e788e740..fb7678297 100644 --- a/cmd/server/db_test.go +++ b/cmd/server/db_test.go @@ -11,22 +11,25 @@ import ( "strings" "testing" "time" - - _ "github.com/mattn/go-sqlite3" ) // setupTestDB creates an in-memory SQLite database with the v3 schema. func setupTestDB(t testing.TB) *DB { + return setupTestDBAtURL(t, postgresTestDSN(t)) +} + +func setupTestDBAtURL(t testing.TB, dsn string) *DB { t.Helper() - conn, err := sql.Open("sqlite3", ":memory:") + conn, err := openFixtureSQL(dsn) if err != nil { t.Fatal(err) } - // Force single connection so all goroutines share the same in-memory DB + t.Cleanup(func() { conn.Close() }) + // One fixture connection keeps write and assertion order deterministic. conn.SetMaxOpenConns(1) // Create schema matching MeshCore Analyzer v3 - schema := ` + schema := testNativeSQL(` CREATE TABLE nodes ( public_key TEXT PRIMARY KEY, name TEXT, @@ -138,12 +141,122 @@ func setupTestDB(t testing.TB) *DB { CREATE INDEX IF NOT EXISTS idx_observations_timestamp ON observations(timestamp); CREATE INDEX IF NOT EXISTS idx_observations_tx_ts ON observations(transmission_id, timestamp); CREATE INDEX IF NOT EXISTS idx_transmissions_channel_hash ON transmissions(channel_hash); - ` + `, ` + CREATE TABLE nodes ( + public_key TEXT PRIMARY KEY, + name TEXT, + role TEXT, + lat DOUBLE PRECISION, + lon DOUBLE PRECISION, + last_seen TEXT, + first_seen TEXT, + advert_count INTEGER DEFAULT 0, + battery_mv INTEGER, + temperature_c DOUBLE PRECISION, + foreign_advert INTEGER DEFAULT 0 + ); + + CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY UNIQUE, + id TEXT PRIMARY KEY, + name TEXT, + iata TEXT, + last_seen TEXT, + first_seen TEXT, + packet_count INTEGER DEFAULT 0, + model TEXT, + firmware TEXT, + client_version TEXT, + radio TEXT, + battery_mv INTEGER, + uptime_secs INTEGER, + noise_floor DOUBLE PRECISION, + inactive INTEGER DEFAULT 0, + last_packet_at TEXT DEFAULT NULL, + clock_skew_seconds INTEGER DEFAULT NULL, + clock_skew_count_24h INTEGER DEFAULT 0, + clock_last_naive_at TEXT DEFAULT NULL + ); + + CREATE TABLE transmissions ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + raw_hex TEXT NOT NULL, + hash TEXT NOT NULL UNIQUE, + first_seen TEXT NOT NULL, + route_type INTEGER, + payload_type INTEGER, + payload_version INTEGER, + decoded_json TEXT, + channel_hash TEXT DEFAULT NULL, + from_pubkey TEXT DEFAULT NULL, + created_at TEXT DEFAULT (to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"')) + ); + + CREATE TABLE observations ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + transmission_id INTEGER NOT NULL REFERENCES transmissions(id), + observer_idx INTEGER, + direction TEXT, + snr DOUBLE PRECISION, + rssi DOUBLE PRECISION, + score INTEGER, + path_json TEXT, + timestamp INTEGER NOT NULL, + resolved_path TEXT, + raw_hex TEXT + ); + + CREATE TABLE IF NOT EXISTS observer_metrics ( + observer_id TEXT NOT NULL, + timestamp TEXT NOT NULL, + noise_floor DOUBLE PRECISION, + tx_air_secs INTEGER, + rx_air_secs INTEGER, + recv_errors INTEGER, + battery_mv INTEGER, + packets_sent INTEGER, + packets_recv INTEGER, + PRIMARY KEY (observer_id, timestamp) + ); + + CREATE INDEX IF NOT EXISTS idx_observer_metrics_timestamp ON observer_metrics(timestamp); + + -- Auto-populate from_pubkey for ADVERT rows so existing test fixtures + -- (which only set decoded_json) still attribute correctly under #1143's + -- exact-match column. Production migration handles legacy data; the + -- ingestor sets the column at write time. + -- + -- m4 alignment: prod ingest leaves from_pubkey NULL when pubKey is + -- missing or empty (cmd/ingestor/db.go ~1289 guards PubKey != empty-string). + -- The trigger mirrors that: only assign when json_extract yields a + -- non-empty string. json_extract returns NULL for missing keys, so + -- the explicit IS NOT NULL AND <> empty-string guard catches the empty-string + -- case too. UPDATE only when we have something to write. + CREATE FUNCTION test_from_pubkey_advert() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN + IF NEW.from_pubkey IS NULL AND NEW.payload_type = 4 AND NEW.decoded_json IS JSON OBJECT THEN + NEW.from_pubkey := NULLIF(NEW.decoded_json::json ->> 'pubKey', ''); + END IF; + RETURN NEW; + END $$; + CREATE TRIGGER test_from_pubkey_advert BEFORE INSERT ON transmissions + FOR EACH ROW EXECUTE FUNCTION test_from_pubkey_advert(); + CREATE INDEX IF NOT EXISTS idx_transmissions_from_pubkey ON transmissions(from_pubkey); + + -- Mirror prod indexes from internal/dbschema/dbschema.go so query plans + -- in tests match prod. idx_observations_transmission_id is required by + -- GetChannelMessages's grouped MAX(timestamp) per tx aggregate + -- (issue #1366 / PR #1368): without it the perf test on 1500 tx × 50 obs + -- blows the 1.5s budget under -race. + CREATE INDEX IF NOT EXISTS idx_observations_transmission_id ON observations(transmission_id); + CREATE INDEX IF NOT EXISTS idx_observations_timestamp ON observations(timestamp); + CREATE INDEX IF NOT EXISTS idx_observations_tx_ts ON observations(transmission_id, timestamp); + CREATE INDEX IF NOT EXISTS idx_transmissions_channel_hash ON transmissions(channel_hash); + `) if _, err := conn.Exec(schema); err != nil { t.Fatal(err) } - return &DB{conn: conn, isV3: true, hasResolvedPath: true} + return &DB{backend: testBackendValue(), conn: conn, path: dsn, stateDir: t.TempDir(), isV3: true, hasResolvedPath: true} } func seedTestData(t *testing.T, db *DB) { @@ -158,37 +271,37 @@ func seedTestData(t *testing.T, db *DB) { // Seed observers db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obs1', 'Observer One', 'SJC', ?, '2026-01-01T00:00:00Z', 100)`, recent) + VALUES ('obs1', 'Observer One', 'SJC', $1, '2026-01-01T00:00:00Z', 100)`, recent) db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obs2', 'Observer Two', 'SFO', ?, '2026-01-01T00:00:00Z', 50)`, yesterday) + VALUES ('obs2', 'Observer Two', 'SFO', $1, '2026-01-01T00:00:00Z', 50)`, yesterday) // Seed nodes db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES ('aabbccdd11223344', 'TestRepeater', 'repeater', 37.5, -122.0, ?, '2026-01-01T00:00:00Z', 50)`, recent) + VALUES ('aabbccdd11223344', 'TestRepeater', 'repeater', 37.5, -122.0, $1, '2026-01-01T00:00:00Z', 50)`, recent) db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES ('eeff00112233aabb', 'TestCompanion', 'companion', 37.6, -122.1, ?, '2026-01-01T00:00:00Z', 10)`, yesterday) + VALUES ('eeff00112233aabb', 'TestCompanion', 'companion', 37.6, -122.1, $1, '2026-01-01T00:00:00Z', 10)`, yesterday) db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES ('1122334455667788', 'TestRoom', 'room', 37.4, -121.9, ?, '2026-01-01T00:00:00Z', 5)`, twoDaysAgo) + VALUES ('1122334455667788', 'TestRoom', 'room', 37.4, -121.9, $1, '2026-01-01T00:00:00Z', 5)`, twoDaysAgo) // Seed transmissions db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash, from_pubkey) - VALUES ('AABB', 'abc123def4567890', ?, 1, 4, '{"pubKey":"aabbccdd11223344","name":"TestRepeater","type":"ADVERT","timestamp":1700000000,"timestampISO":"2023-11-14T22:13:20.000Z","signature":"abcdef","flags":{"isRepeater":true},"lat":37.5,"lon":-122.0}', '#test', 'aabbccdd11223344')`, recent) + VALUES ('AABB', 'abc123def4567890', $1, 1, 4, '{"pubKey":"aabbccdd11223344","name":"TestRepeater","type":"ADVERT","timestamp":1700000000,"timestampISO":"2023-11-14T22:13:20.000Z","signature":"abcdef","flags":{"isRepeater":true},"lat":37.5,"lon":-122.0}', '#test', 'aabbccdd11223344')`, recent) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash) - VALUES ('CCDD', '1234567890abcdef', ?, 1, 5, '{"type":"CHAN","channel":"#test","text":"Hello: World","sender":"TestUser"}', '#test')`, yesterday) + VALUES ('CCDD', '1234567890abcdef', $1, 1, 5, '{"type":"CHAN","channel":"#test","text":"Hello: World","sender":"TestUser"}', '#test')`, yesterday) // Second ADVERT for same node with different hash_size (raw_hex byte 0x1F → hs=1 vs 0xBB → hs=3) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, from_pubkey) - VALUES ('AA1F', 'def456abc1230099', ?, 1, 4, '{"pubKey":"aabbccdd11223344","name":"TestRepeater","type":"ADVERT","timestamp":1700000100,"timestampISO":"2023-11-14T22:14:40.000Z","signature":"fedcba","flags":{"isRepeater":true},"lat":37.5,"lon":-122.0}', 'aabbccdd11223344')`, yesterday) + VALUES ('AA1F', 'def456abc1230099', $1, 1, 4, '{"pubKey":"aabbccdd11223344","name":"TestRepeater","type":"ADVERT","timestamp":1700000100,"timestampISO":"2023-11-14T22:14:40.000Z","signature":"fedcba","flags":{"isRepeater":true},"lat":37.5,"lon":-122.0}', 'aabbccdd11223344')`, yesterday) // Seed observations (use unix timestamps) // resolved_path contains full pubkeys parallel to path_json hops db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp, resolved_path) - VALUES (1, 1, 12.5, -90, '["aa","bb"]', ?, '["aabbccdd11223344","eeff00112233aabb"]')`, recentEpoch) + VALUES (1, 1, 12.5, -90, '["aa","bb"]', $1, '["aabbccdd11223344","eeff00112233aabb"]')`, recentEpoch) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp, resolved_path) - VALUES (1, 2, 8.0, -95, '["aa"]', ?, '["aabbccdd11223344"]')`, recentEpoch-100) + VALUES (1, 2, 8.0, -95, '["aa"]', $1, '["aabbccdd11223344"]')`, recentEpoch-100) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (2, 1, 15.0, -85, '[]', ?)`, yesterdayEpoch) + VALUES (2, 1, 15.0, -85, '[]', $1)`, yesterdayEpoch) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp, resolved_path) - VALUES (3, 1, 10.0, -92, '["cc"]', ?, '["1122334455667788"]')`, yesterdayEpoch) + VALUES (3, 1, 10.0, -92, '["cc"]', $1, '["1122334455667788"]')`, yesterdayEpoch) } // The topology endpoint takes uniqueNodes from CountActiveNodes, so it must @@ -198,7 +311,7 @@ func TestCountActiveNodes(t *testing.T) { defer db.Close() now := time.Now().UTC() for i, seen := range []time.Time{now.Add(-time.Hour), now.Add(-6 * 24 * time.Hour), now.Add(-8 * 24 * time.Hour)} { - db.conn.Exec(`INSERT INTO nodes (public_key, name, last_seen) VALUES (?, ?, ?)`, + db.conn.Exec(`INSERT INTO nodes (public_key, name, last_seen) VALUES (`+db.parameter(1)+`, `+db.parameter(2)+`, `+db.parameter(3)+`)`, "pk"+strconv.Itoa(i), "n"+strconv.Itoa(i), seen.Format(time.RFC3339)) } @@ -266,8 +379,8 @@ func TestGetDBSizeStats(t *testing.T) { stats := db.GetDBSizeStats() // In-memory DB has dbSizeMB=0 and walSizeMB=0 - if stats["dbSizeMB"] != float64(0) { - t.Errorf("expected dbSizeMB=0 for in-memory DB, got %v", stats["dbSizeMB"]) + if stats["dbSizeMB"].(float64) < 0 || (testBackendValue() == "postgres" && stats["dbSizeMB"].(float64) == 0) { + t.Errorf("expected positive PostgreSQL database size, got %v", stats["dbSizeMB"]) } rows, ok := stats["rows"].(map[string]int) @@ -287,17 +400,15 @@ func TestGetDBSizeStats(t *testing.T) { t.Errorf("expected 2 observers rows, got %d", rows["observers"]) } - // Verify new PRAGMA-based fields - if _, ok := stats["freelistMB"]; !ok { - t.Error("expected freelistMB in DB size stats") + if stats["engine"] != testNativeSQL("sqlite", "postgresql") { + t.Fatal("database engine not identified") } - walPages, ok := stats["walPages"].(map[string]interface{}) - if !ok { - t.Fatal("expected walPages object in DB size stats") - } - for _, key := range []string{"total", "checkpointed", "busy"} { - if _, ok := walPages[key]; !ok { - t.Errorf("expected %s in walPages", key) + for _, obsolete := range []string{"freelistMB", "walPages", "walSizeMB"} { + if testBackendValue() == "sqlite" { + continue + } + if _, ok := stats[obsolete]; ok { + t.Fatalf("invented SQLite metric %s", obsolete) } } } @@ -441,7 +552,7 @@ func TestGetObservers_ExcludesInactive(t *testing.T) { defer db.Close() seedTestData(t, db) // Mark obs2 inactive — soft delete simulating a stale-observer prune. - if _, err := db.conn.Exec(`UPDATE observers SET inactive = 1 WHERE id = ?`, "obs2"); err != nil { + if _, err := db.conn.Exec(`UPDATE observers SET inactive = 1 WHERE id = $1`, "obs2"); err != nil { t.Fatalf("update inactive: %v", err) } observers, err := db.GetObservers() @@ -483,7 +594,7 @@ func TestGetObserverLastPacketAt(t *testing.T) { // Set last_packet_at for obs1 ts := "2026-04-24T12:00:00Z" - db.conn.Exec(`UPDATE observers SET last_packet_at = ? WHERE id = ?`, ts, "obs1") + db.conn.Exec(`UPDATE observers SET last_packet_at = $1 WHERE id = $2`, ts, "obs1") // Verify via GetObservers observers, err := db.GetObservers() @@ -795,7 +906,7 @@ func TestGetObserverIdsForRegion(t *testing.T) { t.Run("case and trim normalization", func(t *testing.T) { db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obs3', 'Observer Three', ' sjc ', ?, '2026-01-01T00:00:00Z', 1)`, time.Now().UTC().Format(time.RFC3339)) + VALUES ('obs3', 'Observer Three', ' sjc ', $1, '2026-01-01T00:00:00Z', 1)`, time.Now().UTC().Format(time.RFC3339)) ids, err := db.GetObserverIdsForRegion(" sjc ") if err != nil { t.Fatal(err) @@ -881,15 +992,15 @@ func TestGetChannelMessagesRegionFiltering(t *testing.T) { db.conn.Exec(`INSERT INTO observers (id, name, iata) VALUES ('obs1', 'Observer One', 'SJC')`) db.conn.Exec(`INSERT INTO observers (id, name, iata) VALUES ('obs2', 'Observer Two', ' sfo ')`) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash) - VALUES ('AA', 'chanregion0001', ?, 1, 5, + VALUES ('AA', 'chanregion0001', $1, 1, 5, '{"type":"CHAN","channel":"#region","text":"SjcUser: One","sender":"SjcUser"}', '#region')`, ts1) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash) - VALUES ('BB', 'chanregion0002', ?, 1, 5, + VALUES ('BB', 'chanregion0002', $1, 1, 5, '{"type":"CHAN","channel":"#region","text":"SfoUser: Two","sender":"SfoUser"}', '#region')`, ts2) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 10.0, -90, '[]', ?)`, epoch1) + VALUES (1, 1, 10.0, -90, '[]', $1)`, epoch1) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (2, 2, 9.0, -91, '[]', ?)`, epoch2) + VALUES (2, 2, 9.0, -91, '[]', $1)`, epoch2) msgsSJC, totalSJC, err := db.GetChannelMessages("#region", 100, 0, " sjc ") if err != nil { @@ -1226,13 +1337,13 @@ func TestGetNodesFiltering(t *testing.T) { // where observations use observer_id TEXT instead of observer_idx INTEGER. func setupTestDBV2(t *testing.T) *DB { t.Helper() - conn, err := sql.Open("sqlite3", ":memory:") + conn, err := openPostgresTestDB(t) if err != nil { t.Fatal(err) } conn.SetMaxOpenConns(1) - schema := ` + schema := testNativeSQL(` CREATE TABLE nodes ( public_key TEXT PRIMARY KEY, name TEXT, @@ -1298,12 +1409,76 @@ func setupTestDBV2(t *testing.T) *DB { WHERE id = NEW.id; END; CREATE INDEX IF NOT EXISTS idx_transmissions_from_pubkey ON transmissions(from_pubkey); - ` + `, ` + CREATE TABLE nodes ( + public_key TEXT PRIMARY KEY, + name TEXT, + role TEXT, + lat DOUBLE PRECISION, + lon DOUBLE PRECISION, + last_seen TEXT, + first_seen TEXT, + advert_count INTEGER DEFAULT 0, + battery_mv INTEGER, + temperature_c DOUBLE PRECISION, + foreign_advert INTEGER DEFAULT 0 + ); + + CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY UNIQUE, + id TEXT PRIMARY KEY, + name TEXT, + iata TEXT, + last_seen TEXT, + first_seen TEXT, + packet_count INTEGER DEFAULT 0, + last_packet_at TEXT DEFAULT NULL, + inactive INTEGER DEFAULT 0 + ); + + CREATE TABLE transmissions ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + raw_hex TEXT NOT NULL, + hash TEXT NOT NULL UNIQUE, + first_seen TEXT NOT NULL, + route_type INTEGER, + payload_type INTEGER, + payload_version INTEGER, + decoded_json TEXT, + channel_hash TEXT DEFAULT NULL, + from_pubkey TEXT DEFAULT NULL, + created_at TEXT DEFAULT (to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"')) + ); + + CREATE TABLE observations ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + transmission_id INTEGER NOT NULL REFERENCES transmissions(id), + observer_id TEXT, + observer_name TEXT, + direction TEXT, + snr DOUBLE PRECISION, + rssi DOUBLE PRECISION, + score INTEGER, + path_json TEXT, + timestamp INTEGER NOT NULL, + raw_hex TEXT + ); + + CREATE FUNCTION test_from_pubkey_advert() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN + IF NEW.from_pubkey IS NULL AND NEW.payload_type = 4 AND NEW.decoded_json IS JSON OBJECT THEN + NEW.from_pubkey := NULLIF(NEW.decoded_json::json ->> 'pubKey', ''); + END IF; + RETURN NEW; + END $$; + CREATE TRIGGER test_from_pubkey_advert BEFORE INSERT ON transmissions + FOR EACH ROW EXECUTE FUNCTION test_from_pubkey_advert(); + CREATE INDEX IF NOT EXISTS idx_transmissions_from_pubkey ON transmissions(from_pubkey); + `) if _, err := conn.Exec(schema); err != nil { t.Fatal(err) } - return &DB{conn: conn, isV3: false} + return &DB{backend: testBackendValue(), conn: conn, isV3: false} } func TestGetNodesRegionFilterV2(t *testing.T) { @@ -1316,19 +1491,19 @@ func TestGetNodesRegionFilterV2(t *testing.T) { // Seed observer with IATA code db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obs-v2-1', 'V2 Observer', 'LAX', ?, '2026-01-01T00:00:00Z', 10)`, recent) + VALUES ('obs-v2-1', 'V2 Observer', 'LAX', $1, '2026-01-01T00:00:00Z', 10)`, recent) // Seed a node db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES ('v2pubkey11223344', 'V2Node', 'repeater', 34.0, -118.0, ?, '2026-01-01T00:00:00Z', 5)`, recent) + VALUES ('v2pubkey11223344', 'V2Node', 'repeater', 34.0, -118.0, $1, '2026-01-01T00:00:00Z', 5)`, recent) // Seed an ADVERT transmission for the node db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('AABB', 'v2hash0001', ?, 1, 4, '{"pubKey":"v2pubkey11223344","name":"V2Node","type":"ADVERT"}')`, recent) + VALUES ('AABB', 'v2hash0001', $1, 1, 4, '{"pubKey":"v2pubkey11223344","name":"V2Node","type":"ADVERT"}')`, recent) // Seed v2-style observation: observer_id references observers.id directly db.conn.Exec(`INSERT INTO observations (transmission_id, observer_id, observer_name, snr, rssi, path_json, timestamp) - VALUES (1, 'obs-v2-1', 'V2 Observer', 10.0, -90, '[]', ?)`, recentEpoch) + VALUES (1, 'obs-v2-1', 'V2 Observer', 10.0, -90, '[]', $1)`, recentEpoch) t.Run("v2 region filter match", func(t *testing.T) { nodes, total, _, err := db.GetNodes(NodeQuery{Limit: 50, Region: "LAX"}) @@ -1439,9 +1614,9 @@ func TestGetNetworkStatusDateFormats(t *testing.T) { // Insert nodes with different date formats db.conn.Exec(`INSERT INTO nodes (public_key, name, role, last_seen) - VALUES ('node1111', 'NodeRFC', 'repeater', ?)`, time.Now().Format(time.RFC3339)) + VALUES ('node1111', 'NodeRFC', 'repeater', $1)`, time.Now().Format(time.RFC3339)) db.conn.Exec(`INSERT INTO nodes (public_key, name, role, last_seen) - VALUES ('node2222', 'NodeSQL', 'companion', ?)`, time.Now().Format("2006-01-02 15:04:05")) + VALUES ('node2222', 'NodeSQL', 'companion', $1)`, time.Now().Format("2006-01-02 15:04:05")) db.conn.Exec(`INSERT INTO nodes (public_key, name, role, last_seen) VALUES ('node3333', 'NodeNull', 'room', NULL)`) db.conn.Exec(`INSERT INTO nodes (public_key, name, role, last_seen) @@ -1479,15 +1654,14 @@ func TestGetNetworkStatusDateFormats(t *testing.T) { func TestOpenDBValid(t *testing.T) { // Create a real SQLite database file - dir := t.TempDir() - dbPath := filepath.Join(dir, "test.db") + dbPath := postgresTestDSN(t) // Create DB with a table using a writable connection first - conn, err := sql.Open("sqlite3", dbPath) + conn, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } - _, err = conn.Exec(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY, hash TEXT)`) + _, err = conn.Exec(testNativeSQL(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY AUTOINCREMENT, hash TEXT)`, `CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, hash TEXT)`)) if err != nil { conn.Close() t.Fatal(err) @@ -1496,7 +1670,7 @@ func TestOpenDBValid(t *testing.T) { conn.Close() // Now test OpenDB (read-only) - database, err := OpenDB(dbPath) + database, err := openFixtureReader(t, dbPath) if err != nil { t.Fatalf("OpenDB failed: %v", err) } @@ -1521,16 +1695,15 @@ func TestOpenDBInvalidPath(t *testing.T) { // The existing ScopeStats tests set these flags manually — this test ensures // the flag-setting code itself is covered. func TestDetectSchemaScopeName(t *testing.T) { - dir := t.TempDir() - dbPath := filepath.Join(dir, "detect.db") + dbPath := postgresTestDSN(t) // Create file-based DB with the scope_name and default_scope columns. - conn, err := sql.Open("sqlite3", dbPath) + conn, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } conn.SetMaxOpenConns(1) - if _, err := conn.Exec(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY, hash TEXT, scope_name TEXT)`); err != nil { + if _, err := conn.Exec(testNativeSQL(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY AUTOINCREMENT, hash TEXT, scope_name TEXT)`, `CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, hash TEXT, scope_name TEXT)`)); err != nil { conn.Close() t.Fatalf("create transmissions: %v", err) } @@ -1538,14 +1711,14 @@ func TestDetectSchemaScopeName(t *testing.T) { conn.Close() t.Fatalf("create nodes: %v", err) } - if _, err := conn.Exec(`CREATE TABLE observations (id INTEGER PRIMARY KEY)`); err != nil { + if _, err := conn.Exec(testNativeSQL(`CREATE TABLE observations (id INTEGER PRIMARY KEY AUTOINCREMENT)`, `CREATE TABLE observations (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY)`)); err != nil { conn.Close() t.Fatalf("create observations: %v", err) } ensurePreparable(t, conn) conn.Close() - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatalf("OpenDB: %v", err) } @@ -1559,19 +1732,19 @@ func TestDetectSchemaScopeName(t *testing.T) { } // Verify the flags stay false when the columns are absent. - dbPath2 := filepath.Join(dir, "detect2.db") - conn2, err := sql.Open("sqlite3", dbPath2) + dbPath2 := postgresTestDSN(t) + conn2, err := openFixtureSQL(dbPath2) if err != nil { t.Fatal(err) } conn2.SetMaxOpenConns(1) - conn2.Exec(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY, hash TEXT)`) + conn2.Exec(testNativeSQL(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY AUTOINCREMENT, hash TEXT)`, `CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, hash TEXT)`)) conn2.Exec(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY)`) - conn2.Exec(`CREATE TABLE observations (id INTEGER PRIMARY KEY)`) + conn2.Exec(testNativeSQL(`CREATE TABLE observations (id INTEGER PRIMARY KEY AUTOINCREMENT)`, `CREATE TABLE observations (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY)`)) ensurePreparable(t, conn2) conn2.Close() - db2, err := OpenDB(dbPath2) + db2, err := openFixtureReader(t, dbPath2) if err != nil { t.Fatalf("OpenDB2: %v", err) } @@ -1841,6 +2014,9 @@ func TestNodeTelemetryFields(t *testing.T) { } func TestMain(m *testing.M) { + // Server fixtures need stable build metadata, not a Git subprocess per + // instance. TestResolveCommit still exercises the actual fallback. + Commit = "unit-test" os.Exit(m.Run()) } @@ -1853,13 +2029,13 @@ func TestGetObserverMetrics(t *testing.T) { t2 := now.Add(-1 * time.Hour).Format(time.RFC3339) t3 := now.Format(time.RFC3339) - db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors, battery_mv) VALUES (?, ?, ?, ?, ?, ?, ?)", + db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors, battery_mv) VALUES ($1, $2, $3, $4, $5, $6, $7)", "obs1", t1, -112.5, 100, 500, 3, 3720) - db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors) VALUES (?, ?, ?, ?, ?, ?)", + db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors) VALUES ($1, $2, $3, $4, $5, $6)", "obs1", t2, -110.0, 200, 800, 5) - db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors) VALUES (?, ?, ?, ?, ?, ?)", + db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors) VALUES ($1, $2, $3, $4, $5, $6)", "obs1", t3, -108.0, 300, 1100, 8) - db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor) VALUES (?, ?, ?)", + db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor) VALUES ($1, $2, $3)", "obs2", t1, -115.0) // Query all for obs1 @@ -1910,11 +2086,11 @@ func TestGetMetricsSummary(t *testing.T) { t1 := now.Add(-2 * time.Hour).Format(time.RFC3339) t2 := now.Add(-1 * time.Hour).Format(time.RFC3339) - db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, battery_mv) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, battery_mv) VALUES ($1, $2, $3, $4)", "obs1", t1, -112.0, 3720) - db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor) VALUES (?, ?, ?)", + db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor) VALUES ($1, $2, $3)", "obs1", t2, -108.0) - db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor) VALUES (?, ?, ?)", + db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor) VALUES ($1, $2, $3)", "obs2", t1, -115.0) since := now.Add(-24 * time.Hour).Format(time.RFC3339) @@ -1960,7 +2136,7 @@ func TestObserverMetricsAPIEndpoints(t *testing.T) { now := time.Now().UTC() t1 := now.Add(-1 * time.Hour).Format(time.RFC3339) - db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor) VALUES (?, ?, ?)", + db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor) VALUES ($1, $2, $3)", "obs1", t1, -112.0) // Query directly to verify @@ -2084,13 +2260,13 @@ func TestGetObserverMetricsResolution(t *testing.T) { db := setupTestDB(t) seedTestData(t, db) - db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs) VALUES ($1, $2, $3, $4)", "obs1", "2026-04-05T00:00:00Z", -112.0, 100) - db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs) VALUES ($1, $2, $3, $4)", "obs1", "2026-04-05T00:05:00Z", -110.0, 200) - db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs) VALUES ($1, $2, $3, $4)", "obs1", "2026-04-05T01:00:00Z", -108.0, 500) - db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs) VALUES ($1, $2, $3, $4)", "obs1", "2026-04-05T01:05:00Z", -106.0, 600) // 5m resolution: all 4 rows @@ -2127,9 +2303,9 @@ func TestHourlyResolutionDeltasNotNull(t *testing.T) { // Two hourly buckets, each with one sample. With old MAX+hardcoded gap threshold, // the 3600s gap would exceed sampleInterval*2 (600s) and deltas would be null. - db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors, packets_sent, packets_recv) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors, packets_sent, packets_recv) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)", "obs_hr", "2026-04-05T10:00:00Z", -110.0, 100, 200, 5, 50, 100) - db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors, packets_sent, packets_recv) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors, packets_sent, packets_recv) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)", "obs_hr", "2026-04-05T11:00:00Z", -108.0, 200, 400, 10, 80, 200) m, _, err := db.GetObserverMetrics("obs_hr", "2026-04-04T00:00:00Z", "", "1h", 300) @@ -2152,15 +2328,15 @@ func TestLastValuePreservesReboot(t *testing.T) { // Hour bucket with two samples: pre-reboot (high) and post-reboot (low). // With MAX(), the pre-reboot value wins and the reboot is hidden. // With LAST (latest timestamp), the post-reboot value wins. - db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors, packets_sent, packets_recv) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors, packets_sent, packets_recv) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)", "obs_rb", "2026-04-05T10:00:00Z", -110.0, 1000, 2000, 500, 400, 800) // pre-reboot baseline - db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors, packets_sent, packets_recv) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors, packets_sent, packets_recv) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)", "obs_rb", "2026-04-05T10:20:00Z", -110.0, 5000, 6000, 900, 700, 1200) // pre-reboot peak - db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors, packets_sent, packets_recv) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors, packets_sent, packets_recv) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)", "obs_rb", "2026-04-05T10:40:00Z", -110.0, 10, 20, 1, 5, 10) // post-reboot (counter reset) // Next hour bucket - db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors, packets_sent, packets_recv) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs, recv_errors, packets_sent, packets_recv) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)", "obs_rb", "2026-04-05T11:00:00Z", -108.0, 100, 120, 5, 20, 50) m, reboots, err := db.GetObserverMetrics("obs_rb", "2026-04-04T00:00:00Z", "", "1h", 300) @@ -2224,14 +2400,14 @@ func TestPerObservationRawHexEnrich(t *testing.T) { // Insert transmission with raw_hex txHex := "deadbeef" - db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen) VALUES (?, 'hash1', '2026-04-21T10:00:00Z')`, txHex) + db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen) VALUES ($1, 'hash1', '2026-04-21T10:00:00Z')`, txHex) // Insert two observations: A has its own raw_hex, B has NULL (historical) obsAHex := "c0ffee01" db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp, raw_hex) - VALUES (1, ?, -5.0, -90.0, '[]', 1745236800, ?)`, rowA, obsAHex) + VALUES (1, $1, -5.0, -90.0, '[]', 1745236800, $2)`, rowA, obsAHex) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, ?, -3.0, -85.0, '["aabb"]', 1745236801)`, rowB) + VALUES (1, $1, -3.0, -85.0, '["aabb"]', 1745236801)`, rowB) store := NewPacketStore(db, nil) if err := store.Load(); err != nil { @@ -2265,36 +2441,41 @@ func TestPerObservationRawHexEnrich(t *testing.T) { } func TestGetScopeStats(t *testing.T) { - conn, err := sql.Open("sqlite3", ":memory:") + conn, err := openPostgresTestDB(t) if err != nil { t.Fatalf("sql.Open: %v", err) } conn.SetMaxOpenConns(1) - db := &DB{conn: conn} + db := &DB{backend: testBackendValue(), conn: conn} defer db.conn.Close() // Create minimal schema - db.conn.Exec(`CREATE TABLE IF NOT EXISTS transmissions ( + db.conn.Exec(testNativeSQL(`CREATE TABLE IF NOT EXISTS transmissions ( id INTEGER PRIMARY KEY AUTOINCREMENT, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT, scope_name TEXT DEFAULT NULL, from_pubkey TEXT DEFAULT NULL - )`) + )`, `CREATE TABLE IF NOT EXISTS transmissions ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, + payload_type INTEGER, payload_version INTEGER, decoded_json TEXT, + scope_name TEXT DEFAULT NULL, from_pubkey TEXT DEFAULT NULL + )`)) db.conn.Exec(`CREATE TABLE IF NOT EXISTS nodes (public_key TEXT PRIMARY KEY, role TEXT)`) // Manually set hasScopeName since we bypassed the detector db.hasScopeName = true now := time.Now().UTC().Format(time.RFC3339) // Transport scoped, known region - db.conn.Exec(`INSERT INTO transmissions (hash, first_seen, route_type, scope_name) VALUES ('a', ?, 0, '#belgium')`, now) + db.conn.Exec(`INSERT INTO transmissions (hash, first_seen, route_type, scope_name) VALUES ('a', $1, 0, '#belgium')`, now) // Transport scoped, unknown - db.conn.Exec(`INSERT INTO transmissions (hash, first_seen, route_type, scope_name) VALUES ('b', ?, 0, '')`, now) + db.conn.Exec(`INSERT INTO transmissions (hash, first_seen, route_type, scope_name) VALUES ('b', $1, 0, '')`, now) // Transport unscoped (NULL) - db.conn.Exec(`INSERT INTO transmissions (hash, first_seen, route_type, scope_name) VALUES ('c', ?, 0, NULL)`, now) + db.conn.Exec(`INSERT INTO transmissions (hash, first_seen, route_type, scope_name) VALUES ('c', $1, 0, NULL)`, now) // Non-transport FLOOD (route_type=1) — inherently unscoped per MeshCore protocol (#1838) - db.conn.Exec(`INSERT INTO transmissions (hash, first_seen, route_type, scope_name) VALUES ('d', ?, 1, NULL)`, now) + db.conn.Exec(`INSERT INTO transmissions (hash, first_seen, route_type, scope_name) VALUES ('d', $1, 1, NULL)`, now) // Non-transport DIRECT (route_type=2) — inherently unscoped per MeshCore protocol (#1838) - db.conn.Exec(`INSERT INTO transmissions (hash, first_seen, route_type, scope_name) VALUES ('e', ?, 2, NULL)`, now) + db.conn.Exec(`INSERT INTO transmissions (hash, first_seen, route_type, scope_name) VALUES ('e', $1, 2, NULL)`, now) stats, err := db.GetScopeStats("24h") if err != nil { @@ -2325,20 +2506,24 @@ func TestGetScopeStats(t *testing.T) { // scope_name states (NULL is unscoped, an empty string is unknown scope, a // name is a named scope). func TestGetScopeStatsAdvertsByRole(t *testing.T) { - conn, err := sql.Open("sqlite3", ":memory:") + conn, err := openPostgresTestDB(t) if err != nil { t.Fatalf("sql.Open: %v", err) } conn.SetMaxOpenConns(1) - db := &DB{conn: conn, hasScopeName: true} + db := &DB{backend: testBackendValue(), conn: conn, hasScopeName: true} defer db.conn.Close() for _, stmt := range []string{ - `CREATE TABLE transmissions ( + testNativeSQL(`CREATE TABLE transmissions ( id INTEGER PRIMARY KEY AUTOINCREMENT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, scope_name TEXT DEFAULT NULL, from_pubkey TEXT DEFAULT NULL - )`, + )`, `CREATE TABLE transmissions ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, + scope_name TEXT DEFAULT NULL, from_pubkey TEXT DEFAULT NULL + )`), `CREATE TABLE nodes (public_key TEXT PRIMARY KEY, role TEXT)`, `INSERT INTO nodes (public_key, role) VALUES ('rpt', 'repeater'), ('cmp', 'companion'), ('sns', 'sensor'), ('norole', '')`, } { @@ -2385,7 +2570,7 @@ func TestGetScopeStatsAdvertsByRole(t *testing.T) { } for _, r := range rows { if _, err := db.conn.Exec( - `INSERT INTO transmissions (hash, first_seen, route_type, payload_type, scope_name, from_pubkey) VALUES (?, ?, ?, ?, ?, ?)`, + `INSERT INTO transmissions (hash, first_seen, route_type, payload_type, scope_name, from_pubkey) VALUES ($1, $2, $3, $4, $5, $6)`, r.hash, r.seen, r.route, r.ptype, r.scope, r.fromKey, ); err != nil { t.Fatalf("seed %s: %v", r.hash, err) @@ -2409,14 +2594,14 @@ func TestGetScopeStatsAdvertsByRole(t *testing.T) { } func TestGetScopeStatsAdvertsByRoleEmpty(t *testing.T) { - conn, err := sql.Open("sqlite3", ":memory:") + conn, err := openPostgresTestDB(t) if err != nil { t.Fatalf("sql.Open: %v", err) } conn.SetMaxOpenConns(1) - db := &DB{conn: conn, hasScopeName: true} + db := &DB{backend: testBackendValue(), conn: conn, hasScopeName: true} defer db.conn.Close() - db.conn.Exec(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY, first_seen TEXT, route_type INTEGER, payload_type INTEGER, scope_name TEXT, from_pubkey TEXT)`) + db.conn.Exec(testNativeSQL(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY AUTOINCREMENT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, scope_name TEXT, from_pubkey TEXT)`, `CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, first_seen TEXT, route_type INTEGER, payload_type INTEGER, scope_name TEXT, from_pubkey TEXT)`)) db.conn.Exec(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY, role TEXT)`) stats, err := db.GetScopeStats("1h") @@ -2447,12 +2632,12 @@ func TestLoadIndexesRelayHopsFromResolvedPath(t *testing.T) { const senderPubkey = "sender00112233445566" db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('FF01', 'relaytest0001hash', ?, 1, 4, ?)`, + VALUES ('FF01', 'relaytest0001hash', $1, 1, 4, $2)`, now, `{"pubKey":"`+senderPubkey+`","name":"Sender","type":"ADVERT"}`) // Observer hears the packet via the relay node. db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp, resolved_path) - VALUES (1, 1, 10.0, -90, '["rr"]', ?, ?)`, + VALUES (1, 1, 10.0, -90, '["rr"]', $1, $2)`, epoch, `["`+relayPubkey+`"]`) store := NewPacketStore(db, nil) @@ -2483,7 +2668,7 @@ type failingQuerier struct { } func (f *failingQuerier) QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) { - if f.failOn != "" && strings.Contains(query, f.failOn) { + if f.failOn != "" && (strings.Contains(query, f.failOn) || (len(args) > 0 && args[0] == f.failOn)) { return nil, f.err } return f.real.QueryContext(ctx, query, args...) @@ -2494,28 +2679,28 @@ func (f *failingQuerier) QueryContext(ctx context.Context, query string, args .. // instead of being silently swallowed and leaving the server in v2 mode against // a v3 database for the whole process lifetime. func TestDetectSchemaFailsLoudOnProbeError(t *testing.T) { - dbPath := filepath.Join(t.TempDir(), "v3.db") - conn, err := sql.Open("sqlite3", dbPath) + dbPath := postgresTestDSN(t) + conn, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } conn.SetMaxOpenConns(1) // v3-shaped: observations carries observer_idx. - if _, err := conn.Exec(`CREATE TABLE observations (id INTEGER PRIMARY KEY, observer_idx INTEGER)`); err != nil { + if _, err := conn.Exec(testNativeSQL(`CREATE TABLE observations (id INTEGER PRIMARY KEY AUTOINCREMENT, observer_idx INTEGER)`, `CREATE TABLE observations (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, observer_idx INTEGER)`)); err != nil { conn.Close() t.Fatal(err) } - conn.Exec(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY)`) + conn.Exec(testNativeSQL(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY AUTOINCREMENT)`, `CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY)`)) conn.Exec(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY)`) conn.Close() - real, err := sql.Open("sqlite3", "file:"+dbPath+"?mode=ro") + real, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } defer real.Close() - q := &failingQuerier{real: real, failOn: "table_info(observations)", err: errors.New("injected probe failure")} + q := &failingQuerier{real: real, failOn: "observations", err: errors.New("injected probe failure")} db := &DB{} if err := db.detectSchema(context.Background(), q); err == nil { t.Fatal("detectSchema must return an error when a schema probe fails (#1901)") @@ -2528,20 +2713,19 @@ func TestDetectSchemaFailsLoudOnProbeError(t *testing.T) { // TestDetectSchemaV3AndV2 verifies detection sets isV3 correctly for both schema // shapes via the real OpenDB path, and that both open cleanly. func TestDetectSchemaV3AndV2(t *testing.T) { - dir := t.TempDir() - v3 := filepath.Join(dir, "v3.db") - c, err := sql.Open("sqlite3", v3) + v3 := postgresTestDSN(t) + c, err := openFixtureSQL(v3) if err != nil { t.Fatal(err) } c.SetMaxOpenConns(1) - c.Exec(`CREATE TABLE observations (id INTEGER PRIMARY KEY, observer_idx INTEGER)`) - c.Exec(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY, hash TEXT)`) + c.Exec(testNativeSQL(`CREATE TABLE observations (id INTEGER PRIMARY KEY AUTOINCREMENT, observer_idx INTEGER)`, `CREATE TABLE observations (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, observer_idx INTEGER)`)) + c.Exec(testNativeSQL(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY AUTOINCREMENT, hash TEXT)`, `CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, hash TEXT)`)) c.Exec(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY)`) ensurePreparable(t, c) c.Close() - db, err := OpenDB(v3) + db, err := openFixtureReader(t, v3) if err != nil { t.Fatalf("OpenDB v3: %v", err) } @@ -2550,18 +2734,18 @@ func TestDetectSchemaV3AndV2(t *testing.T) { } db.Close() - v2 := filepath.Join(dir, "v2.db") - c2, err := sql.Open("sqlite3", v2) + v2 := postgresTestDSN(t) + c2, err := openFixtureSQL(v2) if err != nil { t.Fatal(err) } c2.SetMaxOpenConns(1) - c2.Exec(`CREATE TABLE observations (id INTEGER PRIMARY KEY, observer_id INTEGER)`) - c2.Exec(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY, hash TEXT)`) + c2.Exec(testNativeSQL(`CREATE TABLE observations (id INTEGER PRIMARY KEY AUTOINCREMENT, observer_id INTEGER)`, `CREATE TABLE observations (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, observer_id INTEGER)`)) + c2.Exec(testNativeSQL(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY AUTOINCREMENT, hash TEXT)`, `CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, hash TEXT)`)) c2.Exec(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY)`) ensurePreparable(t, c2) c2.Close() - db2, err := OpenDB(v2) + db2, err := openFixtureReader(t, v2) if err != nil { t.Fatalf("OpenDB v2: %v", err) } @@ -2635,7 +2819,7 @@ func TestGetChannelsPreviewRespectsRegionFilter(t *testing.T) { // #2146: /api/perf counted every row of transmissions, observations, nodes // and observers on each call (116 ms for the observations count on an -// 18M-row database). The counts are cached for perfRowCountsTTL. +// 18M-row database). The counts are cached for postgresStatsTTL. func TestGetDBSizeStatsTypedCachesRowCounts(t *testing.T) { db := setupTestDB(t) defer db.Close() @@ -2647,9 +2831,9 @@ func TestGetDBSizeStatsTypedCachesRowCounts(t *testing.T) { t.Fatalf("within the TTL the cached count must be served: got %d, want %d", got, first) } - db.rowCountsMu.Lock() - db.rowCountsAt = time.Now().Add(-2 * perfRowCountsTTL) - db.rowCountsMu.Unlock() + db.pgStatsMu.Lock() + db.pgStatsExpires = time.Now().Add(-2 * postgresStatsTTL) + db.pgStatsMu.Unlock() if got := db.GetDBSizeStatsTyped().Rows.Transmissions; got != first+1 { t.Fatalf("after the TTL the count must be refreshed: got %d, want %d", got, first+1) } @@ -2663,9 +2847,9 @@ func TestGetDBSizeStatsTypedDoesNotCacheFailedCounts(t *testing.T) { t.Fatal(err) } db.GetDBSizeStatsTyped() - db.rowCountsMu.Lock() - cached := db.rowCounts != nil - db.rowCountsMu.Unlock() + db.pgStatsMu.Lock() + cached := !db.pgStatsCache.SampledAt.IsZero() + db.pgStatsMu.Unlock() if cached { t.Fatal("row counts with a failed COUNT(*) must not be cached") } diff --git a/cmd/server/decoded_rewrite_race_test.go b/cmd/server/decoded_rewrite_race_test.go new file mode 100644 index 000000000..9bba9567e --- /dev/null +++ b/cmd/server/decoded_rewrite_race_test.go @@ -0,0 +1,172 @@ +package main + +import ( + "encoding/json" + "fmt" + "runtime" + "sync" + "testing" + "time" +) + +func TestTxRewriteFeedConcurrentDecodedReaders(t *testing.T) { + db, writer := txRewriteFixture(t, true) + s := NewPacketStore(db, &PacketStoreConfig{}) + if err := s.Load(); err != nil { + t.Fatal(err) + } + tx := s.byTxID[1] + original := `{"type":"CHAN","channel":"#initial","flags":{"revision":"initial"}}` + backfillRewrite(t, writer, original, "#initial") + if err := s.pollTxRewrites(500); err != nil { + t.Fatal(err) + } + retained := tx.ParsedDecoded() + stop := make(chan struct{}) + var ready, readers sync.WaitGroup + ready.Add(4) + readers.Add(4) + for range 4 { + go func() { + defer readers.Done() + tx.ParsedDecoded() + ready.Done() + for { + select { + case <-stop: + return + default: + d := tx.ParsedDecoded() + if d == nil || d["type"] != "CHAN" { + t.Error("rewrite exposed an empty or partially parsed cache") + return + } + if flags, ok := d["flags"].(map[string]interface{}); !ok || flags["revision"] == nil { + t.Error("rewrite exposed a partially parsed nested map") + return + } + runtime.Gosched() + } + } + }() + } + ready.Wait() + defer func() { + close(stop) + done := make(chan struct{}) + go func() { readers.Wait(); close(done) }() + select { + case <-done: + case <-time.After(10 * time.Second): + t.Error("decoded readers blocked after rewrite") + } + }() + rewritten := make(chan struct{}) + go func() { + defer close(rewritten) + for revision := 0; revision < 24; revision++ { + decoded := fmt.Sprintf(`{"type":"CHAN","channel":"#revision-%d","flags":{"revision":"%d"}}`, revision, revision) + backfillRewrite(t, writer, decoded, "#updated") + if err := s.pollTxRewrites(500); err != nil { + t.Error(err) + return + } + want := fmt.Sprintf("#revision-%d", revision) + if got := tx.ParsedDecoded()["channel"]; got != want { + t.Errorf("rewrite cache stale: got %v, want %s", got, want) + return + } + if retained["channel"] != "#initial" || retained["flags"].(map[string]interface{})["revision"] != "initial" { + t.Error("rewrite mutated a map retained by an earlier reader") + return + } + } + }() + select { + case <-rewritten: + case <-time.After(10 * time.Second): + t.Fatal("rewrite/cache reader did not finish") + } +} + +// Model a reader already parsing an old generation. A writer holding store.mu +// must not wait for it, and its eventual completion must not replace new data. +func TestDecodedRewriteDoesNotWaitForOldParse(t *testing.T) { + const oldJSON = `{"type":"old"}` + tx := &StoreTx{DecodedJSON: oldJSON} + old := &txDecodedCache{source: oldJSON} + tx.decodedCache.Store(old) + entered, release, parsed := make(chan struct{}), make(chan struct{}), make(chan struct{}) + go func() { + defer close(parsed) + old.once.Do(func() { + close(entered) + <-release + json.Unmarshal([]byte(old.source), &old.parsed) + }) + }() + <-entered + var releaseOnce sync.Once + finishParse := func() { releaseOnce.Do(func() { close(release) }); <-parsed } + defer finishParse() + rewritten := make(chan struct{}) + go func() { + tx.setDecodedJSON(`{"type":"new"}`) + close(rewritten) + }() + select { + case <-rewritten: + case <-time.After(time.Second): + t.Fatal("rewrite waited for an old generation's JSON parse") + } + current := tx.ParsedDecoded() + if current["type"] != "new" { + t.Fatal("new generation did not parse the rewritten source") + } + cache := tx.decodedCache.Load() + if tx.setDecodedJSON(`{"type":"new"}`) || tx.decodedCache.Load() != cache { + t.Fatal("unchanged JSON discarded its warm cache") + } + finishParse() + if old.parsed["type"] != "old" || tx.ParsedDecoded()["type"] != "new" { + t.Fatal("old parser completion replaced or mutated the new generation") + } +} + +func TestDecodedGenerationIncludedInMemoryBudget(t *testing.T) { + tx := &StoreTx{} + before := estimateStoreTxBytes(tx) + if before < int64(storeTxBaseBytes+numIndexesPerTx*indexEntryBytes+decodedCacheGenerationBytes) { + t.Fatal("cache generation is not charged before parsing") + } + allocs := testing.AllocsPerRun(100, func() { + if tx.ParsedDecoded() != nil { + t.Fatal("empty source must stay empty") + } + }) + if allocs != 0 { + t.Fatalf("warm decoded cache allocates: %v", allocs) + } +} + +func BenchmarkParsedDecodedWarm(b *testing.B) { + tx := &StoreTx{DecodedJSON: decodedCacheFixture} + tx.ParsedDecoded() + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + if tx.ParsedDecoded()["type"] != "ADVERT" { + b.Fatal("cached decoded payload changed") + } + } +} + +func BenchmarkParsedDecodedCold(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + tx := &StoreTx{DecodedJSON: decodedCacheFixture} + if tx.ParsedDecoded()["type"] != "ADVERT" { + b.Fatal("decoded payload changed") + } + } +} diff --git a/cmd/server/decoder.go b/cmd/server/decoder.go index 8f7981ef5..875e6052a 100644 --- a/cmd/server/decoder.go +++ b/cmd/server/decoder.go @@ -1,7 +1,6 @@ package main import ( - "crypto/sha256" "encoding/binary" "encoding/hex" "encoding/json" @@ -669,55 +668,7 @@ func DecodePacket(hexString string, validateSignatures bool) (*DecodedPacket, er // route-independent identifier for the same logical packet. For TRACE packets, // path_len is included in the hash to match firmware behavior. func ComputeContentHash(rawHex string) string { - buf, err := hex.DecodeString(rawHex) - if err != nil || len(buf) < 2 { - if len(rawHex) >= 16 { - return rawHex[:16] - } - return rawHex - } - - headerByte := buf[0] - offset := 1 - if isTransportRoute(int(headerByte & 0x03)) { - offset += 4 - } - if offset >= len(buf) { - if len(rawHex) >= 16 { - return rawHex[:16] - } - return rawHex - } - pathByte := buf[offset] - offset++ - hashSize := int((pathByte>>6)&0x3) + 1 - hashCount := int(pathByte & 0x3F) - pathBytes := hashSize * hashCount - - payloadStart := offset + pathBytes - if payloadStart > len(buf) { - if len(rawHex) >= 16 { - return rawHex[:16] - } - return rawHex - } - - payload := buf[payloadStart:] - - // Hash payload-type byte only (bits 2-5 of header), not the full header. - // Firmware: SHA256(payload_type + [path_len for TRACE] + payload) - // Using the full header caused different hashes for the same logical packet - // when route type or version bits differed. See issue #786. - payloadType := (headerByte >> 2) & 0x0F - toHash := []byte{payloadType} - if int(payloadType) == PayloadTRACE { - // Firmware uses uint16_t path_len (2 bytes, little-endian) - toHash = append(toHash, pathByte, 0x00) - } - toHash = append(toHash, payload...) - - h := sha256.Sum256(toHash) - return hex.EncodeToString(h[:])[:16] + return packetpath.ContentHash(rawHex) } // PayloadJSON serializes the payload to JSON for DB storage. diff --git a/cmd/server/encrypted_channels_test.go b/cmd/server/encrypted_channels_test.go index d8632b886..2c19b296b 100644 --- a/cmd/server/encrypted_channels_test.go +++ b/cmd/server/encrypted_channels_test.go @@ -16,15 +16,15 @@ func seedEncryptedChannelData(t *testing.T, db *DB) { // Two encrypted GRP_TXT packets on channel hash "A1B2" db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash) - VALUES ('EE01', 'enc_hash_001', ?, 1, 5, '{"type":"GRP_TXT","channelHashHex":"A1B2","decryptionStatus":"no_key"}', 'enc_A1B2')`, recent) + VALUES ('EE01', 'enc_hash_001', $1, 1, 5, '{"type":"GRP_TXT","channelHashHex":"A1B2","decryptionStatus":"no_key"}', 'enc_A1B2')`, recent) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash) - VALUES ('EE02', 'enc_hash_002', ?, 1, 5, '{"type":"GRP_TXT","channelHashHex":"A1B2","decryptionStatus":"no_key"}', 'enc_A1B2')`, recent) + VALUES ('EE02', 'enc_hash_002', $1, 1, 5, '{"type":"GRP_TXT","channelHashHex":"A1B2","decryptionStatus":"no_key"}', 'enc_A1B2')`, recent) // Observations for both db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES ((SELECT id FROM transmissions WHERE hash='enc_hash_001'), 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES ((SELECT id FROM transmissions WHERE hash='enc_hash_001'), 1, 10.0, -90, '[]', $1)`, recentEpoch) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES ((SELECT id FROM transmissions WHERE hash='enc_hash_002'), 1, 10.0, -90, '[]', ?)`, recentEpoch) + VALUES ((SELECT id FROM transmissions WHERE hash='enc_hash_002'), 1, 10.0, -90, '[]', $1)`, recentEpoch) } func TestGetEncryptedChannels(t *testing.T) { diff --git a/cmd/server/eviction_resolved_pubkeys_test.go b/cmd/server/eviction_resolved_pubkeys_test.go index 078ca351f..6be9f1319 100644 --- a/cmd/server/eviction_resolved_pubkeys_test.go +++ b/cmd/server/eviction_resolved_pubkeys_test.go @@ -13,12 +13,12 @@ func evictionResolvedStore(t testing.TB, n int) (*PacketStore, []int) { pk := func(i int) string { return fmt.Sprintf("%064x", i) } ids := make([]int, 0, n) for i := 1; i <= n; i++ { - if _, err := db.conn.Exec(`INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (?, '00', ?, '2026-01-01T00:00:00Z')`, i, fmt.Sprintf("h%015d", i)); err != nil { + if _, err := db.conn.Exec(`INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES ($1, '00', $2, '2026-01-01T00:00:00Z')`, i, fmt.Sprintf("h%015d", i)); err != nil { t.Fatal(err) } for o := 0; o < 3; o++ { rp := `["` + pk(i%9) + `",null,"` + pk(o+20) + `","` + pk(i%9) + `"]` - if _, err := db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp, resolved_path) VALUES (?, 1, '["aa","bb","cc","dd"]', 0, ?)`, i, rp); err != nil { + if _, err := db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp, resolved_path) VALUES ($1, 1, '["aa","bb","cc","dd"]', 0, $2)`, i, rp); err != nil { t.Fatal(err) } } diff --git a/cmd/server/eviction_test.go b/cmd/server/eviction_test.go index 761fc02f4..1d6fd5aae 100644 --- a/cmd/server/eviction_test.go +++ b/cmd/server/eviction_test.go @@ -290,9 +290,9 @@ func TestEvictStale_CleansResolvedPathNodeIndexes(t *testing.T) { tx.Observations = append(tx.Observations, obs) // Insert into DB so on-demand SQL fetch works during eviction - db.conn.Exec("INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (?, '', ?, ?)", + db.conn.Exec("INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES ($1, '', $2, $3)", txID, tx.Hash, tx.FirstSeen) - db.conn.Exec("INSERT INTO observations (id, transmission_id, observer_idx, path_json, timestamp, resolved_path) VALUES (?, ?, 1, ?, ?, ?)", + db.conn.Exec("INSERT INTO observations (id, transmission_id, observer_idx, path_json, timestamp, resolved_path) VALUES ($1, $2, 1, $3, $4, $5)", obsID, txID, `["aa"]`, now.Add(-48*time.Hour).Unix(), `["`+relayPK+`"]`) store.packets = append(store.packets, tx) @@ -560,10 +560,11 @@ func TestEstimateStoreTxBytes(t *testing.T) { PathJSON: `["aa","bb"]`, } est := estimateStoreTxBytes(tx) - // Manual calculation: base + string lengths + index entries + perTxMaps + path hops + // Manual calculation: base + string lengths + index entries + decoded generation + path hops // (subpaths are counted per request, so they cost the store nothing) hops := int64(len(txGetParsedPath(tx))) manualCalc := int64(storeTxBaseBytes) + int64(len(tx.RawHex)+len(tx.Hash)+len(tx.DecodedJSON)+len(tx.PathJSON)) + int64(numIndexesPerTx*indexEntryBytes) + manualCalc += decodedCacheGenerationBytes manualCalc += int64(decodedCacheFactor * len(tx.DecodedJSON)) manualCalc += hops * perPathHopBytes if est != manualCalc { diff --git a/cmd/server/first_seen_1166_test.go b/cmd/server/first_seen_1166_test.go index f056c1274..1c7f386ba 100644 --- a/cmd/server/first_seen_1166_test.go +++ b/cmd/server/first_seen_1166_test.go @@ -24,7 +24,7 @@ func TestFirstSeen_1166_HandleNodesSurface(t *testing.T) { last := time.Now().UTC().Format("2006-01-02T15:04:05.000Z") if _, err := db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, 'rpt', 'repeater', 37.5, -122.0, ?, ?, 5)`, + VALUES ($1, 'rpt', 'repeater', 37.5, -122.0, $2, $3, 5)`, pk, last, first); err != nil { t.Fatal(err) } diff --git a/cmd/server/go.mod b/cmd/server/go.mod index badbe96db..8cec7a049 100644 --- a/cmd/server/go.mod +++ b/cmd/server/go.mod @@ -1,6 +1,6 @@ module github.com/corescope/server -go 1.22 +go 1.25.0 require ( github.com/gorilla/mux v1.8.1 @@ -38,9 +38,8 @@ require github.com/meshcore-analyzer/prunequeue v0.0.0 replace github.com/meshcore-analyzer/prunequeue => ../../internal/prunequeue require ( - github.com/mattn/go-sqlite3 v1.14.52 github.com/meshcore-analyzer/mbcapqueue v0.0.0 - golang.org/x/sync v0.10.0 + golang.org/x/sync v0.13.0 ) replace github.com/meshcore-analyzer/mbcapqueue => ../../internal/mbcapqueue @@ -52,22 +51,39 @@ require ( replace github.com/meshcore-analyzer/users => ../../internal/users -require github.com/meshcore-analyzer/channel v0.0.0 - -replace github.com/meshcore-analyzer/channel => ../../internal/channel +require ( + github.com/jackc/pgx/v5 v5.7.6 + github.com/meshcore-analyzer/channel v0.0.0 + modernc.org/sqlite v1.34.5 +) require ( github.com/dustin/go-humanize v1.0.1 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/ncruces/go-strftime v0.1.9 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect - golang.org/x/crypto v0.31.0 // indirect - golang.org/x/sys v0.28.0 // indirect + golang.org/x/text v0.24.0 // indirect modernc.org/libc v1.55.3 // indirect modernc.org/mathutil v1.6.0 // indirect modernc.org/memory v1.8.0 // indirect - modernc.org/sqlite v1.34.5 // indirect +) + +replace github.com/meshcore-analyzer/channel => ../../internal/channel + +require ( + github.com/mattn/go-sqlite3 v1.14.52 + github.com/meshcore-analyzer/pgutil v0.0.0 + github.com/meshcore-analyzer/sqliteutil v0.0.0 + golang.org/x/crypto v0.37.0 // indirect + golang.org/x/sys v0.32.0 // indirect ) replace github.com/meshcore-analyzer/mailer => ../../internal/mailer + +replace github.com/meshcore-analyzer/pgutil => ../../internal/pgutil + +replace github.com/meshcore-analyzer/sqliteutil => ../../internal/sqliteutil diff --git a/cmd/server/go.sum b/cmd/server/go.sum index 74e587a22..c370cf1a4 100644 --- a/cmd/server/go.sum +++ b/cmd/server/go.sum @@ -1,3 +1,6 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo= @@ -8,25 +11,46 @@ github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.7.6 h1:rWQc5FwZSPX58r1OQmkuaNicxdmExaEz5A2DO2hUuTk= +github.com/jackc/pgx/v5 v5.7.6/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-sqlite3 v1.14.52 h1:wVbm2Qnf4OXkqhBTSPuCRZDRnxfbVrrmiCEroVdog8U= github.com/mattn/go-sqlite3 v1.14.52/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w= github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= -golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= -golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= -golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= -golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= +golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= +golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= -golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw= -golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc= +golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= +golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= +golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= modernc.org/cc/v4 v4.21.4 h1:3Be/Rdo1fpr8GrQ7IVw9OHtplU4gWbb+wNgeoBMmGLQ= modernc.org/cc/v4 v4.21.4/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ= modernc.org/ccgo/v4 v4.19.2 h1:lwQZgvboKD0jBwdaeVCTouxhxAyN6iawF3STraAal8Y= diff --git a/cmd/server/grouped_history_page_test.go b/cmd/server/grouped_history_page_test.go new file mode 100644 index 000000000..374a5fb7e --- /dev/null +++ b/cmd/server/grouped_history_page_test.go @@ -0,0 +1,157 @@ +package main + +import ( + "encoding/json" + "fmt" + "strings" + "testing" + "time" +) + +func TestGroupedHistoryPagePreservesFiltersAndDetails(t *testing.T) { + db, writer := openPostgresValueFixture(t) + now := time.Date(2026, 10, 12, 12, 0, 0, 0, time.UTC) + exec := func(query string, args ...any) { + t.Helper() + if _, err := writer.Exec(query, args...); err != nil { + t.Fatal(err) + } + } + exec(`INSERT INTO observers(rowid,id,name,iata) VALUES(1,'history-a','Observer A','AAA'),(2,'history-b','Observer B','BBB')`) + for id, age := range []time.Duration{48 * time.Hour, time.Minute, 2 * time.Minute, 12 * time.Hour} { + exec(`INSERT INTO transmissions(id,raw_hex,hash,first_seen,last_seen,payload_type,scope_name) VALUES($1,'AA',$2,$3,0,4,'history-scope')`, id+1, fmt.Sprintf("history-%d", id+1), now.Add(-age).Format(time.RFC3339)) + } + exec(`INSERT INTO observations(id,transmission_id,observer_idx,path_json,timestamp,snr) VALUES + (1,1,1,'["aa","bb","cc"]',$1,3), + (2,1,2,'["dd"]',$2,2), + (3,1,1,'["aa"]',$3,1), + (4,2,1,'[]',$4,4), + (5,4,1,'[]',$2,5)`, now.Add(-25*time.Hour).Unix(), now.Add(-5*time.Minute).Unix(), now.Add(-26*time.Hour).Unix(), now.Add(-72*time.Hour).Unix()) + q := PacketQuery{Since: now.Add(-24 * time.Hour).Format(time.RFC3339), Observer: "history-a", Limit: 1, Offset: 1} + result, err := db.QueryGroupedPackets(q) + if err != nil { + t.Fatal(err) + } + if result.Total != 2 || len(result.Packets) != 1 { + t.Fatalf("filtered page/total = %#v", result) + } + row := result.Packets[0] + // Equal latest timestamps are paged deterministically by descending ID. + // The since/observer predicates match independently, as before; details + // still include every observation, including ones outside that window. + if row["hash"] != "history-1" || row["count"] != 3 || row["observer_count"] != 2 || row["observer_id"] != "history-a" || row["path_json"] != `["aa","bb","cc"]` || row["scope_name"] != "history-scope" { + t.Fatalf("history details changed or tie page unstable: %#v", row) + } + if got := strings.Join(row["distinct_iatas"].([]string), ","); got != "AAA,BBB" { + t.Fatalf("IATAs restricted to filter window: %q", got) + } + if row["latest"] != now.Add(-5*time.Minute).Format("2006-01-02T15:04:05.000Z") { + t.Fatalf("latest changed: %#v", row["latest"]) + } + all, err := db.QueryGroupedPackets(PacketQuery{Limit: 10}) + if err != nil || all.Total != 4 || len(all.Packets) != 4 { + t.Fatalf("all history: %#v / %v", all, err) + } + if all.Packets[0]["hash"] != "history-3" || all.Packets[0]["latest"] != now.Add(-2*time.Minute).Format(time.RFC3339) { + t.Fatalf("no-observation first_seen fallback changed: %#v", all.Packets[0]) + } + if all.Packets[3]["hash"] != "history-2" { + t.Fatalf("ordered by first_seen/denormalized last_seen instead of observations: %#v", all.Packets) + } + end, err := db.QueryGroupedPackets(PacketQuery{Limit: 1, Offset: 10}) + if err != nil || end.Total != 4 || len(end.Packets) != 0 { + t.Fatalf("empty page lost total: %#v / %v", end, err) + } +} + +func TestPostgresGroupedHistoryHydratesOnlyPage(t *testing.T) { + postgresOnly(t) + db := startupMemoryFixture(t, 1000, 65, false) + sql := db.groupedPacketSelectSQL("", 0) + var raw []byte + if err := db.conn.QueryRow("EXPLAIN (ANALYZE, VERBOSE, FORMAT JSON) "+sql, 2, 10).Scan(&raw); err != nil { + t.Fatal(err) + } + type planNode struct { + Type string `json:"Node Type"` + Rows float64 `json:"Actual Rows"` + Loops float64 `json:"Actual Loops"` + Output []string `json:"Output"` + SortKey []string `json:"Sort Key"` + Relation string `json:"Relation Name"` + Direction string `json:"Scan Direction"` + Condition string `json:"Index Cond"` + Plans []planNode `json:"Plans"` + } + var plans []struct{ Plan planNode } + if err := json.Unmarshal(raw, &plans); err != nil || len(plans) != 1 { + t.Fatalf("invalid plan: %v", err) + } + pending := []planNode{plans[0].Plan} + foundBestPath := false + foundLatestSeek := false + for len(pending) > 0 { + n := pending[0] + pending = append(pending[1:], n.Plans...) + if strings.Contains(strings.Join(n.SortKey, " "), "length(") { + foundBestPath = true + if n.Loops > 2 { + t.Fatalf("two-row page hydrated %g best-path groups", n.Loops) + } + } + if n.Type == "Aggregate" && strings.Contains(strings.Join(n.Output, " "), "max(to_char(") { + t.Fatal("latest still formats every observation before aggregating") + } + if n.Relation == "observations" { + t.Logf("observation scan: %s direction=%s rows=%g loops=%g condition=%s", n.Type, n.Direction, n.Rows, n.Loops, n.Condition) + } + condition := strings.ReplaceAll(n.Condition, `"`, "") + if n.Relation == "observations" && strings.Contains(n.Type, "Index") && n.Direction == "Backward" && strings.Contains(condition, "timestamp IS NOT NULL") { + foundLatestSeek = true + if n.Rows != 1 || !strings.Contains(condition, "transmission_id") { + t.Fatalf("latest lookup scanned observations instead of seeking one: rows=%g, condition=%s", n.Rows, n.Condition) + } + } + } + if !foundBestPath { + t.Fatal("plan did not exercise best-path selection") + } + if !foundLatestSeek { + t.Fatal("latest lookup did not use the existing timestamp index") + } +} + +func TestGroupedHistoryLegacyV2(t *testing.T) { + dsn := postgresTestDSN(t) + createTestDBAt(t, dsn, 2) + db, err := openFixtureReader(t, dsn) + if err != nil { + t.Fatal(err) + } + defer db.Close() + result, err := db.QueryGroupedPackets(PacketQuery{Limit: 1}) + if err != nil || result.Total != 2 || len(result.Packets) != 1 { + t.Fatalf("legacy page: %#v / %v", result, err) + } + row := result.Packets[0] + if row["hash"] != "h0002" || row["latest"] != "2026-01-01T00:02:00Z" || row["count"] != 1 || row["observer_id"] != "obs1" { + t.Fatalf("legacy timestamp or observer changed: %#v", row) + } +} + +func BenchmarkGroupedPacketHistory(b *testing.B) { + db := startupMemoryFixture(b, 6000, 65, false) + for _, limit := range []int{1, 5000} { + b.Run(fmt.Sprintf("page-%d", limit), func(b *testing.B) { + q := PacketQuery{Since: time.Now().UTC().Add(-24 * time.Hour).Format(time.RFC3339), Limit: limit} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + result, err := db.QueryGroupedPackets(q) + if err != nil || result.Total != 6000 || len(result.Packets) != limit { + b.Fatalf("history query: %v / %v", result, err) + } + } + }) + } +} diff --git a/cmd/server/hash_migrate.go b/cmd/server/hash_migrate.go index a85d58bc1..1dd52090c 100644 --- a/cmd/server/hash_migrate.go +++ b/cmd/server/hash_migrate.go @@ -5,132 +5,34 @@ import ( "time" ) -// migrateContentHashesAsync recomputes content hashes in batches after the -// server is already serving HTTP. Packets whose hash changes are updated in -// both the DB and the in-memory byHash index. The migration is idempotent: -// once all hashes match the current formula it completes instantly. -func migrateContentHashesAsync(store *PacketStore, batchSize int, yieldDuration time.Duration) { - // #1856: every DB failure below continues to the next batch, so the loop - // always reaches the deferred completion. Setting the flag unconditionally - // therefore reported success after migrating nothing, which is exactly what - // happens on the read-only handle the server holds since #1283: begin, - // prepare and commit all fail, every batch is skipped, and /api/stats then - // answers hashMigrationComplete: true. - failedBatches := 0 - defer func() { - if r := recover(); r != nil { - log.Printf("[hash-migrate] panic recovered: %v", r) - failedBatches++ - } - if failedBatches > 0 { - log.Printf("[hash-migrate] INCOMPLETE: %d batch(es) could not be written; "+ - "hashMigrationComplete stays false. On a read-only DB handle this is expected "+ - "and the migration belongs in the ingestor (#1856).", failedBatches) - return - } - store.hashMigrationComplete.Store(true) - }() - - // Snapshot the packet slice length under lock (packets only grow). +// verifyContentHashesAsync reports whether the loaded hashes are current. The +// server never rewrites telemetry or its hash index independently of storage. +// Legacy hash repair is a separate offline operation; import preserves hashes. +func verifyContentHashesAsync(store *PacketStore, batchSize int, yieldDuration time.Duration) { + if batchSize <= 0 { + batchSize = 5000 + } store.mu.RLock() total := len(store.packets) store.mu.RUnlock() - - migrated := 0 for offset := 0; offset < total; offset += batchSize { - end := offset + batchSize - if end > total { - end = total - } - - // Collect stale hashes in this batch under RLock. - type hashUpdate struct { - tx *StoreTx - oldHash string - newHash string - } - var updates []hashUpdate - store.mu.RLock() - for _, tx := range store.packets[offset:end] { - if tx.RawHex == "" { - continue - } - newHash := ComputeContentHash(tx.RawHex) - if newHash != tx.Hash { - updates = append(updates, hashUpdate{tx: tx, oldHash: tx.Hash, newHash: newHash}) + end := min(offset+batchSize, min(total, len(store.packets))) + stale := false + for _, tx := range store.packets[min(offset, end):end] { + if tx.RawHex != "" && ComputeContentHash(tx.RawHex) != tx.Hash { + stale = true + break } } store.mu.RUnlock() - - if len(updates) == 0 { - continue - } - - // Write batch to DB in a single transaction. - dbTx, err := store.db.conn.Begin() - if err != nil { - log.Printf("[hash-migrate] begin tx: %v", err) - failedBatches++ - continue - } - stmt, err := dbTx.Prepare("UPDATE transmissions SET hash = ? WHERE id = ?") - if err != nil { - log.Printf("[hash-migrate] prepare: %v", err) - dbTx.Rollback() - failedBatches++ - continue - } - - for _, u := range updates { - if _, err := stmt.Exec(u.newHash, u.tx.ID); err != nil { - // UNIQUE constraint = two old hashes map to the same new hash (duplicate). - // Merge observations to the surviving tx, delete the duplicate. - log.Printf("[hash-migrate] tx %d collides — merging duplicate", u.tx.ID) - var survID int - if err2 := dbTx.QueryRow("SELECT id FROM transmissions WHERE hash = ?", u.newHash).Scan(&survID); err2 == nil { - dbTx.Exec("UPDATE observations SET transmission_id = ? WHERE transmission_id = ?", survID, u.tx.ID) - dbTx.Exec("DELETE FROM transmissions WHERE id = ?", u.tx.ID) - u.newHash = "" // mark for in-memory removal only - } - } + if stale { + log.Print("[hash-verify] stale content hashes found; perform explicit offline hash repair on a working copy of the legacy snapshot before import. The importer preserves existing hashes") + return } - stmt.Close() - - if err := dbTx.Commit(); err != nil { - log.Printf("[hash-migrate] commit: %v", err) - failedBatches++ - continue + if yieldDuration > 0 { + time.Sleep(yieldDuration) } - - // Update in-memory index under write lock. - store.mu.Lock() - for _, u := range updates { - delete(store.byHash, u.oldHash) - if u.newHash == "" { - // Merged duplicate — remove from packets slice and indexes. - delete(store.byTxID, u.tx.ID) - // Move observations to survivor if present. - if surv := store.byHash[ComputeContentHash(u.tx.RawHex)]; surv != nil { - for _, obs := range u.tx.Observations { - surv.Observations = append(surv.Observations, obs) - surv.ObservationCount++ - } - } - } else { - u.tx.Hash = u.newHash - store.byHash[u.newHash] = u.tx - } - } - store.mu.Unlock() - - migrated += len(updates) - - // Yield to let HTTP handlers run. - time.Sleep(yieldDuration) - } - - if migrated > 0 { - log.Printf("[hash-migrate] Migrated %d content hashes to new formula", migrated) } + store.hashMigrationComplete.Store(true) } diff --git a/cmd/server/hash_migrate_test.go b/cmd/server/hash_migrate_test.go index 0cc7f21e6..3df5c46f1 100644 --- a/cmd/server/hash_migrate_test.go +++ b/cmd/server/hash_migrate_test.go @@ -5,7 +5,7 @@ import ( "time" ) -func TestMigrateContentHashesAsync(t *testing.T) { +func TestVerifyContentHashesDoesNotWrite(t *testing.T) { db := setupTestDBv2(t) store := NewPacketStore(db, nil) @@ -14,8 +14,9 @@ func TestMigrateContentHashesAsync(t *testing.T) { correctHash := ComputeContentHash(rawHex) wrongHash := "deadbeef12345678" - _, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type) - VALUES (?, ?, datetime('now'), 0, 2)`, rawHex, wrongHash) + _, err := db.conn.Exec(testNativeSQL(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type) + VALUES ($1, $2, strftime('%Y-%m-%dT%H:%M:%SZ','now'), 0, 2)`, `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type) + VALUES ($1, $2, to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"'), 0, 2)`), rawHex, wrongHash) if err != nil { t.Fatal(err) } @@ -28,25 +29,25 @@ func TestMigrateContentHashesAsync(t *testing.T) { t.Fatal("expected packet under wrong hash before migration") } - migrateContentHashesAsync(store, 100, time.Millisecond) + verifyContentHashesAsync(store, 100, time.Millisecond) - if !store.hashMigrationComplete.Load() { - t.Error("expected hashMigrationComplete to be true") + if store.hashMigrationComplete.Load() { + t.Error("stale hashes must not report migration complete") } - if store.byHash[wrongHash] != nil { - t.Error("old hash should be removed from index") + if store.byHash[wrongHash] == nil { + t.Error("read-only verification must preserve the old hash index") } - if store.byHash[correctHash] == nil { - t.Error("new hash should be in index") + if store.byHash[correctHash] != nil { + t.Error("verification must not invent a new hash") } var dbHash string - err = db.conn.QueryRow("SELECT hash FROM transmissions WHERE raw_hex = ?", rawHex).Scan(&dbHash) + err = db.conn.QueryRow("SELECT hash FROM transmissions WHERE raw_hex = $1", rawHex).Scan(&dbHash) if err != nil { t.Fatal(err) } - if dbHash != correctHash { - t.Errorf("DB hash = %s, want %s", dbHash, correctHash) + if dbHash != wrongHash { + t.Errorf("DB hash = %s, want unchanged %s", dbHash, wrongHash) } } @@ -57,8 +58,9 @@ func TestMigrateContentHashesAsync_NoOp(t *testing.T) { rawHex := "0A00D69FD7A5A7475DB07337749AE61FA53A4788E976" correctHash := ComputeContentHash(rawHex) - _, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type) - VALUES (?, ?, datetime('now'), 0, 2)`, rawHex, correctHash) + _, err := db.conn.Exec(testNativeSQL(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type) + VALUES ($1, $2, strftime('%Y-%m-%dT%H:%M:%SZ','now'), 0, 2)`, `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type) + VALUES ($1, $2, to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"'), 0, 2)`), rawHex, correctHash) if err != nil { t.Fatal(err) } @@ -67,7 +69,7 @@ func TestMigrateContentHashesAsync_NoOp(t *testing.T) { t.Fatal(err) } - migrateContentHashesAsync(store, 100, time.Millisecond) + verifyContentHashesAsync(store, 100, time.Millisecond) if !store.hashMigrationComplete.Load() { t.Error("expected hashMigrationComplete to be true") @@ -93,8 +95,9 @@ func TestMigrateContentHashesAsyncDoesNotClaimCompletionWhenWritesFail(t *testin rawHex := "0A00D69FD7A5A7475DB07337749AE61FA53A4788E976" wrongHash := "deadbeef12345678" - if _, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type) - VALUES (?, ?, datetime('now'), 0, 2)`, rawHex, wrongHash); err != nil { + if _, err := db.conn.Exec(testNativeSQL(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type) + VALUES ($1, $2, strftime('%Y-%m-%dT%H:%M:%SZ','now'), 0, 2)`, `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type) + VALUES ($1, $2, to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"'), 0, 2)`), rawHex, wrongHash); err != nil { t.Fatal(err) } if err := store.Load(); err != nil { @@ -109,7 +112,7 @@ func TestMigrateContentHashesAsyncDoesNotClaimCompletionWhenWritesFail(t *testin t.Fatal(err) } - migrateContentHashesAsync(store, 100, time.Millisecond) + verifyContentHashesAsync(store, 100, time.Millisecond) if store.hashMigrationComplete.Load() { t.Error("hashMigrationComplete must stay false when no batch could be written; " + diff --git a/cmd/server/healthz.go b/cmd/server/healthz.go index 3f554952f..3039a5fa0 100644 --- a/cmd/server/healthz.go +++ b/cmd/server/healthz.go @@ -7,15 +7,24 @@ import ( ) // readiness tracks whether background init goroutines have completed. -// Set to 1 once store.Load, pickBestObservation, and neighbor graph build are done. +// Startup loading is gated separately by StartupLoadDone in handleHealthz. var readiness atomic.Int32 // handleHealthz returns 200 when the server is ready to serve queries, -// or 503 while background initialization is still running. +// or 503 while startup is incomplete or the startup load has failed. func (s *Server) handleHealthz(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") - if readiness.Load() == 0 { + ready := readiness.Load() != 0 + if ready && s.store != nil { + select { + case <-s.store.StartupLoadDone(): + ready = !s.store.backgroundLoadFailed.Load() + default: + ready = false + } + } + if !ready { w.WriteHeader(http.StatusServiceUnavailable) json.NewEncoder(w).Encode(map[string]interface{}{ "ready": false, diff --git a/cmd/server/healthz_test.go b/cmd/server/healthz_test.go index f343b0c7f..3bbae1494 100644 --- a/cmd/server/healthz_test.go +++ b/cmd/server/healthz_test.go @@ -12,7 +12,9 @@ func TestHealthzNotReady(t *testing.T) { readiness.Store(0) defer readiness.Store(0) - srv := &Server{store: &PacketStore{}} + store := &PacketStore{clockSkew: &ClockSkewEngine{}} + store.signalStartupLoadDone() + srv := &Server{store: store} req := httptest.NewRequest("GET", "/api/healthz", nil) w := httptest.NewRecorder() @@ -38,7 +40,9 @@ func TestHealthzReady(t *testing.T) { readiness.Store(1) defer readiness.Store(0) - srv := &Server{store: &PacketStore{}} + store := &PacketStore{clockSkew: &ClockSkewEngine{}} + store.signalStartupLoadDone() + srv := &Server{store: store} req := httptest.NewRequest("GET", "/api/healthz", nil) w := httptest.NewRecorder() @@ -78,3 +82,91 @@ func TestHealthzAntiTautology(t *testing.T) { t.Fatal("anti-tautology: handler returned 200 when readiness=0; gating is broken") } } + +// FirstChunkReady permits HTTP serving while either startup phase is still +// loading. Deployment readiness must wait for both phases and reject failures. +func TestHealthzWaitsForSuccessfulStartupLoad(t *testing.T) { + oldReadiness := readiness.Swap(1) + t.Cleanup(func() { readiness.Store(oldReadiness) }) + for _, phase := range []string{"hot_chunks", "background_fill"} { + for _, fail := range []bool{false, true} { + name := phase + "/success" + if fail { + name = phase + "/failure" + } + t.Run(name, func(t *testing.T) { + dbPath := createTestDBWithAgedPackets(t, 6, 6) + db, err := openFixtureReader(t, dbPath) + if err != nil { + t.Fatal(err) + } + defer db.conn.Close() + store := NewPacketStore(db, &PacketStoreConfig{RetentionHours: 72, HotStartupHours: 1}) + srv := &Server{store: store} + checkHealth := func(want int) { + t.Helper() + w := httptest.NewRecorder() + srv.handleHealthz(w, httptest.NewRequest(http.MethodGet, "/api/healthz", nil)) + if w.Code != want { + t.Errorf("health status = %d, want %d; body=%s", w.Code, want, w.Body.String()) + } + var response struct { + Ready bool `json:"ready"` + } + if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil { + t.Fatal(err) + } + if response.Ready != (want == http.StatusOK) { + t.Errorf("ready=%t, want %t", response.Ready, want == http.StatusOK) + } + } + hookCalled := false + checkLoading := func() { + if hookCalled { + return + } + hookCalled = true + if !isClosedForTest(store.FirstChunkReady()) { + t.Error("HTTP startup must be unblocked before all chunks finish") + } + if isClosedForTest(store.StartupLoadDone()) { + t.Error("startup unexpectedly complete inside loader") + } + if got, want := store.LoadComplete(), phase == "background_fill"; got != want { + t.Errorf("LoadComplete=%t, want %t during %s", got, want, phase) + } + checkHealth(http.StatusServiceUnavailable) + if fail { + if err := db.conn.Close(); err != nil { + t.Fatal(err) + } + } + } + if phase == "hot_chunks" { + store.OnChunkLoaded(func(_, _ int) { checkLoading() }) + } else { + store.bgLoaderEntryHook = checkLoading + } + err = store.RunStartupLoad(2) + if fail && phase == "hot_chunks" { + if err == nil { + t.Error("expected a late chunk read error") + } + } else if err != nil { + t.Fatal(err) + } + if !hookCalled || !isClosedForTest(store.StartupLoadDone()) { + t.Fatal("startup did not exercise the hook and reach a terminal state") + } + if store.backgroundLoadFailed.Load() != fail { + t.Fatalf("backgroundLoadFailed=%t, want %t", store.backgroundLoadFailed.Load(), fail) + } + want := http.StatusOK + if fail { + want = http.StatusServiceUnavailable + } + checkHealth(want) + }) + } + } +} diff --git a/cmd/server/hidden_name_prefix_1181_endpoints_test.go b/cmd/server/hidden_name_prefix_1181_endpoints_test.go index 8b30bb020..dab2224b6 100644 --- a/cmd/server/hidden_name_prefix_1181_endpoints_test.go +++ b/cmd/server/hidden_name_prefix_1181_endpoints_test.go @@ -23,7 +23,7 @@ func TestHiddenNamePrefix_1181_NodeHealth(t *testing.T) { pk := "deadbeef00001184" if _, err := srv.db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, ?, ?, 0, 0, '2026-06-01T00:00:00Z', '2026-06-01T00:00:00Z', 1)`, + VALUES ($1, $2, $3, 0, 0, '2026-06-01T00:00:00Z', '2026-06-01T00:00:00Z', 1)`, pk, "🚫 health me", "companion"); err != nil { t.Fatalf("insert: %v", err) } @@ -57,7 +57,7 @@ func TestHiddenNamePrefix_1181_BulkHealth(t *testing.T) { pk := "deadbeef00001185" if _, err := srv.db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, ?, ?, 0, 0, '2026-06-01T00:00:00Z', '2026-06-01T00:00:00Z', 1)`, + VALUES ($1, $2, $3, 0, 0, '2026-06-01T00:00:00Z', '2026-06-01T00:00:00Z', 1)`, pk, "🚫 bulk me", "companion"); err != nil { t.Fatalf("insert: %v", err) } @@ -91,7 +91,7 @@ func TestHiddenNamePrefix_1181_Paths(t *testing.T) { pk := "deadbeef00001186" if _, err := srv.db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, ?, ?, 0, 0, '2026-06-01T00:00:00Z', '2026-06-01T00:00:00Z', 1)`, + VALUES ($1, $2, $3, 0, 0, '2026-06-01T00:00:00Z', '2026-06-01T00:00:00Z', 1)`, pk, "🚫 paths me", "companion"); err != nil { t.Fatalf("insert: %v", err) } @@ -113,7 +113,7 @@ func TestHiddenNamePrefix_1181_Analytics(t *testing.T) { pk := "deadbeef00001187" if _, err := srv.db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, ?, ?, 0, 0, '2026-06-01T00:00:00Z', '2026-06-01T00:00:00Z', 1)`, + VALUES ($1, $2, $3, 0, 0, '2026-06-01T00:00:00Z', '2026-06-01T00:00:00Z', 1)`, pk, "🚫 analytics me", "companion"); err != nil { t.Fatalf("insert: %v", err) } diff --git a/cmd/server/hidden_name_prefix_1181_test.go b/cmd/server/hidden_name_prefix_1181_test.go index d449fcae6..9dee2da4a 100644 --- a/cmd/server/hidden_name_prefix_1181_test.go +++ b/cmd/server/hidden_name_prefix_1181_test.go @@ -19,7 +19,7 @@ func TestHiddenNamePrefix_1181_NodesList(t *testing.T) { // Insert a node whose name starts with the configured 🚫 prefix. _, err := srv.db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, ?, ?, 0, 0, '2026-06-01T00:00:00Z', '2026-06-01T00:00:00Z', 1)`, + VALUES ($1, $2, $3, 0, 0, '2026-06-01T00:00:00Z', '2026-06-01T00:00:00Z', 1)`, "deadbeef00001181", "🚫 ban me", "companion") if err != nil { t.Fatalf("insert hidden node: %v", err) @@ -70,7 +70,7 @@ func TestHiddenNamePrefix_1181_Search(t *testing.T) { if _, err := srv.db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, ?, ?, 0, 0, '2026-06-01T00:00:00Z', '2026-06-01T00:00:00Z', 1)`, + VALUES ($1, $2, $3, 0, 0, '2026-06-01T00:00:00Z', '2026-06-01T00:00:00Z', 1)`, "deadbeef00001182", "🚫 search me", "companion"); err != nil { t.Fatalf("insert: %v", err) } @@ -105,7 +105,7 @@ func TestHiddenNamePrefix_1181_Detail(t *testing.T) { pk := "deadbeef00001183" if _, err := srv.db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, ?, ?, 0, 0, '2026-06-01T00:00:00Z', '2026-06-01T00:00:00Z', 1)`, + VALUES ($1, $2, $3, 0, 0, '2026-06-01T00:00:00Z', '2026-06-01T00:00:00Z', 1)`, pk, "🚫 detail me", "companion"); err != nil { t.Fatalf("insert: %v", err) } diff --git a/cmd/server/hop_disambig_e2e_test.go b/cmd/server/hop_disambig_e2e_test.go index 41ad1f127..4e993f9c3 100644 --- a/cmd/server/hop_disambig_e2e_test.go +++ b/cmd/server/hop_disambig_e2e_test.go @@ -76,7 +76,7 @@ func t1201InsertNode(t *testing.T, db *DB, n t1201Node) { // "Berlin would win tier-3" premise of this fixture weakens silently — // update both this insert and the candidate scoring assertions. _, err := db.conn.Exec( - `INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) VALUES (?, ?, 'repeater', ?, ?, ?, '2026-01-01T00:00:00Z', ?)`, + `INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) VALUES ($1, $2, 'repeater', $3, $4, $5, '2026-01-01T00:00:00Z', $6)`, n.pk, "node-"+n.pk[:4], n.lat, n.lon, "2026-05-01T00:00:00Z", n.obsCount, ) if err != nil { @@ -109,7 +109,7 @@ func TestTopHopsRespectsContextAcrossAllCallSites(t *testing.T) { // Insert observer row (referenced by observations via observer_idx). if _, err := db.conn.Exec( - `INSERT INTO observers (id, name, last_seen, first_seen, packet_count) VALUES (?, ?, ?, '2026-01-01T00:00:00Z', 100)`, + `INSERT INTO observers (id, name, last_seen, first_seen, packet_count) VALUES ($1, $2, $3, '2026-01-01T00:00:00Z', 100)`, t1201Observer, "obs-ca", "2026-05-01T00:00:00Z", ); err != nil { t.Fatal(err) @@ -128,17 +128,18 @@ func TestTopHopsRespectsContextAcrossAllCallSites(t *testing.T) { for i := 0; i < 50; i++ { ts := baseTime.Add(time.Duration(i) * time.Minute).Format(time.RFC3339) hash := fmt.Sprintf("hash1201_%03d", i) - res, err := tx.Exec( - `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) VALUES (?, ?, ?, 1, 1, ?)`, + var txID int64 + err = tx.QueryRow( + `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) VALUES ($1, $2, $3, 1, 1, $4) RETURNING id`, "AA", hash, ts, string(decoded), - ) + ).Scan(&txID) if err != nil { _ = tx.Rollback() t.Fatal(err) } - txID, _ := res.LastInsertId() + if _, err := tx.Exec( - `INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) VALUES (?, 1, 12.0, -90, ?, ?)`, + `INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) VALUES ($1, 1, 12.0, -90, $2, $3)`, txID, pathJSON, baseTime.Add(time.Duration(i)*time.Minute).Unix(), ); err != nil { _ = tx.Rollback() diff --git a/cmd/server/hot_startup_consistency_test.go b/cmd/server/hot_startup_consistency_test.go index 709333993..50929d6c5 100644 --- a/cmd/server/hot_startup_consistency_test.go +++ b/cmd/server/hot_startup_consistency_test.go @@ -26,7 +26,7 @@ func TestHotStartup_loadChunk_IndexSliceConsistency(t *testing.T) { // spans 3 batches, widening the inconsistency window for the reader. dbPath := createTestDBWithAgedPackets(t, 10, 1200) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } diff --git a/cmd/server/hot_startup_test.go b/cmd/server/hot_startup_test.go index 238369c3c..362344835 100644 --- a/cmd/server/hot_startup_test.go +++ b/cmd/server/hot_startup_test.go @@ -1,19 +1,16 @@ package main import ( - "database/sql" "encoding/json" "fmt" "net/http" "net/http/httptest" - "path/filepath" "sync" "sync/atomic" "testing" "time" "github.com/gorilla/mux" - _ "github.com/mattn/go-sqlite3" ) // createTestDBMultiDay creates a test DB with packets spread across numDays days. @@ -21,10 +18,9 @@ import ( // Packets within each day are spaced 1 minute apart. func createTestDBMultiDay(t *testing.T, numDays, txPerDay int) string { t.Helper() - dir := t.TempDir() - dbPath := filepath.Join(dir, "test.db") + dbPath := postgresTestDSN(t) - conn, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL") + conn, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } @@ -35,33 +31,25 @@ func createTestDBMultiDay(t *testing.T, numDays, txPerDay int) string { t.Fatalf("createTestDBMultiDay setup: %v", err) } } - execOrFail(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT)`) - execOrFail(`CREATE TABLE observations (id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, direction TEXT, snr REAL, rssi REAL, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT)`) - execOrFail(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) - execOrFail(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, frequency REAL)`) + execOrFail(testNativeSQL(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY AUTOINCREMENT, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT)`, `CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT)`)) + execOrFail(testNativeSQL(`CREATE TABLE observations (id INTEGER PRIMARY KEY AUTOINCREMENT, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT)`, `CREATE TABLE observations (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT)`)) + execOrFail(testNativeSQL(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY AUTOINCREMENT, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, `CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`)) + execOrFail(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, first_seen TEXT, frequency DOUBLE PRECISION)`) execOrFail(`CREATE TABLE schema_version (version INTEGER)`) execOrFail(`INSERT INTO schema_version (version) VALUES (1)`) execOrFail(`CREATE INDEX idx_tx_first_seen ON transmissions(first_seen)`) - id := 1 now := time.Now().UTC() - for day := numDays; day >= 1; day-- { - // Offset by +30 minutes so day boundaries don't coincide exactly with - // hotStartupHours/retentionHours cutoffs, preventing timing-boundary flakiness. - // E.g. for numDays=3: day3 starts at now-71.5h, day2 at now-47.5h, day1 at now-23.5h. - base := now.Add(-time.Duration(day)*24*time.Hour + 30*time.Minute) - for i := 0; i < txPerDay; i++ { - ts := base.Add(time.Duration(i) * time.Minute).Format(time.RFC3339) - hash := fmt.Sprintf("hash%06d", id) - if _, err := conn.Exec("INSERT INTO transmissions VALUES (?,?,?,?,0,4,1,?)", id, "aa", hash, ts, `{}`); err != nil { - t.Fatalf("createTestDBMultiDay insert tx: %v", err) - } - if _, err := conn.Exec("INSERT INTO observations VALUES (?,?,?,?,?,?,?,?,?,?,?)", id, id, "obs1", "Obs1", "RX", -10.0, -80.0, 5, `[]`, ts, ""); err != nil { - t.Fatalf("createTestDBMultiDay insert obs: %v", err) - } - id++ - } + stamp := func(n int) string { + day, offset := numDays-n/txPerDay, n%txPerDay + return now.Add(-time.Duration(day)*24*time.Hour + 30*time.Minute + time.Duration(offset)*time.Minute).Format(time.RFC3339) } + copyTestRows(t, conn, "transmissions", []string{"id", "raw_hex", "hash", "first_seen", "route_type", "payload_type", "payload_version", "decoded_json"}, numDays*txPerDay, func(n int) []any { + return []any{n + 1, "aa", fmt.Sprintf("hash%06d", n+1), stamp(n), 0, 4, 1, "{}"} + }) + copyTestRows(t, conn, "observations", []string{"id", "transmission_id", "observer_id", "observer_name", "direction", "snr", "rssi", "score", "path_json", "timestamp", "raw_hex"}, numDays*txPerDay, func(n int) []any { + return []any{n + 1, n + 1, "obs1", "Obs1", "RX", -10.0, -80.0, 5, "[]", stamp(n), ""} + }) return dbPath } @@ -81,7 +69,7 @@ func waitForBackgroundLoad(t *testing.T, store *PacketStore, timeout time.Durati func TestHotStartupConfig_Clamp(t *testing.T) { dbPath := createTestDB(t, 10) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } @@ -100,7 +88,7 @@ func TestHotStartupConfig_Clamp(t *testing.T) { func TestHotStartupConfig_ZeroIsDisabled(t *testing.T) { dbPath := createTestDB(t, 10) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } @@ -119,7 +107,7 @@ func TestHotStartup_LoadsOnlyHotWindow(t *testing.T) { // 50 old packets (48h ago), 10 recent (30min ago) dbPath := createTestDBWithAgedPackets(t, 10, 50) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } @@ -156,7 +144,7 @@ func TestHotStartup_DisabledWhenZero(t *testing.T) { // 50 old (48h ago), 10 recent (30min ago) — all within 72h retention dbPath := createTestDBWithAgedPackets(t, 10, 50) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } @@ -180,7 +168,7 @@ func TestHotStartup_loadChunk_AddsOlderData(t *testing.T) { // 50 old packets (48h ago), 10 recent (30min ago) dbPath := createTestDBWithAgedPackets(t, 10, 50) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } @@ -229,7 +217,7 @@ func TestHotStartup_BackgroundFillsToRetention(t *testing.T) { // 3 days × 50 tx/day = 150 total dbPath := createTestDBMultiDay(t, 3, 50) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } @@ -269,7 +257,7 @@ func TestHotStartup_BackgroundFillsToRetention(t *testing.T) { func TestHotStartup_ChunkErrorRecovery(t *testing.T) { dbPath := createTestDBWithAgedPackets(t, 10, 50) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } @@ -314,7 +302,7 @@ func TestHotStartup_SQLFallback_TriggeredForOldDate(t *testing.T) { // 50 old packets (48h ago), 10 recent (30min ago) dbPath := createTestDBWithAgedPackets(t, 10, 50) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } @@ -345,7 +333,7 @@ func TestHotStartup_SQLFallback_TriggeredForOldDate(t *testing.T) { func TestHotStartup_PerfStats(t *testing.T) { dbPath := createTestDBWithAgedPackets(t, 10, 50) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } @@ -376,7 +364,7 @@ func TestHotStartup_SQLFallback_NotTriggeredForRecentDate(t *testing.T) { // 50 old packets (48h ago), 10 recent (30min ago) dbPath := createTestDBWithAgedPackets(t, 10, 50) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } @@ -405,7 +393,7 @@ func TestHotStartup_SQLFallback_Until(t *testing.T) { // 50 old packets (48h ago), 10 recent (30min ago) dbPath := createTestDBWithAgedPackets(t, 10, 50) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } @@ -436,7 +424,7 @@ func TestHotStartup_SQLFallback_Until(t *testing.T) { func TestHotStartup_PerfStoreHTTP(t *testing.T) { dbPath := createTestDBWithAgedPackets(t, 10, 50) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } @@ -485,7 +473,7 @@ func TestHotStartup_ConcurrentQueryDuringBackgroundLoad(t *testing.T) { // large enough to give pollers >=1 query during the background fill. dbPath := createTestDBMultiDay(t, 5, 200) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } @@ -583,7 +571,7 @@ func (a *atomicSamples) get() int64 { func TestHotStartup_BackgroundLoadFailureSurfacesInPerf(t *testing.T) { dbPath := createTestDBMultiDay(t, 3, 50) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } diff --git a/cmd/server/issue1189_distinct_iatas_test.go b/cmd/server/issue1189_distinct_iatas_test.go index 3a096fe95..2bf397336 100644 --- a/cmd/server/issue1189_distinct_iatas_test.go +++ b/cmd/server/issue1189_distinct_iatas_test.go @@ -27,22 +27,22 @@ func TestQueryGroupedPacketsReturnsDistinctIATAs(t *testing.T) { // Observers: SJC + SFO + a third with no IATA (should be excluded). db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obsA', 'A', 'SJC', ?, '2026-01-01T00:00:00Z', 10)`, now.Format(time.RFC3339)) + VALUES ('obsA', 'A', 'SJC', $1, '2026-01-01T00:00:00Z', 10)`, now.Format(time.RFC3339)) db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obsB', 'B', 'SFO', ?, '2026-01-01T00:00:00Z', 10)`, now.Format(time.RFC3339)) + VALUES ('obsB', 'B', 'SFO', $1, '2026-01-01T00:00:00Z', 10)`, now.Format(time.RFC3339)) db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obsC', 'C', '', ?, '2026-01-01T00:00:00Z', 10)`, now.Format(time.RFC3339)) + VALUES ('obsC', 'C', '', $1, '2026-01-01T00:00:00Z', 10)`, now.Format(time.RFC3339)) // One transmission with 3 observations (SJC, SFO, no-IATA). db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('AABB', 'deadbeefcafef00d', ?, 1, 4, '{}')`, now.Format(time.RFC3339)) + VALUES ('AABB', 'deadbeefcafef00d', $1, 1, 4, '{}')`, now.Format(time.RFC3339)) // v3 schema: observer_idx = observers.rowid (auto-assigned 1,2,3 in insert order). db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 12.0, -80, '["aa"]', ?)`, recentEpoch) + VALUES (1, 1, 12.0, -80, '["aa"]', $1)`, recentEpoch) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 2, 8.0, -90, '["aa"]', ?)`, recentEpoch-30) + VALUES (1, 2, 8.0, -90, '["aa"]', $1)`, recentEpoch-30) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 3, 5.0, -95, '["aa"]', ?)`, recentEpoch-60) + VALUES (1, 3, 5.0, -95, '["aa"]', $1)`, recentEpoch-60) result, err := db.QueryGroupedPackets(PacketQuery{Limit: 50}) if err != nil { @@ -79,11 +79,11 @@ func TestQueryGroupedPacketsDistinctIATAsEmptyWhenNoIATA(t *testing.T) { recentEpoch := now.Add(-1 * time.Hour).Unix() db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obsX', 'X', '', ?, '2026-01-01T00:00:00Z', 1)`, now.Format(time.RFC3339)) + VALUES ('obsX', 'X', '', $1, '2026-01-01T00:00:00Z', 1)`, now.Format(time.RFC3339)) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('AA', '1111222233334444', ?, 1, 4, '{}')`, now.Format(time.RFC3339)) + VALUES ('AA', '1111222233334444', $1, 1, 4, '{}')`, now.Format(time.RFC3339)) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 10.0, -85, '[]', ?)`, recentEpoch) + VALUES (1, 1, 10.0, -85, '[]', $1)`, recentEpoch) result, err := db.QueryGroupedPackets(PacketQuery{Limit: 50}) if err != nil { diff --git a/cmd/server/issue1690_cold_load_test.go b/cmd/server/issue1690_cold_load_test.go index 1668e04c9..9fc02bacd 100644 --- a/cmd/server/issue1690_cold_load_test.go +++ b/cmd/server/issue1690_cold_load_test.go @@ -14,12 +14,8 @@ package main import ( "encoding/json" - "path/filepath" - "strings" "testing" "time" - - _ "github.com/mattn/go-sqlite3" ) // createTestDBWithLastSeen seeds a DB with the post-fix schema (last_seen @@ -49,15 +45,14 @@ func createTestDBWithLastSeen(t *testing.T, dbPath string, numTx, obsPerTx int, // loads zero rows; the post-fix code (filtering on last_seen) must load // all 1000. func Test1690_ColdLoad_TimeAxis(t *testing.T) { - dir := t.TempDir() - dbPath := filepath.Join(dir, "test.db") + dbPath := postgresTestDSN(t) nowSec := time.Now().UTC().Unix() createTestDBWithLastSeen(t, dbPath, 1000, 1, nowSec, 30*24*time.Hour, // first_seen = 30d ago 30*time.Minute) // last_seen = 30min ago - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatalf("OpenDB: %v", err) } @@ -81,13 +76,10 @@ func Test1690_ColdLoad_TimeAxis(t *testing.T) { } } -// Test1690_BackgroundLoadHonesty seeds 1000 transmissions but caps the -// store's memory budget so it can only fit a fraction. After -// loadBackgroundChunks runs, backgroundLoadDone must be FALSE and -// backgroundLoadFailed must be TRUE because actual coverage is < 90%. +// A configured memory cap is successful bounded loading, not a failed startup. +// Keep reporting actual database coverage rather than pretending it is 100%. func Test1690_BackgroundLoadHonesty(t *testing.T) { - dir := t.TempDir() - dbPath := filepath.Join(dir, "test.db") + dbPath := postgresTestDSN(t) nowSec := time.Now().UTC().Unix() // 5000 rows; chunkSize=500 + maxMemoryMB=1 (→ maxPackets ≈ 1000) so @@ -96,7 +88,7 @@ func Test1690_BackgroundLoadHonesty(t *testing.T) { createTestDBWithLastSeen(t, dbPath, 5000, 1, nowSec, 30*time.Minute, 30*time.Minute) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatalf("OpenDB: %v", err) } @@ -112,34 +104,27 @@ func Test1690_BackgroundLoadHonesty(t *testing.T) { } store.loadBackgroundChunks() - if store.backgroundLoadDone.Load() { - t.Errorf("backgroundLoadDone=true with only %d/5000 packets loaded; "+ - "must be false until coverage ≥ 90%%", len(store.packets)) + if !store.backgroundLoadDone.Load() { + t.Error("memory-bounded background load must reach a terminal state") } - if !store.backgroundLoadFailed.Load() { - t.Errorf("backgroundLoadFailed=false despite under-coverage "+ - "(%d/5000 packets loaded); must be true with a reason", len(store.packets)) + if store.backgroundLoadFailed.Load() || store.BackgroundLoadError() != "" { + t.Errorf("configured memory budget treated as failure: %s", store.BackgroundLoadError()) } - // The error message must mention a percentage so operators can see - // the actual ratio surface in the perf endpoint. - errMsg := store.BackgroundLoadError() - if !strings.Contains(errMsg, "%") { - t.Errorf("backgroundLoadError=%q; expected human-readable ratio "+ - "(e.g. 'loaded X%% of Y rows')", errMsg) + if got, want := store.LoadCoverageRatio(), float64(len(store.packets))/5000; got != want || got >= 0.9 { + t.Errorf("coverage = %g, want actual bounded fraction %g", got, want) } } // Test1690_PerfStats_NewFields asserts the typed perf payload exposes the // retention/coverage fields needed for prod observability. func Test1690_PerfStats_NewFields(t *testing.T) { - dir := t.TempDir() - dbPath := filepath.Join(dir, "test.db") + dbPath := postgresTestDSN(t) nowSec := time.Now().UTC().Unix() createTestDBWithLastSeen(t, dbPath, 10, 1, nowSec, 30*time.Minute, 30*time.Minute) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatalf("OpenDB: %v", err) } diff --git a/cmd/server/issue1809_bg_load_race_test.go b/cmd/server/issue1809_bg_load_race_test.go index eca19727c..df799c824 100644 --- a/cmd/server/issue1809_bg_load_race_test.go +++ b/cmd/server/issue1809_bg_load_race_test.go @@ -21,9 +21,7 @@ package main // kept intact; we add the coverage assertions on top. import ( - "database/sql" "fmt" - "path/filepath" "testing" "time" ) @@ -34,8 +32,7 @@ import ( // produces backgroundLoadFailed=true deterministically because the bg // loader reads oldestLoaded="" and bails. func Test1809_StartupLoad_BgLoaderSeesOldestLoaded(t *testing.T) { - dir := t.TempDir() - dbPath := filepath.Join(dir, "test.db") + dbPath := postgresTestDSN(t) nowSec := time.Now().UTC().Unix() // Seed 100 rows spread over 14 days with last_seen also spread, so @@ -47,7 +44,7 @@ func Test1809_StartupLoad_BgLoaderSeesOldestLoaded(t *testing.T) { const spanDays = 14 createTestDBSpreadOverDays(t, dbPath, totalRows, spanDays, nowSec) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatalf("OpenDB: %v", err) } @@ -138,7 +135,7 @@ func createTestDBSpreadOverDays(t *testing.T, dbPath string, numTx, spanDays int // (adv #11: prior code carried a duplicated annotation on every line)." func seedTestDBRows(t *testing.T, dbPath string, numTx, obsPerTx int, rowTimes func(i int) (string, int64)) { t.Helper() - conn, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL") + conn, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } @@ -150,23 +147,33 @@ func seedTestDBRows(t *testing.T, dbPath string, numTx, obsPerTx int, rowTimes f } } // PREFLIGHT: async=true reason="unit-test fixture seeder; in-memory ephemeral SQLite; all CREATE/INSERT below covered by this block annotation (pr-preflight 25-line lookback window)" - execOrFail(`CREATE TABLE transmissions ( - id INTEGER PRIMARY KEY, + execOrFail(testNativeSQL(`CREATE TABLE transmissions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT, last_seen INTEGER NOT NULL DEFAULT 0 - )`) + )`, `CREATE TABLE transmissions ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + raw_hex TEXT, hash TEXT, first_seen TEXT, + route_type INTEGER, payload_type INTEGER, + payload_version INTEGER, decoded_json TEXT, + last_seen INTEGER NOT NULL DEFAULT 0 + )`)) // PREFLIGHT: async=true reason="unit-test fixture seeder" - execOrFail(`CREATE TABLE observations ( - id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, - direction TEXT, snr REAL, rssi REAL, score INTEGER, + execOrFail(testNativeSQL(`CREATE TABLE observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT - )`) + )`, `CREATE TABLE observations ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, + path_json TEXT, timestamp TEXT, raw_hex TEXT + )`)) // PREFLIGHT: async=true reason="unit-test fixture seeder" - execOrFail(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) + execOrFail(testNativeSQL(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY AUTOINCREMENT, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, `CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`)) // PREFLIGHT: async=true reason="unit-test fixture seeder" - execOrFail(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, frequency REAL)`) + execOrFail(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, first_seen TEXT, frequency DOUBLE PRECISION)`) // PREFLIGHT: async=true reason="unit-test fixture seeder" execOrFail(`CREATE TABLE schema_version (version INTEGER)`) execOrFail(`INSERT INTO schema_version (version) VALUES (1)`) @@ -175,30 +182,15 @@ func seedTestDBRows(t *testing.T, dbPath string, numTx, obsPerTx int, rowTimes f // PREFLIGHT: async=true reason="unit-test fixture seeder" execOrFail(`CREATE INDEX idx_tx_last_seen ON transmissions(last_seen)`) - txStmt, err := conn.Prepare("INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json, last_seen) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)") - if err != nil { - t.Fatalf("prepare tx: %v", err) - } - defer txStmt.Close() - obsStmt, err := conn.Prepare("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") - if err != nil { - t.Fatalf("prepare obs: %v", err) - } - defer obsStmt.Close() - - obsID := 1 - for i := 1; i <= numTx; i++ { - firstSeenStr, lastSeenUnix := rowTimes(i) - hash := fmt.Sprintf("h%06d", i) - if _, err := txStmt.Exec(i, "aabb", hash, firstSeenStr, 0, 4, 1, "{}", lastSeenUnix); err != nil { - t.Fatalf("insert tx %d: %v", i, err) - } - for j := 0; j < obsPerTx; j++ { - obsTs := time.Unix(lastSeenUnix, 0).UTC().Add(-time.Duration(j) * time.Minute).Format(time.RFC3339) - if _, err := obsStmt.Exec(obsID, i, "obs1", "Obs1", "RX", -10.0, -80.0, 5, "[]", obsTs); err != nil { - t.Fatalf("insert obs: %v", err) - } - obsID++ - } - } + copyTestRows(t, conn, "transmissions", []string{"id", "raw_hex", "hash", "first_seen", "route_type", "payload_type", "payload_version", "decoded_json", "last_seen"}, numTx, func(n int) []any { + i := n + 1 + firstSeen, lastSeen := rowTimes(i) + return []any{i, "aabb", fmt.Sprintf("h%06d", i), firstSeen, 0, 4, 1, "{}", lastSeen} + }) + copyTestRows(t, conn, "observations", []string{"id", "transmission_id", "observer_id", "observer_name", "direction", "snr", "rssi", "score", "path_json", "timestamp"}, numTx*obsPerTx, func(n int) []any { + i, j := n/obsPerTx+1, n%obsPerTx + _, lastSeen := rowTimes(i) + stamp := time.Unix(lastSeen, 0).UTC().Add(-time.Duration(j) * time.Minute).Format(time.RFC3339) + return []any{n + 1, i, "obs1", "Obs1", "RX", -10.0, -80.0, 5, "[]", stamp} + }) } diff --git a/cmd/server/issue673_test.go b/cmd/server/issue673_test.go index 85337c9d7..0295270b8 100644 --- a/cmd/server/issue673_test.go +++ b/cmd/server/issue673_test.go @@ -4,8 +4,6 @@ import ( "encoding/json" "testing" "time" - - _ "github.com/mattn/go-sqlite3" ) const issue673NodePK = "7502f19f44cad6d7b626e1d811c00a914af452636182ccded3fd019803395ec9" @@ -18,7 +16,7 @@ func setupIssue673Store(t *testing.T) (*PacketStore, *DB) { db := setupTestDB(t) _, err := db.conn.Exec( - "INSERT INTO nodes (public_key, name, role) VALUES (?, ?, ?)", + "INSERT INTO nodes (public_key, name, role) VALUES ($1, $2, $3)", issue673NodePK, "Quail Hollow Park", "repeater", ) if err != nil { diff --git a/cmd/server/issue804_repeater_region_test.go b/cmd/server/issue804_repeater_region_test.go index 05e7a7a5c..a6bd86600 100644 --- a/cmd/server/issue804_repeater_region_test.go +++ b/cmd/server/issue804_repeater_region_test.go @@ -34,19 +34,19 @@ func TestIssue804_AnalyticsAttributesByRepeaterRegion(t *testing.T) { // Observers: one in PDX, one in SJC db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obs-pdx', 'Obs PDX', 'PDX', ?, '2026-01-01T00:00:00Z', 100)`, recent) + VALUES ('obs-pdx', 'Obs PDX', 'PDX', $1, '2026-01-01T00:00:00Z', 100)`, recent) db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obs-sjc', 'Obs SJC', 'SJC', ?, '2026-01-01T00:00:00Z', 100)`, recent) + VALUES ('obs-sjc', 'Obs SJC', 'SJC', $1, '2026-01-01T00:00:00Z', 100)`, recent) // PDX-Repeater node (lives in Portland) pdxPK := "pdx0000000000001" db.conn.Exec(`INSERT INTO nodes (public_key, name, role) - VALUES (?, 'PDX-Repeater', 'repeater')`, pdxPK) + VALUES ($1, 'PDX-Repeater', 'repeater')`, pdxPK) // SJC-Repeater node (lives in San Jose) — sanity baseline sjcPK := "sjc0000000000001" db.conn.Exec(`INSERT INTO nodes (public_key, name, role) - VALUES (?, 'SJC-Repeater', 'repeater')`, sjcPK) + VALUES ($1, 'SJC-Repeater', 'repeater')`, sjcPK) pdxDecoded := `{"pubKey":"` + pdxPK + `","name":"PDX-Repeater","type":"ADVERT","flags":{"isRepeater":true}}` sjcDecoded := `{"pubKey":"` + sjcPK + `","name":"SJC-Repeater","type":"ADVERT","flags":{"isRepeater":true}}` @@ -56,31 +56,31 @@ func TestIssue804_AnalyticsAttributesByRepeaterRegion(t *testing.T) { // raw_hex header 0x12 = route_type 2 (direct), payload_type 4 // pathByte 0x40 (hashSize bits=01 → 2, hop_count=0) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('1240aabbccdd', 'pdx_zh_direct', ?, 2, 4, ?)`, recent, pdxDecoded) + VALUES ('1240aabbccdd', 'pdx_zh_direct', $1, 2, 4, $2)`, recent, pdxDecoded) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 12.0, -85, '[]', ?)`, recentEpoch) + VALUES (1, 1, 12.0, -85, '[]', $1)`, recentEpoch) // 2) PDX-Repeater FLOOD advert with hashSize=2 (reliable). // Heard ONLY by obs-SJC via a relay path (this is the polluting case). // raw_hex header 0x11 = route_type 1 (flood), payload_type 4 // pathByte 0x41 (hashSize bits=01 → 2, hop_count=1) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('1141aabbccdd', 'pdx_flood', ?, 1, 4, ?)`, recent, pdxDecoded) + VALUES ('1141aabbccdd', 'pdx_flood', $1, 1, 4, $2)`, recent, pdxDecoded) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (2, 2, 8.0, -95, '["aa11"]', ?)`, recentEpoch) + VALUES (2, 2, 8.0, -95, '["aa11"]', $1)`, recentEpoch) // 3) SJC-Repeater zero-hop DIRECT advert heard only by obs-SJC. // Establishes SJC as the repeater's home region. db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('1240ccddeeff', 'sjc_zh_direct', ?, 2, 4, ?)`, recent, sjcDecoded) + VALUES ('1240ccddeeff', 'sjc_zh_direct', $1, 2, 4, $2)`, recent, sjcDecoded) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (3, 2, 14.0, -82, '[]', ?)`, recentEpoch) + VALUES (3, 2, 14.0, -82, '[]', $1)`, recentEpoch) // 4) SJC-Repeater FLOOD advert with hashSize=2, heard by obs-SJC. db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('1141ccddeeff', 'sjc_flood', ?, 1, 4, ?)`, recent, sjcDecoded) + VALUES ('1141ccddeeff', 'sjc_flood', $1, 1, 4, $2)`, recent, sjcDecoded) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (4, 2, 11.0, -88, '["cc22"]', ?)`, recentEpoch) + VALUES (4, 2, 11.0, -88, '["cc22"]', $1)`, recentEpoch) store := NewPacketStore(db, nil) store.Load() diff --git a/cmd/server/issue810_repro_test.go b/cmd/server/issue810_repro_test.go index cd784d9f7..d24b3d118 100644 --- a/cmd/server/issue810_repro_test.go +++ b/cmd/server/issue810_repro_test.go @@ -18,14 +18,14 @@ func TestRepro810(t *testing.T) { now := time.Now().UTC() recent := now.Add(-1 * time.Hour).Format(time.RFC3339) recentEpoch := now.Add(-1 * time.Hour).Unix() - db.conn.Exec(`INSERT INTO observers (id, name, last_seen, first_seen, packet_count) VALUES ('obs1','O1',?, '2026-01-01T00:00:00Z', 100)`, recent) - db.conn.Exec(`INSERT INTO observers (id, name, last_seen, first_seen, packet_count) VALUES ('obs2','O2',?, '2026-01-01T00:00:00Z', 100)`, recent) - db.conn.Exec(`INSERT INTO nodes (public_key, name, role, last_seen, first_seen, advert_count) VALUES ('aabbccdd11223344','R','repeater',?, '2026-01-01T00:00:00Z', 1)`, recent) - db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) VALUES ('AABB','testhash00000001',?,1,4,'{"pubKey":"aabbccdd11223344","type":"ADVERT"}')`, recent) + db.conn.Exec(`INSERT INTO observers (id, name, last_seen, first_seen, packet_count) VALUES ('obs1','O1',$1, '2026-01-01T00:00:00Z', 100)`, recent) + db.conn.Exec(`INSERT INTO observers (id, name, last_seen, first_seen, packet_count) VALUES ('obs2','O2',$1, '2026-01-01T00:00:00Z', 100)`, recent) + db.conn.Exec(`INSERT INTO nodes (public_key, name, role, last_seen, first_seen, advert_count) VALUES ('aabbccdd11223344','R','repeater',$1, '2026-01-01T00:00:00Z', 1)`, recent) + db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) VALUES ('AABB','testhash00000001',$1,1,4,'{"pubKey":"aabbccdd11223344","type":"ADVERT"}')`, recent) // Longest-path obs WITHOUT resolved_path - db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) VALUES (1,1,12.5,-90,'["aa","bb","cc"]',?)`, recentEpoch) + db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) VALUES (1,1,12.5,-90,'["aa","bb","cc"]',$1)`, recentEpoch) // Shorter-path obs WITH resolved_path - db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp, resolved_path) VALUES (1,2,8.0,-95,'["aa","bb"]',?,'["aabbccdd11223344","eeff00112233aabb"]')`, recentEpoch-100) + db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp, resolved_path) VALUES (1,2,8.0,-95,'["aa","bb"]',$1,'["aabbccdd11223344","eeff00112233aabb"]')`, recentEpoch-100) cfg := &Config{Port: 3000} hub := NewHub() diff --git a/cmd/server/issue871_test.go b/cmd/server/issue871_test.go index 43f2768aa..2539df728 100644 --- a/cmd/server/issue871_test.go +++ b/cmd/server/issue871_test.go @@ -19,7 +19,7 @@ func TestIssue871_NoNullHashOrTimestamp(t *testing.T) { // Insert bad legacy data: packet with empty hash now := time.Now().UTC().Add(-30 * time.Minute).Format(time.RFC3339) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('DEAD', '', ?, 1, 4, '{}')`, now) + VALUES ('DEAD', '', $1, 1, 4, '{}')`, now) // Insert bad legacy data: packet with NULL first_seen (timestamp) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) VALUES ('BEEF', 'aa11bb22cc33dd44', NULL, 1, 4, '{}')`) diff --git a/cmd/server/load_graph_invariant_test.go b/cmd/server/load_graph_invariant_test.go index 07ae649f0..32cd88320 100644 --- a/cmd/server/load_graph_invariant_test.go +++ b/cmd/server/load_graph_invariant_test.go @@ -1,12 +1,8 @@ package main import ( - "database/sql" - "path/filepath" "testing" "time" - - _ "github.com/mattn/go-sqlite3" ) // TestLoad_PanicsWhenGraphNotLoadedAndEdgesExist pins the startup-ordering @@ -19,10 +15,9 @@ import ( // Load() must panic when neighbor_edges has rows but s.graph.Load() returns // nil. Fast-fail at startup beats silently-wrong attribution. func TestLoad_PanicsWhenGraphNotLoadedAndEdgesExist(t *testing.T) { - dir := t.TempDir() - dbPath := filepath.Join(dir, "test.db") + dbPath := postgresTestDSN(t) - rw, err := sql.Open("sqlite3", "file:"+dbPath+"?_journal_mode=WAL") + rw, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } @@ -35,24 +30,34 @@ func TestLoad_PanicsWhenGraphNotLoadedAndEdgesExist(t *testing.T) { } // Minimal CoreScope schema. PREFLIGHT: async=true reason="test fixture, in-memory tmpdir DB" - exec(`CREATE TABLE transmissions ( - id INTEGER PRIMARY KEY, + exec(testNativeSQL(`CREATE TABLE transmissions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT - )`) + )`, `CREATE TABLE transmissions ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + raw_hex TEXT, hash TEXT, first_seen TEXT, + route_type INTEGER, payload_type INTEGER, payload_version INTEGER, + decoded_json TEXT + )`)) // PREFLIGHT: async=true reason="test fixture, in-memory tmpdir DB" - exec(`CREATE TABLE observations ( - id INTEGER PRIMARY KEY, transmission_id INTEGER, + exec(testNativeSQL(`CREATE TABLE observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, - direction TEXT, snr REAL, rssi REAL, score INTEGER, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT, resolved_path TEXT - )`) + )`, `CREATE TABLE observations ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, + observer_id TEXT, observer_name TEXT, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, + path_json TEXT, timestamp TEXT, raw_hex TEXT, resolved_path TEXT + )`)) // PREFLIGHT: async=true reason="test fixture, in-memory tmpdir DB" - exec(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) + exec(testNativeSQL(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY AUTOINCREMENT, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, `CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`)) // PREFLIGHT: async=true reason="test fixture, in-memory tmpdir DB" exec(`CREATE TABLE nodes ( - public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, + public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, first_seen TEXT, advert_count INTEGER DEFAULT 0 )`) // PREFLIGHT: async=true reason="test fixture, in-memory tmpdir DB" @@ -67,10 +72,10 @@ func TestLoad_PanicsWhenGraphNotLoadedAndEdgesExist(t *testing.T) { PRIMARY KEY (node_a, node_b) )`) now := time.Now().UTC().Format(time.RFC3339) - exec(`INSERT INTO neighbor_edges (node_a, node_b, count, last_seen) VALUES (?, ?, ?, ?)`, + exec(`INSERT INTO neighbor_edges (node_a, node_b, count, last_seen) VALUES ($1, $2, $3, $4)`, "aaa", "bbb", 5, now) - d, err := OpenDB(dbPath) + d, err := openFixtureReader(t, dbPath) if err != nil { t.Fatalf("OpenDB: %v", err) } diff --git a/cmd/server/loadchunk_ambiguous_prefix_test.go b/cmd/server/loadchunk_ambiguous_prefix_test.go index 3c0e1596b..c4ab913cf 100644 --- a/cmd/server/loadchunk_ambiguous_prefix_test.go +++ b/cmd/server/loadchunk_ambiguous_prefix_test.go @@ -1,13 +1,9 @@ package main import ( - "database/sql" "fmt" - "path/filepath" "testing" "time" - - _ "github.com/mattn/go-sqlite3" ) // createTestDBAmbiguousPrefix builds a fixture where TWO repeaters share the @@ -21,10 +17,9 @@ import ( // picks today's tier-4 winner against ~7-day-old observations. func createTestDBAmbiguousPrefix(t *testing.T, relayA, relayB, hop, firstSeen string) string { t.Helper() - dir := t.TempDir() - dbPath := filepath.Join(dir, "test.db") + dbPath := postgresTestDSN(t) - conn, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL") + conn, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } @@ -37,27 +32,40 @@ func createTestDBAmbiguousPrefix(t *testing.T, relayA, relayB, hop, firstSeen st } // PREFLIGHT: async=true reason="test fixture: in-memory t.TempDir SQLite, never touches a real DB." - exec(`CREATE TABLE transmissions ( - id INTEGER PRIMARY KEY, + exec(testNativeSQL(`CREATE TABLE transmissions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT - )`) + )`, `CREATE TABLE transmissions ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + raw_hex TEXT, hash TEXT, first_seen TEXT, + route_type INTEGER, payload_type INTEGER, payload_version INTEGER, + decoded_json TEXT + )`)) // PREFLIGHT: async=true reason="test fixture, in-memory tmpdir DB" - exec(`CREATE TABLE observations ( - id INTEGER PRIMARY KEY, + exec(testNativeSQL(`CREATE TABLE observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, - direction TEXT, snr REAL, rssi REAL, score INTEGER, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT, resolved_path TEXT - )`) + )`, `CREATE TABLE observations ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + transmission_id INTEGER, + observer_id TEXT, observer_name TEXT, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, + path_json TEXT, timestamp TEXT, + raw_hex TEXT, + resolved_path TEXT + )`)) // PREFLIGHT: async=true reason="test fixture, in-memory tmpdir DB" - exec(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) + exec(testNativeSQL(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY AUTOINCREMENT, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, `CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`)) // PREFLIGHT: async=true reason="test fixture, in-memory tmpdir DB" exec(`CREATE TABLE nodes ( - public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, + public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, first_seen TEXT, advert_count INTEGER DEFAULT 0 )`) // PREFLIGHT: async=true reason="test fixture, in-memory tmpdir DB" @@ -69,15 +77,15 @@ func createTestDBAmbiguousPrefix(t *testing.T, relayA, relayB, hop, firstSeen st // Two repeaters sharing the same 2-char prefix `hop`. // Different advert_counts so tier-4 tiebreak deterministically picks one // (proving the bug: it over-attributes to the higher-count node). - exec(`INSERT INTO nodes (public_key, name, role, advert_count) VALUES (?,?,?,?)`, + exec(`INSERT INTO nodes (public_key, name, role, advert_count) VALUES ($1,$2,$3,$4)`, relayA, "Relay A", "repeater", 50) - exec(`INSERT INTO nodes (public_key, name, role, advert_count) VALUES (?,?,?,?)`, + exec(`INSERT INTO nodes (public_key, name, role, advert_count) VALUES ($1,$2,$3,$4)`, relayB, "Relay B", "repeater", 10) // Aged 48h so it lands in the background window (loadChunk path). - exec("INSERT INTO transmissions VALUES (?,?,?,?,0,4,1,?)", + exec("INSERT INTO transmissions VALUES ($1,$2,$3,$4,0,4,1,$5)", 1, "aa", "hashamb_1", firstSeen, `{}`) - exec("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp, raw_hex, resolved_path) VALUES (?,?,?,?,?,?,?,?,?,?,?,NULL)", + exec("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp, raw_hex, resolved_path) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,NULL)", 1, 1, "obs1", "Obs1", "RX", -10.0, -80.0, 5, fmt.Sprintf(`[%q]`, hop), firstSeen, "") return dbPath @@ -97,7 +105,7 @@ func TestLoadChunk_AmbiguousPrefix_SkipsAttribution(t *testing.T) { aged := time.Now().UTC().Add(-48 * time.Hour).Format(time.RFC3339) dbPath := createTestDBAmbiguousPrefix(t, relayA, relayB, hop, aged) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } @@ -148,7 +156,7 @@ func TestLoad_AmbiguousPrefix_SkipsAttribution(t *testing.T) { ts := time.Now().UTC().Format(time.RFC3339) dbPath := createTestDBAmbiguousPrefix(t, relayA, relayB, hop, ts) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } diff --git a/cmd/server/loadchunk_observer_iata_test.go b/cmd/server/loadchunk_observer_iata_test.go index 715518ca5..880714e60 100644 --- a/cmd/server/loadchunk_observer_iata_test.go +++ b/cmd/server/loadchunk_observer_iata_test.go @@ -1,9 +1,6 @@ package main import ( - "database/sql" - "os" - "path/filepath" "testing" "time" ) @@ -13,11 +10,9 @@ import ( // IATA code, so packets older than the hot window lost observer_iata in the // packets API and the iata filter (#1188), while Load and ingest kept it. func TestLoadChunkKeepsObserverIATA(t *testing.T) { - dir := t.TempDir() - dbPath := filepath.Join(dir, "iata.db") + dbPath := postgresTestDSN(t) createTestDBAt(t, dbPath, 20) - t.Cleanup(func() { os.RemoveAll(dir) }) - conn, err := sql.Open("sqlite3", dbPath) + conn, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } @@ -26,7 +21,7 @@ func TestLoadChunkKeepsObserverIATA(t *testing.T) { } conn.Close() - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatalf("OpenDB: %v", err) } diff --git a/cmd/server/loadchunk_pathjson_fallback_test.go b/cmd/server/loadchunk_pathjson_fallback_test.go index 9d28b6203..70757cdd8 100644 --- a/cmd/server/loadchunk_pathjson_fallback_test.go +++ b/cmd/server/loadchunk_pathjson_fallback_test.go @@ -1,13 +1,9 @@ package main import ( - "database/sql" "fmt" - "path/filepath" "testing" "time" - - _ "github.com/mattn/go-sqlite3" ) // createTestDBPathJSONNoResolvedPath builds a fixture that mirrors the LIVE @@ -22,10 +18,9 @@ import ( // s.byNode is via path_json → resolvePathForObs relay-hop resolution. func createTestDBPathJSONNoResolvedPath(t *testing.T, relayPubkey, hopPrefix, firstSeen string) string { t.Helper() - dir := t.TempDir() - dbPath := filepath.Join(dir, "test.db") + dbPath := postgresTestDSN(t) - conn, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL") + conn, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } @@ -38,30 +33,43 @@ func createTestDBPathJSONNoResolvedPath(t *testing.T, relayPubkey, hopPrefix, fi } // PREFLIGHT: async=true reason="test fixture: in-memory t.TempDir SQLite, never touches a real DB. Tables are CREATE-from-empty in a one-shot OpenDB call, not a schema migration over existing data." - exec(`CREATE TABLE transmissions ( - id INTEGER PRIMARY KEY, + exec(testNativeSQL(`CREATE TABLE transmissions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT - )`) + )`, `CREATE TABLE transmissions ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + raw_hex TEXT, hash TEXT, first_seen TEXT, + route_type INTEGER, payload_type INTEGER, payload_version INTEGER, + decoded_json TEXT + )`)) // resolved_path column present (matches live schema) but left NULL. // PREFLIGHT: async=true reason="test fixture, in-memory tmpdir DB" - exec(`CREATE TABLE observations ( - id INTEGER PRIMARY KEY, + exec(testNativeSQL(`CREATE TABLE observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, - direction TEXT, snr REAL, rssi REAL, score INTEGER, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT, resolved_path TEXT - )`) + )`, `CREATE TABLE observations ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + transmission_id INTEGER, + observer_id TEXT, observer_name TEXT, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, + path_json TEXT, timestamp TEXT, + raw_hex TEXT, + resolved_path TEXT + )`)) // PREFLIGHT: async=true reason="test fixture, in-memory tmpdir DB" - exec(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) + exec(testNativeSQL(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY AUTOINCREMENT, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, `CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`)) // Production nodes schema uses public_key (not pubkey) — getAllNodes / // buildPrefixMap reads public_key, role, advert_count, first_seen. // PREFLIGHT: async=true reason="test fixture, in-memory tmpdir DB" exec(`CREATE TABLE nodes ( - public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, + public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, first_seen TEXT, advert_count INTEGER DEFAULT 0 )`) // PREFLIGHT: async=true reason="test fixture, in-memory tmpdir DB" @@ -71,13 +79,13 @@ func createTestDBPathJSONNoResolvedPath(t *testing.T, relayPubkey, hopPrefix, fi exec(`CREATE INDEX idx_tx_first_seen ON transmissions(first_seen)`) // Repeater node so canAppearInPath() admits it to the prefix map. - exec(`INSERT INTO nodes (public_key, name, role, advert_count) VALUES (?,?,?,?)`, + exec(`INSERT INTO nodes (public_key, name, role, advert_count) VALUES ($1,$2,$3,$4)`, relayPubkey, "Relay One", "repeater", 10) - exec("INSERT INTO transmissions VALUES (?,?,?,?,0,4,1,?)", + exec("INSERT INTO transmissions VALUES ($1,$2,$3,$4,0,4,1,$5)", 1, "aa", "hashpjf_1", firstSeen, `{}`) // resolved_path explicitly NULL; path_json carries the relay hop prefix. - exec("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp, raw_hex, resolved_path) VALUES (?,?,?,?,?,?,?,?,?,?,?,NULL)", + exec("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp, raw_hex, resolved_path) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,NULL)", 1, 1, "obs1", "Obs1", "RX", -10.0, -80.0, 5, fmt.Sprintf(`[%q]`, hopPrefix), firstSeen, "") return dbPath @@ -103,7 +111,7 @@ func TestLoadChunked_ResolvesRelayHopsFromPathJSON_WhenResolvedPathEmpty(t *test ts := time.Now().UTC().Format(time.RFC3339) dbPath := createTestDBPathJSONNoResolvedPath(t, relayPK, hop, ts) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } @@ -146,7 +154,7 @@ func TestLoadChunk_ResolvesRelayHopsFromPathJSON_WhenResolvedPathEmpty(t *testin aged := time.Now().UTC().Add(-48 * time.Hour).Format(time.RFC3339) dbPath := createTestDBPathJSONNoResolvedPath(t, relayPK, hop, aged) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } diff --git a/cmd/server/loadchunk_resolved_path_1558_test.go b/cmd/server/loadchunk_resolved_path_1558_test.go index 9cd83ecd5..81373a899 100644 --- a/cmd/server/loadchunk_resolved_path_1558_test.go +++ b/cmd/server/loadchunk_resolved_path_1558_test.go @@ -1,13 +1,9 @@ package main import ( - "database/sql" "fmt" - "path/filepath" "testing" "time" - - _ "github.com/mattn/go-sqlite3" ) // createTestDBWithResolvedPath creates a fixture DB containing numTx old @@ -17,10 +13,9 @@ import ( // column so loadChunk's hasResolvedPath branch is exercised. func createTestDBWithResolvedPath(t *testing.T, numTx int, relayPubkeys []string) string { t.Helper() - dir := t.TempDir() - dbPath := filepath.Join(dir, "test.db") + dbPath := postgresTestDSN(t) - conn, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL") + conn, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } @@ -32,23 +27,36 @@ func createTestDBWithResolvedPath(t *testing.T, numTx int, relayPubkeys []string } } - exec(`CREATE TABLE transmissions ( - id INTEGER PRIMARY KEY, + exec(testNativeSQL(`CREATE TABLE transmissions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT - )`) - exec(`CREATE TABLE observations ( - id INTEGER PRIMARY KEY, + )`, `CREATE TABLE transmissions ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + raw_hex TEXT, hash TEXT, first_seen TEXT, + route_type INTEGER, payload_type INTEGER, payload_version INTEGER, + decoded_json TEXT + )`)) + exec(testNativeSQL(`CREATE TABLE observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + transmission_id INTEGER, + observer_id TEXT, observer_name TEXT, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, + path_json TEXT, timestamp TEXT, + raw_hex TEXT, + resolved_path TEXT + )`, `CREATE TABLE observations ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, - direction TEXT, snr REAL, rssi REAL, score INTEGER, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT, resolved_path TEXT - )`) - exec(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) - exec(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, frequency REAL)`) + )`)) + exec(testNativeSQL(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY AUTOINCREMENT, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, `CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`)) + exec(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, first_seen TEXT, frequency DOUBLE PRECISION)`) exec(`CREATE TABLE schema_version (version INTEGER)`) exec(`INSERT INTO schema_version (version) VALUES (1)`) exec(`CREATE INDEX idx_tx_first_seen ON transmissions(first_seen)`) @@ -67,9 +75,9 @@ func createTestDBWithResolvedPath(t *testing.T, numTx int, relayPubkeys []string for i := 0; i < numTx; i++ { ts := now.Add(-48 * time.Hour).Add(time.Duration(i) * time.Second).Format(time.RFC3339) hash := fmt.Sprintf("hash1558_%d", i) - exec("INSERT INTO transmissions VALUES (?,?,?,?,0,4,1,?)", + exec("INSERT INTO transmissions VALUES ($1,$2,$3,$4,0,4,1,$5)", i+1, "aa", hash, ts, `{}`) - exec("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp, raw_hex, resolved_path) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)", + exec("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp, raw_hex, resolved_path) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12)", i+1, i+1, "obs1", "Obs1", "RX", -10.0, -80.0, 5, `[]`, ts, "", rpJSON) } return dbPath @@ -107,7 +115,7 @@ func TestLoadChunk_IndexesResolvedPathPubkeys_Issue1558(t *testing.T) { dbPath := createTestDBWithResolvedPath(t, 3, []string{relayPK1, relayPK2}) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } diff --git a/cmd/server/main.go b/cmd/server/main.go index 582708db3..c025e1d32 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -2,7 +2,6 @@ package main import ( "context" - "database/sql" "flag" "fmt" "log" @@ -18,7 +17,7 @@ import ( "time" "github.com/gorilla/mux" - "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/dbconfig" ) // Set via -ldflags at build time @@ -86,24 +85,30 @@ func main() { } var ( - configDir string - port int - dbPath string - publicDir string - pollMs int + configDir string + port int + legacyDB string + backend string + databaseURL string + stateDir string + publicDir string + pollMs int ) flag.StringVar(&configDir, "config-dir", ".", "Directory containing config.json") flag.IntVar(&port, "port", 0, "HTTP port (overrides config)") - flag.StringVar(&dbPath, "db", "", "SQLite database path (overrides config/env)") + flag.StringVar(&legacyDB, "db", "", "SQLite telemetry path (bootstrap override)") + flag.StringVar(&backend, "backend", "", "Storage backend: sqlite or postgres (bootstrap choice)") + flag.StringVar(&databaseURL, "database-url", "", "PostgreSQL telemetry reader URL (overrides config/env)") + flag.StringVar(&stateDir, "state-dir", "", "Local directory for queues and sidecars") flag.StringVar(&publicDir, "public", "public", "Directory to serve static files from") - flag.IntVar(&pollMs, "poll-ms", 1000, "SQLite poll interval for WebSocket broadcast (ms)") + flag.IntVar(&pollMs, "poll-ms", 1000, "Database poll interval for WebSocket broadcast (ms)") flag.Parse() // Load config cfg, err := LoadConfig(configDir) if err != nil { - log.Printf("[config] warning: %v (using defaults)", err) + log.Fatalf("[config] %v", err) } // CLI flags override config @@ -113,9 +118,7 @@ func main() { if cfg.Port == 0 { cfg.Port = 3000 } - if dbPath != "" { - cfg.DBPath = dbPath - } + if cfg.APIKey == "" { log.Printf("[security] WARNING: no apiKey configured — write endpoints are BLOCKED (set apiKey in config.json to enable them)") } else if IsWeakAPIKey(cfg.APIKey) { @@ -157,9 +160,18 @@ func main() { warnIfMemlimitUnderprovisioned(limit) } - // Resolve DB path - resolvedDB := cfg.ResolveDBPath(configDir) - log.Printf("[config] port=%d db=%s public=%s", cfg.Port, resolvedDB, publicDir) + rawStorage, err := cfg.storageInputs(configDir, storageFlags{Backend: dbconfig.Backend(backend), DBPath: legacyDB, DatabaseURL: databaseURL, StateDir: stateDir}, os.Getenv) + if err != nil { + log.Fatalf("[config] %v", err) + } + storage, selectionLease, err := resolveRuntimeStorage(rawStorage) + if err != nil { + log.Fatalf("[storage] %v", err) + } + defer selectionLease.Close() + runtimeStateDir = storage.StateDir + log.Printf("[config] port=%d database=%s public=%s", cfg.Port, storage.Backend, publicDir) + if len(cfg.NodeBlacklist) > 0 { log.Printf("[config] nodeBlacklist: %d node(s) will be hidden from API", len(cfg.NodeBlacklist)) for _, pk := range cfg.NodeBlacklist { @@ -170,10 +182,11 @@ func main() { } // Open database - database, err := OpenDB(resolvedDB) + database, err := OpenStorage(storage) if err != nil { - log.Fatalf("[db] failed to open %s: %v", resolvedDB, err) + log.Fatalf("[db] failed to open selected database: %v", err) } + database.stateDir = runtimeStateDir var dbCloseOnce sync.Once dbClose := func() error { var err error @@ -182,13 +195,6 @@ func main() { } defer dbClose() - // Verify DB has expected tables - var tableName string - err = database.conn.QueryRow("SELECT name FROM sqlite_master WHERE type='table' AND name='transmissions'").Scan(&tableName) - if err == sql.ErrNoRows { - log.Fatalf("[db] table 'transmissions' not found — is this a CoreScope database?") - } - stats, err := database.GetStats() if err != nil { log.Printf("[db] warning: could not read stats: %v", err) @@ -204,8 +210,8 @@ func main() { // (#1287). The server NEVER migrates — it only reads. If a required // column/index/table is missing, the operator must restart the // ingestor (which owns dbschema.Apply) before this server can start. - if err := dbschema.AssertReady(database.conn); err != nil { - log.Fatalf("[db] schema not ready (ingestor must run migrations first): %v", err) + if err := database.AssertReady(); err != nil { + log.Fatalf("[db] schema not ready (run the selected backend setup/writer first): %v", err) } // In-memory packet store @@ -285,7 +291,6 @@ func main() { // loaded above, before the packet load. Per #1287 schema migrations // all live in the ingestor; the server only reads the snapshot and // then refreshes it via the recompNeighborGraph slot every 60s. - dbPath = database.path database.hasResolvedPath = true // dbschema.AssertReady above already verified observations.resolved_path exists // WaitGroup for background init steps that gate /api/healthz readiness. @@ -369,7 +374,7 @@ func main() { srv.store = store // Optional user management (off by default). Fails startup on a bad // config rather than running with registration that cannot work. - if err := srv.initUserManagement(resolvedDB); err != nil { + if err := srv.initUserManagementStorage(storage); err != nil { log.Fatalf("[users] %v", err) } router := mux.NewRouter() @@ -392,7 +397,7 @@ func main() { }) } - // Start SQLite poller for WebSocket broadcast + // Start PostgreSQL poller for WebSocket broadcast poller := NewPoller(database, hub, time.Duration(pollMs)*time.Millisecond) poller.store = store go poller.Start() @@ -555,7 +560,7 @@ func main() { // 1c. Stop steady-state analytics recomputers (issue #1240). // Must happen before dbClose so any in-flight compute that - // reaches into SQLite has finished. + // queries PostgreSQL has finished. if stopAnalyticsRecomp != nil { stopAnalyticsRecomp() } @@ -573,7 +578,7 @@ func main() { // 3b. Close users.db (user management, opt-in; no-op when off). srv.closeUserManagement() - // 4. Close database (release SQLite WAL lock) + // 4. Close the database connection pool if err := dbClose(); err != nil { log.Printf("[server] DB close error: %v", err) } @@ -587,8 +592,8 @@ func main() { // process. The server reads the results via the periodic // recompNeighborGraph / fetchResolvedPathForObs paths. - // Migrate old content hashes in background (one-time, idempotent). - go migrateContentHashesAsync(store, 5000, 100*time.Millisecond) + // Verify loaded content hashes without issuing telemetry writes. + go verifyContentHashesAsync(store, 5000, 100*time.Millisecond) if err := httpServer.ListenAndServe(); err != http.ErrServerClosed { log.Fatalf("[server] %v", err) diff --git a/cmd/server/memlimit.go b/cmd/server/memlimit.go index f8e3df5c8..4cbfa1d3b 100644 --- a/cmd/server/memlimit.go +++ b/cmd/server/memlimit.go @@ -37,20 +37,9 @@ func applyMemoryLimit(maxMemoryMB int, envSet bool) (int64, string) { // transient peaks (cold-load row-scan / decode pipeline, Go's NextGC // trigger at ~2x live heap). See issue #836 heap profile. // - // GOMEMLIMIT covers memory the Go runtime manages — heap, stacks and - // runtime structures — and nothing else. Since the SQLite driver became - // cgo (github.com/mattn/go-sqlite3), what SQLite allocates in C is - // outside it, so this limit is not an RSS ceiling: actual RSS is this - // plus the native allocations plus whatever the runtime has reserved - // but not returned. - // - // The page cache, at least, is deliberately bounded: both DSNs pin - // _cache_size=-2000, i.e. ~2 MiB per connection, ~8 MiB across - // SetMaxOpenConns(4). That fits inside the headroom above rather than - // eating into the store budget. It does not bound SQLite's other native - // allocations, so if the connection count or _cache_size grows - // materially, or a workload starts holding many prepared statements, - // this derivation needs revisiting against measured RSS. + // GOMEMLIMIT covers Go-managed memory, not an RSS ceiling. PostgreSQL + // runs in another process and needs its own memory budget; its buffers + // and work memory are not represented by this packet-store limit. limit := int64(maxMemoryMB) * 1024 * 1024 * 3 / 2 debug.SetMemoryLimit(limit) return limit, "derived" diff --git a/cmd/server/memory.go b/cmd/server/memory.go index 8ad52b516..2b43a67c0 100644 --- a/cmd/server/memory.go +++ b/cmd/server/memory.go @@ -32,22 +32,10 @@ import ( // eviction; both can shrink when packets age out. goHeapInuseMB and goSysMB // fluctuate with GC. // -// There is no cgo-allocation field, but for a different reason than before: -// the driver is github.com/mattn/go-sqlite3 now, so SQLite's page cache IS a C -// allocation — the Go runtime simply exposes no byte counter for it. -// -// processRSSMB does include it, since VmRSS counts every resident page whoever -// allocated it. Resist the temptation to read `processRSSMB - goSysMB` as "the -// C share": goSysMB is address space the runtime has reserved, not what is -// resident, so the subtraction mixes two different things and can even go -// negative. It is a smell test, not a measurement. To size native usage, -// measure RSS against a build with the store disabled, or profile. -// -// What is bounded is SQLite's page cache specifically: both DSNs pin -// _cache_size=-2000, i.e. ~2 MiB per connection, so roughly 8 MiB across -// SetMaxOpenConns(4) here and 2 MiB in the ingestor. That is a cap on the page -// cache, not on everything SQLite allocates — statement and schema memory sit -// outside it. See memlimit.go for why any of it matters to GOMEMLIMIT. +// These are server-process measurements. PostgreSQL buffers and query work +// memory live in the database process and must be measured separately. +// processRSSMB - goSysMB is not an allocation counter: goSysMB includes address +// space reserved but not resident, so that subtraction can be negative. type MemorySnapshot struct { ProcessRSSMB float64 `json:"processRSSMB"` GoHeapInuseMB float64 `json:"goHeapInuseMB"` diff --git a/cmd/server/multibyte_capability_test.go b/cmd/server/multibyte_capability_test.go index 53267533f..11327d8f0 100644 --- a/cmd/server/multibyte_capability_test.go +++ b/cmd/server/multibyte_capability_test.go @@ -1,14 +1,11 @@ package main import ( - "database/sql" "encoding/json" "fmt" "strings" "testing" "time" - - _ "github.com/mattn/go-sqlite3" ) // recentTS returns a timestamp string N hours ago, ensuring test data @@ -20,23 +17,27 @@ func recentTS(hoursAgo int) string { // setupCapabilityTestDB creates a minimal in-memory DB with nodes table. func setupCapabilityTestDB(t *testing.T) *DB { t.Helper() - conn, err := sql.Open("sqlite3", ":memory:") + conn, err := openPostgresTestDB(t) if err != nil { t.Fatal(err) } conn.SetMaxOpenConns(1) conn.Exec(`CREATE TABLE nodes ( public_key TEXT PRIMARY KEY, name TEXT, role TEXT, - lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, - advert_count INTEGER DEFAULT 0, battery_mv INTEGER, temperature_c REAL + lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, first_seen TEXT, + advert_count INTEGER DEFAULT 0, battery_mv INTEGER, temperature_c DOUBLE PRECISION )`) - conn.Exec(`CREATE TABLE observers ( + conn.Exec(testNativeSQL(`CREATE TABLE observers (id TEXT PRIMARY KEY, name TEXT, iata TEXT, last_seen TEXT, + first_seen TEXT, packet_count INTEGER DEFAULT 0, model TEXT, + firmware TEXT, client_version TEXT, radio TEXT, battery_mv INTEGER, + uptime_secs INTEGER + )`, `CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY UNIQUE, id TEXT PRIMARY KEY, name TEXT, iata TEXT, last_seen TEXT, first_seen TEXT, packet_count INTEGER DEFAULT 0, model TEXT, firmware TEXT, client_version TEXT, radio TEXT, battery_mv INTEGER, uptime_secs INTEGER - )`) - return &DB{conn: conn} + )`)) + return &DB{backend: testBackendValue(), conn: conn} } // addTestPacket adds a StoreTx to the store's internal structures including @@ -86,7 +87,7 @@ func TestMultiByteCapability_Confirmed(t *testing.T) { db := setupCapabilityTestDB(t) defer db.conn.Close() - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ($1, $2, $3, $4)", "aabbccdd11223344", "RepA", "repeater", recentTS(24)) store := NewPacketStore(db, nil) @@ -113,7 +114,7 @@ func TestMultiByteCapability_Suspected(t *testing.T) { db := setupCapabilityTestDB(t) defer db.conn.Close() - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ($1, $2, $3, $4)", "aabbccdd11223344", "RepB", "repeater", recentTS(48)) store := NewPacketStore(db, nil) @@ -151,7 +152,7 @@ func TestMultiByteCapability_Unknown(t *testing.T) { db := setupCapabilityTestDB(t) defer db.conn.Close() - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ($1, $2, $3, $4)", "aabbccdd11223344", "RepC", "repeater", recentTS(72)) store := NewPacketStore(db, nil) @@ -179,9 +180,9 @@ func TestMultiByteCapability_PrefixCollision(t *testing.T) { defer db.conn.Close() // Two repeaters sharing 1-byte prefix "aa" - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ($1, $2, $3, $4)", "aabb000000000001", "RepConfirmed", "repeater", recentTS(24)) - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ($1, $2, $3, $4)", "aacc000000000002", "RepOther", "repeater", recentTS(24)) store := NewPacketStore(db, nil) @@ -227,7 +228,7 @@ func TestMultiByteCapability_TraceExcluded(t *testing.T) { db := setupCapabilityTestDB(t) defer db.conn.Close() - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ($1, $2, $3, $4)", "aabbccdd11223344", "RepTrace", "repeater", recentTS(48)) store := NewPacketStore(db, nil) @@ -259,7 +260,7 @@ func TestMultiByteCapability_NonTraceStillSuspected(t *testing.T) { db := setupCapabilityTestDB(t) defer db.conn.Close() - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ($1, $2, $3, $4)", "aabbccdd11223344", "RepNonTrace", "repeater", recentTS(48)) store := NewPacketStore(db, nil) @@ -291,7 +292,7 @@ func TestMultiByteCapability_ConfirmedUnaffectedByTraceExclusion(t *testing.T) { db := setupCapabilityTestDB(t) defer db.conn.Close() - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ($1, $2, $3, $4)", "aabbccdd11223344", "RepConfirmedTrace", "repeater", recentTS(24)) store := NewPacketStore(db, nil) @@ -326,7 +327,7 @@ func TestMultiByteCapability_CompanionConfirmed(t *testing.T) { db := setupCapabilityTestDB(t) defer db.conn.Close() - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ($1, $2, $3, $4)", "aabbccdd11223344", "CompA", "companion", recentTS(24)) store := NewPacketStore(db, nil) @@ -353,11 +354,11 @@ func TestMultiByteCapability_RoleColumnPopulated(t *testing.T) { db := setupCapabilityTestDB(t) defer db.conn.Close() - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ($1, $2, $3, $4)", "aabb000000000001", "Rep1", "repeater", recentTS(24)) - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ($1, $2, $3, $4)", "ccdd000000000002", "Comp1", "companion", recentTS(24)) - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ($1, $2, $3, $4)", "eeff000000000003", "Room1", "room_server", recentTS(24)) store := NewPacketStore(db, nil) @@ -392,7 +393,7 @@ func TestMultiByteCapability_AdopterEvidenceTakesPrecedence(t *testing.T) { db := setupCapabilityTestDB(t) defer db.conn.Close() - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ($1, $2, $3, $4)", "aabbccdd11223344", "RepAdopter", "repeater", recentTS(24)) store := NewPacketStore(db, nil) @@ -446,26 +447,26 @@ func TestMultiByteCapability_AdopterEvidenceTakesPrecedence(t *testing.T) { // setupPersistTestDB creates an in-memory DB with multibyte_sup/multibyte_evidence columns. func setupPersistTestDB(t *testing.T) *DB { t.Helper() - conn, err := sql.Open("sqlite3", ":memory:") + conn, err := openPostgresTestDB(t) if err != nil { t.Fatal(err) } conn.SetMaxOpenConns(1) conn.Exec(`CREATE TABLE nodes ( public_key TEXT PRIMARY KEY, name TEXT, role TEXT, - lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, - advert_count INTEGER DEFAULT 0, battery_mv INTEGER, temperature_c REAL, + lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, first_seen TEXT, + advert_count INTEGER DEFAULT 0, battery_mv INTEGER, temperature_c DOUBLE PRECISION, foreign_advert INTEGER DEFAULT 0, default_scope TEXT, multibyte_sup INTEGER NOT NULL DEFAULT 0, multibyte_evidence TEXT )`) conn.Exec(`CREATE TABLE inactive_nodes ( public_key TEXT PRIMARY KEY, name TEXT, role TEXT, - lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, - advert_count INTEGER DEFAULT 0, battery_mv INTEGER, temperature_c REAL, + lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, first_seen TEXT, + advert_count INTEGER DEFAULT 0, battery_mv INTEGER, temperature_c DOUBLE PRECISION, foreign_advert INTEGER DEFAULT 0, default_scope TEXT, multibyte_sup INTEGER NOT NULL DEFAULT 0, multibyte_evidence TEXT )`) - return &DB{conn: conn, hasMultibyteSupCols: true} + return &DB{backend: testBackendValue(), conn: conn, hasMultibyteSupCols: true} } // TestMultibyteCapGetMultibyteCapForO1 verifies that GetMultibyteCapFor returns diff --git a/cmd/server/multibyte_region_filter_test.go b/cmd/server/multibyte_region_filter_test.go index a66fedc13..6fe8d6719 100644 --- a/cmd/server/multibyte_region_filter_test.go +++ b/cmd/server/multibyte_region_filter_test.go @@ -30,9 +30,9 @@ func TestMultiByteCapability_RegionFiltered_PreservesConfirmedStatus(t *testing. // Two observers in different regions. db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obs-sjc', 'Obs SJC', 'SJC', ?, '2026-01-01T00:00:00Z', 100)`, recent) + VALUES ('obs-sjc', 'Obs SJC', 'SJC', $1, '2026-01-01T00:00:00Z', 100)`, recent) db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obs-jkg', 'Obs JKG', 'JKG', ?, '2026-01-01T00:00:00Z', 100)`, recent) + VALUES ('obs-jkg', 'Obs JKG', 'JKG', $1, '2026-01-01T00:00:00Z', 100)`, recent) // Node A: a JKG-region repeater that advertises multi-byte (hash_size=2). // Its zero-hop direct advert is only heard by obs-SJC (e.g. an out-of-region @@ -42,7 +42,7 @@ func TestMultiByteCapability_RegionFiltered_PreservesConfirmedStatus(t *testing. // to "confirmed" via the global advert evidence. pkA := "aaa0000000000001" db.conn.Exec(`INSERT INTO nodes (public_key, name, role) - VALUES (?, 'Node-A', 'repeater')`, pkA) + VALUES ($1, 'Node-A', 'repeater')`, pkA) decodedA := `{"pubKey":"` + pkA + `","name":"Node-A","type":"ADVERT","flags":{"isRepeater":true}}` @@ -50,17 +50,17 @@ func TestMultiByteCapability_RegionFiltered_PreservesConfirmedStatus(t *testing. // pathByte 0x40 → hash_size bits 01 → 2 bytes. // Heard by obs-SJC ONLY. db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('1240aabbccdd', 'a_zh_direct', ?, 2, 4, ?)`, recent, decodedA) + VALUES ('1240aabbccdd', 'a_zh_direct', $1, 2, 4, $2)`, recent, decodedA) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 12.0, -85, '[]', ?)`, recentEpoch) + VALUES (1, 1, 12.0, -85, '[]', $1)`, recentEpoch) // Node A also appears as a path hop in a JKG-observed packet, so it // shows up in the JKG region's node list. // route_type=1 (flood), payload_type=4, pathByte 0x41 (hs=2, hops=1) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('1141aabbccdd', 'a_jkg_relay', ?, 1, 4, ?)`, recent, decodedA) + VALUES ('1141aabbccdd', 'a_jkg_relay', $1, 1, 4, $2)`, recent, decodedA) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (2, 2, 8.0, -95, '["aa"]', ?)`, recentEpoch) + VALUES (2, 2, 8.0, -95, '["aa"]', $1)`, recentEpoch) store := NewPacketStore(db, nil) store.Load() diff --git a/cmd/server/neighbor_api_test.go b/cmd/server/neighbor_api_test.go index e3ab0b96c..78e1b5740 100644 --- a/cmd/server/neighbor_api_test.go +++ b/cmd/server/neighbor_api_test.go @@ -1,7 +1,6 @@ package main import ( - "database/sql" "encoding/json" "net/http" "net/http/httptest" @@ -9,7 +8,6 @@ import ( "time" "github.com/gorilla/mux" - _ "github.com/mattn/go-sqlite3" ) // ─── Helpers ─────────────────────────────────────────────────────────────────── @@ -464,10 +462,9 @@ func TestNeighborGraphAPI_ResponseShape(t *testing.T) { func TestBuildNodeInfoMap_ObserverEnrichment(t *testing.T) { // Create a temp SQLite DB with nodes and observers tables. - tmpDir := t.TempDir() - dbPath := tmpDir + "/test.db" + dbPath := postgresTestDSN(t) - conn, err := sql.Open("sqlite3", dbPath) + conn, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } @@ -475,8 +472,8 @@ func TestBuildNodeInfoMap_ObserverEnrichment(t *testing.T) { // Create tables for _, stmt := range []string{ - "CREATE TABLE nodes (public_key TEXT, name TEXT, role TEXT, lat REAL, lon REAL)", - "CREATE TABLE observers (id TEXT, name TEXT, iata TEXT, inactive INTEGER)", + "CREATE TABLE nodes (public_key TEXT, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION)", + testNativeSQL(`CREATE TABLE observers (id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, "CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY UNIQUE, id TEXT, name TEXT, iata TEXT, inactive INTEGER)"), "INSERT INTO nodes VALUES ('AAAA1111', 'Repeater-1', 'repeater', 0, 0)", "INSERT INTO observers (id, name, iata) VALUES ('BBBB2222', 'Observer-Alpha', '')", "INSERT INTO observers (id, name, iata) VALUES ('AAAA1111', 'Obs-also-repeater', '')", @@ -489,7 +486,7 @@ func TestBuildNodeInfoMap_ObserverEnrichment(t *testing.T) { conn.Close() // Open via our DB wrapper - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } @@ -539,18 +536,17 @@ func TestBuildNodeInfoMap_ObserverEnrichment(t *testing.T) { // existing 30s node cache, both calls return the original value — same as // every other nodeInfo field that comes from getAllNodes(). func TestBuildNodeInfoMap_FirstSeenIsCached(t *testing.T) { - tmpDir := t.TempDir() - dbPath := tmpDir + "/test.db" + dbPath := postgresTestDSN(t) // Seed via rw connection. - rw, err := sql.Open("sqlite3", dbPath) + rw, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } defer rw.Close() for _, stmt := range []string{ - "CREATE TABLE nodes (public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, advert_count INTEGER)", - "CREATE TABLE observers (id TEXT, name TEXT, iata TEXT, inactive INTEGER)", + "CREATE TABLE nodes (public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, first_seen TEXT, advert_count INTEGER)", + testNativeSQL(`CREATE TABLE observers (id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, "CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY UNIQUE, id TEXT, name TEXT, iata TEXT, inactive INTEGER)"), "INSERT INTO nodes VALUES ('AAAA1111', 'Repeater-1', 'repeater', 0, 0, '', '2024-01-01T00:00:00Z', 0)", } { if _, err := rw.Exec(stmt); err != nil { @@ -559,7 +555,7 @@ func TestBuildNodeInfoMap_FirstSeenIsCached(t *testing.T) { } ensurePreparable(t, rw) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } @@ -606,18 +602,17 @@ func TestBuildNodeInfoMap_FirstSeenIsCached(t *testing.T) { // values must have empty FirstSeen with no panic. Regression coverage for the // existing fallback branch (#1632 review loop 1). func TestGetAllNodes_FirstSeenSchemaFallback(t *testing.T) { - tmpDir := t.TempDir() - dbPath := tmpDir + "/test.db" + dbPath := postgresTestDSN(t) // Seed a nodes table WITHOUT first_seen (advert_count + last_seen present). - rw, err := sql.Open("sqlite3", dbPath) + rw, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } defer rw.Close() for _, stmt := range []string{ - "CREATE TABLE nodes (public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, advert_count INTEGER)", - "CREATE TABLE observers (id TEXT, name TEXT, iata TEXT, inactive INTEGER)", + "CREATE TABLE nodes (public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, advert_count INTEGER)", + testNativeSQL(`CREATE TABLE observers (id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, "CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY UNIQUE, id TEXT, name TEXT, iata TEXT, inactive INTEGER)"), "INSERT INTO nodes VALUES ('BBBB2222', 'Repeater-2', 'repeater', 0, 0, '2024-02-02T00:00:00Z', 3)", } { if _, err := rw.Exec(stmt); err != nil { @@ -626,7 +621,7 @@ func TestGetAllNodes_FirstSeenSchemaFallback(t *testing.T) { } ensurePreparable(t, rw) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } diff --git a/cmd/server/neighbor_persist_legacy_test.go b/cmd/server/neighbor_persist_legacy_test.go index 798b96e4c..051c7ba64 100644 --- a/cmd/server/neighbor_persist_legacy_test.go +++ b/cmd/server/neighbor_persist_legacy_test.go @@ -1,12 +1,8 @@ package main import ( - "database/sql" - "path/filepath" "testing" "time" - - _ "github.com/mattn/go-sqlite3" ) // TestNeighborPersist_LegacyEdgeInvariant (#1638 adv-#1): edges loaded from @@ -16,9 +12,8 @@ import ( // pre-existing observations land in bucket 0 (legacy/unknown, conservative // weight in the JS confidence indicator). func TestNeighborPersist_LegacyEdgeInvariant(t *testing.T) { - dir := t.TempDir() - dbPath := filepath.Join(dir, "neighbor_legacy.db") - rw, err := sql.Open("sqlite3", "file:"+dbPath+"?_journal_mode=WAL") + dbPath := postgresTestDSN(t) + rw, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } @@ -34,7 +29,7 @@ func TestNeighborPersist_LegacyEdgeInvariant(t *testing.T) { } now := time.Now().UTC().Format(time.RFC3339) if _, err := rw.Exec( - `INSERT INTO neighbor_edges (node_a, node_b, count, last_seen) VALUES (?, ?, ?, ?)`, + `INSERT INTO neighbor_edges (node_a, node_b, count, last_seen) VALUES ($1, $2, $3, $4)`, "aaaa", "bbbb", 7, now, ); err != nil { t.Fatal(err) @@ -77,9 +72,8 @@ func TestNeighborPersist_LegacyEdgeInvariant(t *testing.T) { // in the snapshot" path (loader's else-branch): subsequent counts must // accumulate into bucket 0 too, preserving the invariant. func TestNeighborPersist_LegacyEdgeMergeOnReload(t *testing.T) { - dir := t.TempDir() - dbPath := filepath.Join(dir, "neighbor_legacy_merge.db") - rw, err := sql.Open("sqlite3", "file:"+dbPath+"?_journal_mode=WAL") + dbPath := postgresTestDSN(t) + rw, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } @@ -97,7 +91,7 @@ func TestNeighborPersist_LegacyEdgeMergeOnReload(t *testing.T) { now := time.Now().UTC().Format(time.RFC3339) for _, cnt := range []int{3, 4} { if _, err := rw.Exec( - `INSERT INTO neighbor_edges (node_a, node_b, count, last_seen) VALUES (?, ?, ?, ?)`, + `INSERT INTO neighbor_edges (node_a, node_b, count, last_seen) VALUES ($1, $2, $3, $4)`, "aaaa", "bbbb", cnt, now, ); err != nil { t.Fatal(err) diff --git a/cmd/server/neighbor_recomputer_test.go b/cmd/server/neighbor_recomputer_test.go index cdcf7b8f6..80a986883 100644 --- a/cmd/server/neighbor_recomputer_test.go +++ b/cmd/server/neighbor_recomputer_test.go @@ -1,12 +1,9 @@ package main import ( - "database/sql" - "path/filepath" "testing" "time" - _ "github.com/mattn/go-sqlite3" "github.com/meshcore-analyzer/dbschema" ) @@ -15,11 +12,10 @@ import ( // snapshot the ingestor writes. After a write to neighbor_edges (here // done synthetically), the recomputer's atomic-swap must reflect it. func TestNeighborGraphRecomputerLoadsSnapshot(t *testing.T) { - dir := t.TempDir() - dbPath := filepath.Join(dir, "neighbor_recomp.db") + dbPath := postgresTestDSN(t) // Bootstrap a WAL DB with the neighbor_edges table. - rw, err := sql.Open("sqlite3", "file:"+dbPath+"?_journal_mode=WAL") + rw, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } @@ -38,14 +34,14 @@ func TestNeighborGraphRecomputerLoadsSnapshot(t *testing.T) { // Stage one edge. now := time.Now().UTC().Format(time.RFC3339) if _, err := rw.Exec( - `INSERT INTO neighbor_edges (node_a, node_b, count, last_seen) VALUES (?, ?, ?, ?)`, + `INSERT INTO neighbor_edges (node_a, node_b, count, last_seen) VALUES ($1, $2, $3, $4)`, "aaa", "bbb", 5, now, ); err != nil { t.Fatal(err) } // Server opens read-only and refreshes via the recomputer. - d, err := OpenDB(dbPath) + d, err := openFixtureReader(t, dbPath) if err != nil { t.Fatalf("OpenDB: %v", err) } @@ -64,7 +60,7 @@ func TestNeighborGraphRecomputerLoadsSnapshot(t *testing.T) { // Add another row, refresh, assert the new total. if _, err := rw.Exec( - `INSERT INTO neighbor_edges (node_a, node_b, count, last_seen) VALUES (?, ?, ?, ?)`, + `INSERT INTO neighbor_edges (node_a, node_b, count, last_seen) VALUES ($1, $2, $3, $4)`, "ccc", "ddd", 2, now, ); err != nil { t.Fatal(err) @@ -81,8 +77,7 @@ func TestNeighborGraphRecomputerLoadsSnapshot(t *testing.T) { // AssertReady on a DB missing the required columns returns an error // listing every missing surface; main.go then calls log.Fatalf. func TestServerStartupRequiresMigratedSchema(t *testing.T) { - dir := t.TempDir() - dbPath := filepath.Join(dir, "unmigrated.db") + dbPath := postgresTestDSN(t) // Bootstrap with ONLY transmissions/observations (the things // server tries to read) but WITHOUT the columns dbschema asserts @@ -94,15 +89,15 @@ func TestServerStartupRequiresMigratedSchema(t *testing.T) { // them selects on inactive, so a fixture without it cannot reach // AssertReady at all. The other missing surfaces above still make // AssertReady fail, which is what this test asserts. - rw, err := sql.Open("sqlite3", "file:"+dbPath+"?_journal_mode=WAL") + rw, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } defer rw.Close() for _, s := range []string{ - `CREATE TABLE transmissions (id INTEGER PRIMARY KEY, hash TEXT, payload_type INTEGER)`, - `CREATE TABLE observations (id INTEGER PRIMARY KEY, transmission_id INTEGER)`, - `CREATE TABLE observers (id TEXT PRIMARY KEY, name TEXT, inactive INTEGER)`, + testNativeSQL(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY AUTOINCREMENT, hash TEXT, payload_type INTEGER)`, `CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, hash TEXT, payload_type INTEGER)`), + testNativeSQL(`CREATE TABLE observations (id INTEGER PRIMARY KEY AUTOINCREMENT, transmission_id INTEGER)`, `CREATE TABLE observations (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER)`), + testNativeSQL(`CREATE TABLE observers (id TEXT PRIMARY KEY, name TEXT, inactive INTEGER)`, `CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY UNIQUE, id TEXT PRIMARY KEY, name TEXT, inactive INTEGER)`), `CREATE TABLE nodes (public_key TEXT PRIMARY KEY)`, `CREATE TABLE inactive_nodes (public_key TEXT PRIMARY KEY)`, } { @@ -115,7 +110,7 @@ func TestServerStartupRequiresMigratedSchema(t *testing.T) { // Open the read-only server handle and call AssertReady directly // (production path: main.go does this before any business logic). - d, err := OpenDB(dbPath) + d, err := openFixtureReader(t, dbPath) if err != nil { t.Fatalf("OpenDB: %v", err) } diff --git a/cmd/server/node_analytics_coverage_test.go b/cmd/server/node_analytics_coverage_test.go index 850edaba8..6f35f8769 100644 --- a/cmd/server/node_analytics_coverage_test.go +++ b/cmd/server/node_analytics_coverage_test.go @@ -15,7 +15,7 @@ func setupCoverageStore(t *testing.T, memHours int) *PacketStore { t.Helper() db := setupTestDB(t) t.Cleanup(func() { db.Close() }) - if _, err := db.conn.Exec("INSERT INTO nodes (public_key, name, role) VALUES (?, ?, ?)", coverageNodePK, "Coverage", "repeater"); err != nil { + if _, err := db.conn.Exec("INSERT INTO nodes (public_key, name, role) VALUES ($1, $2, $3)", coverageNodePK, "Coverage", "repeater"); err != nil { t.Fatal(err) } ps := NewPacketStore(db, nil) @@ -300,13 +300,12 @@ func TestNodeAnalyticsCoverageTooShortForRates(t *testing.T) { } } -// A load cut short by the memory cap leaves a gap between the last loaded -// row and what the poller picks up, so the store is only complete from the -// end of the load; packets[0] or the retention edge would overstate it. +// A newest-first load knows the conservative edge of its omitted row group. +// That edge is complete; replacing it with "now" would hide retained history. func TestLoadChunkedMemoryCapSetsCoverageEdge(t *testing.T) { store := openReverseTimeStore(t, 1100) defer store.db.conn.Close() - store.maxMemoryMB = 1 // floors to 1000 packets + store.maxMemoryMB = 1 store.retentionHours = 30 * 24 before := time.Now().UTC().Truncate(time.Second) if err := store.LoadChunked(500); err != nil { @@ -318,8 +317,8 @@ func TestLoadChunkedMemoryCapSetsCoverageEdge(t *testing.T) { store.mu.RLock() start, reason := store.coverageStart(time.Now().UTC()) store.mu.RUnlock() - if start.Before(before) || reason != "memory" { - t.Errorf("capped load: coverage start %v (%q), want at or after %v (\"memory\")", start, reason, before) + if start.Format(time.RFC3339) != store.oldestLoaded || !start.Before(before) || reason != "memory" { + t.Errorf("capped load: coverage start %v (%q), want known boundary %s before %v (\"memory\")", start, reason, store.oldestLoaded, before) } } diff --git a/cmd/server/node_battery.go b/cmd/server/node_battery.go index d768ba1d5..d4ba4df59 100644 --- a/cmd/server/node_battery.go +++ b/cmd/server/node_battery.go @@ -46,9 +46,9 @@ type NodeBatterySample struct { const nodeBatteryHistorySQL = ` SELECT timestamp, battery_mv FROM observer_metrics - WHERE observer_id IN (?, ?) + WHERE observer_id IN ($1, $2) AND battery_mv IS NOT NULL - AND timestamp >= ? + AND timestamp >= $3 ORDER BY timestamp ASC` // GetNodeBatteryHistory returns time-ordered battery_mv samples for a node, diff --git a/cmd/server/node_battery_test.go b/cmd/server/node_battery_test.go index 0dc5b6320..06c46590a 100644 --- a/cmd/server/node_battery_test.go +++ b/cmd/server/node_battery_test.go @@ -20,19 +20,19 @@ func TestGetNodeBatteryHistory_FromObserverMetrics(t *testing.T) { // node + observer with matching pubkey (cases differ on purpose) pkLower := "deadbeefcafef00d11223344" idUpper := strings.ToUpper(pkLower) - db.conn.Exec(`INSERT INTO nodes (public_key, name, role, last_seen, first_seen) VALUES (?, 'BatNode', 'repeater', ?, ?)`, + db.conn.Exec(`INSERT INTO nodes (public_key, name, role, last_seen, first_seen) VALUES ($1, 'BatNode', 'repeater', $2, $3)`, pkLower, now.Format(time.RFC3339), now.Add(-72*time.Hour).Format(time.RFC3339)) - db.conn.Exec(`INSERT INTO observers (id, name, last_seen, first_seen) VALUES (?, 'BatNode', ?, ?)`, + db.conn.Exec(`INSERT INTO observers (id, name, last_seen, first_seen) VALUES ($1, 'BatNode', $2, $3)`, idUpper, now.Format(time.RFC3339), now.Add(-72*time.Hour).Format(time.RFC3339)) // 3 metrics samples: 3700, 3500, 3200 mV for i, mv := range []int{3700, 3500, 3200} { ts := now.Add(time.Duration(-2+i) * time.Hour).Format(time.RFC3339) - db.conn.Exec(`INSERT INTO observer_metrics (observer_id, timestamp, battery_mv) VALUES (?, ?, ?)`, + db.conn.Exec(`INSERT INTO observer_metrics (observer_id, timestamp, battery_mv) VALUES ($1, $2, $3)`, idUpper, ts, mv) } // One sample with NULL battery should be skipped - db.conn.Exec(`INSERT INTO observer_metrics (observer_id, timestamp) VALUES (?, ?)`, + db.conn.Exec(`INSERT INTO observer_metrics (observer_id, timestamp) VALUES ($1, $2)`, idUpper, now.Add(-3*time.Hour).Format(time.RFC3339)) since := now.Add(-24 * time.Hour).Format(time.RFC3339) @@ -57,11 +57,11 @@ func TestNodeBatteryEndpoint(t *testing.T) { now := time.Now().UTC() pkLower := "aabbccdd11223344" idUpper := strings.ToUpper(pkLower) - db.conn.Exec(`INSERT INTO observers (id, name, last_seen, first_seen) VALUES (?, 'TestRepeater', ?, ?)`, + db.conn.Exec(`INSERT INTO observers (id, name, last_seen, first_seen) VALUES ($1, 'TestRepeater', $2, $3)`, idUpper, now.Format(time.RFC3339), now.Add(-72*time.Hour).Format(time.RFC3339)) for i, mv := range []int{3800, 3600, 3200} { ts := now.Add(time.Duration(-2+i) * time.Hour).Format(time.RFC3339) - db.conn.Exec(`INSERT INTO observer_metrics (observer_id, timestamp, battery_mv) VALUES (?, ?, ?)`, + db.conn.Exec(`INSERT INTO observer_metrics (observer_id, timestamp, battery_mv) VALUES ($1, $2, $3)`, idUpper, ts, mv) } @@ -165,6 +165,9 @@ func TestBatteryThresholds_ConfigOverride(t *testing.T) { // metrics in the window: 29 ms warm and 174 ms cold on 283k rows, against // 5.5 ms through the primary key. func TestNodeBatteryQueryUsesObserverIndex(t *testing.T) { + if testBackendValue() != "sqlite" { + t.Skip("SQLite query-plan assertion") + } db := setupTestDB(t) for _, s := range []string{ `ANALYZE`, @@ -203,8 +206,10 @@ func TestGetNodeBatteryHistory_LowercaseStoredID(t *testing.T) { db := setupTestDB(t) now := time.Now().UTC() pk := "deadbeefcafef00d11223344" - db.conn.Exec(`INSERT INTO observer_metrics (observer_id, timestamp, battery_mv) VALUES (?, ?, ?)`, - pk, now.Add(-time.Hour).Format(time.RFC3339), 3600) + if _, err := db.conn.Exec(`INSERT INTO observer_metrics (observer_id, timestamp, battery_mv) VALUES ($1, $2, $3)`, + pk, now.Add(-time.Hour).Format(time.RFC3339), 3600); err != nil { + t.Fatal(err) + } samples, err := db.GetNodeBatteryHistory(strings.ToUpper(pk), now.Add(-24*time.Hour).Format(time.RFC3339)) if err != nil { t.Fatal(err) diff --git a/cmd/server/node_blacklist_test.go b/cmd/server/node_blacklist_test.go index a6d877693..b22d7f0a8 100644 --- a/cmd/server/node_blacklist_test.go +++ b/cmd/server/node_blacklist_test.go @@ -71,8 +71,8 @@ func TestConfigBlacklistEmptyEntries(t *testing.T) { func TestBlacklistFiltersHandleNodes(t *testing.T) { db := setupTestDB(t) - db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role, last_seen) VALUES ('goodnode', 'GoodNode', 'companion', datetime('now'))") - db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role, last_seen) VALUES ('badnode', 'BadNode', 'companion', datetime('now'))") + db.conn.Exec(testNativeSQL(`INSERT INTO nodes (public_key, name, role, last_seen) VALUES ('goodnode', 'GoodNode', 'companion', strftime('%Y-%m-%dT%H:%M:%SZ','now')) ON CONFLICT DO NOTHING`, "INSERT INTO nodes (public_key, name, role, last_seen) VALUES ('goodnode', 'GoodNode', 'companion', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')) ON CONFLICT DO NOTHING")) + db.conn.Exec(testNativeSQL(`INSERT INTO nodes (public_key, name, role, last_seen) VALUES ('badnode', 'BadNode', 'companion', strftime('%Y-%m-%dT%H:%M:%SZ','now')) ON CONFLICT DO NOTHING`, "INSERT INTO nodes (public_key, name, role, last_seen) VALUES ('badnode', 'BadNode', 'companion', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')) ON CONFLICT DO NOTHING")) cfg := &Config{ NodeBlacklist: []string{"badnode"}, @@ -105,7 +105,7 @@ func TestBlacklistFiltersHandleNodes(t *testing.T) { func TestBlacklistFiltersNodeDetail(t *testing.T) { db := setupTestDB(t) - db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role, last_seen) VALUES ('badnode', 'BadNode', 'companion', datetime('now'))") + db.conn.Exec(testNativeSQL(`INSERT INTO nodes (public_key, name, role, last_seen) VALUES ('badnode', 'BadNode', 'companion', strftime('%Y-%m-%dT%H:%M:%SZ','now')) ON CONFLICT DO NOTHING`, "INSERT INTO nodes (public_key, name, role, last_seen) VALUES ('badnode', 'BadNode', 'companion', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')) ON CONFLICT DO NOTHING")) cfg := &Config{ NodeBlacklist: []string{"badnode"}, @@ -124,8 +124,8 @@ func TestBlacklistFiltersNodeDetail(t *testing.T) { func TestBlacklistFiltersNodeSearch(t *testing.T) { db := setupTestDB(t) - db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role, last_seen) VALUES ('badnode', 'TrollNode', 'companion', datetime('now'))") - db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role, last_seen) VALUES ('goodnode', 'GoodNode', 'companion', datetime('now'))") + db.conn.Exec(testNativeSQL(`INSERT INTO nodes (public_key, name, role, last_seen) VALUES ('badnode', 'TrollNode', 'companion', strftime('%Y-%m-%dT%H:%M:%SZ','now')) ON CONFLICT DO NOTHING`, "INSERT INTO nodes (public_key, name, role, last_seen) VALUES ('badnode', 'TrollNode', 'companion', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')) ON CONFLICT DO NOTHING")) + db.conn.Exec(testNativeSQL(`INSERT INTO nodes (public_key, name, role, last_seen) VALUES ('goodnode', 'GoodNode', 'companion', strftime('%Y-%m-%dT%H:%M:%SZ','now')) ON CONFLICT DO NOTHING`, "INSERT INTO nodes (public_key, name, role, last_seen) VALUES ('goodnode', 'GoodNode', 'companion', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')) ON CONFLICT DO NOTHING")) cfg := &Config{ NodeBlacklist: []string{"badnode"}, @@ -155,7 +155,7 @@ func TestBlacklistFiltersNodeSearch(t *testing.T) { func TestNoBlacklistPassesAll(t *testing.T) { db := setupTestDB(t) - db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role, last_seen) VALUES ('somenode', 'SomeNode', 'companion', datetime('now'))") + db.conn.Exec(testNativeSQL(`INSERT INTO nodes (public_key, name, role, last_seen) VALUES ('somenode', 'SomeNode', 'companion', strftime('%Y-%m-%dT%H:%M:%SZ','now')) ON CONFLICT DO NOTHING`, "INSERT INTO nodes (public_key, name, role, last_seen) VALUES ('somenode', 'SomeNode', 'companion', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')) ON CONFLICT DO NOTHING")) cfg := &Config{} srv := NewServer(db, cfg, NewHub()) @@ -234,7 +234,7 @@ func TestBlacklistFiltersNeighborGraph(t *testing.T) { func TestBlacklistFiltersResolveHops(t *testing.T) { db := setupTestDB(t) - db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role, last_seen) VALUES ('badnode', 'BadNode', 'companion', datetime('now'))") + db.conn.Exec(testNativeSQL(`INSERT INTO nodes (public_key, name, role, last_seen) VALUES ('badnode', 'BadNode', 'companion', strftime('%Y-%m-%dT%H:%M:%SZ','now')) ON CONFLICT DO NOTHING`, "INSERT INTO nodes (public_key, name, role, last_seen) VALUES ('badnode', 'BadNode', 'companion', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')) ON CONFLICT DO NOTHING")) cfg := &Config{ NodeBlacklist: []string{"badnode"}, diff --git a/cmd/server/node_declared_regions_api_test.go b/cmd/server/node_declared_regions_api_test.go index 480ee6b46..3c79ed471 100644 --- a/cmd/server/node_declared_regions_api_test.go +++ b/cmd/server/node_declared_regions_api_test.go @@ -163,7 +163,7 @@ func TestScopeAuditAndNodesAgreeOnDeclaredRegions(t *testing.T) { for pk, csv := range cases { if _, err := srv.db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count, configured_scope, configured_scope_at) - VALUES (?, ?, 'repeater', 51.0, 4.0, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z', 1, ?, '2026-01-01T00:00:00Z')`, + VALUES ($1, $2, 'repeater', 51.0, 4.0, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z', 1, $3, '2026-01-01T00:00:00Z')`, pk, "rp-"+pk, csv, ); err != nil { t.Fatal(err) diff --git a/cmd/server/node_health_can_relay_case_1290_test.go b/cmd/server/node_health_can_relay_case_1290_test.go index 59d0aa6fc..0a3159dd6 100644 --- a/cmd/server/node_health_can_relay_case_1290_test.go +++ b/cmd/server/node_health_can_relay_case_1290_test.go @@ -36,7 +36,7 @@ func TestNodeHealth_CanRelayCaseInsensitive_Issue1290(t *testing.T) { } if _, err := srv.store.db.conn.Exec( `INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count, can_relay, can_relay_seen) - VALUES (?, 'ListenerOnly', 'SJC', ?, '2026-01-01T00:00:00Z', 1, 0, 1)`, + VALUES ($1, 'ListenerOnly', 'SJC', $2, '2026-01-01T00:00:00Z', 1, 0, 1)`, obsIDLower, now); err != nil { t.Fatalf("seed observer: %v", err) } diff --git a/cmd/server/node_hop_analytics_test.go b/cmd/server/node_hop_analytics_test.go index 5c3dd2c1c..a0a41606f 100644 --- a/cmd/server/node_hop_analytics_test.go +++ b/cmd/server/node_hop_analytics_test.go @@ -287,16 +287,16 @@ func TestNodeHopPackets_UniquenessIsPerPrefixLength(t *testing.T) { func hopInsertNode(t *testing.T, db *DB, pk, name string, adverts int) { t.Helper() mustExec(t, db, `INSERT INTO nodes (public_key, name, role, last_seen, first_seen, advert_count) - VALUES (?, ?, 'repeater', ?, '2026-01-01', ?)`, pk, name, time.Now().UTC().Format(time.RFC3339), adverts) + VALUES ($1, $2, 'repeater', $3, '2026-01-01', $4)`, pk, name, time.Now().UTC().Format(time.RFC3339), adverts) } func hopInsertTx(t *testing.T, db *DB, id int, hash string, routeType, payloadType int, paths ...string) { t.Helper() ts := time.Now().Add(-1 * time.Hour) - mustExec(t, db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type) VALUES (?, 'AA', ?, ?, ?, ?)`, + mustExec(t, db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type) VALUES ($1, 'AA', $2, $3, $4, $5)`, id, hash, ts.UTC().Format(time.RFC3339), routeType, payloadType) for _, p := range paths { - mustExec(t, db, `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp, resolved_path) VALUES (?, NULL, ?, ?, NULL)`, + mustExec(t, db, `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp, resolved_path) VALUES ($1, NULL, $2, $3, NULL)`, id, p, ts.Unix()) } } @@ -352,7 +352,7 @@ func TestNodeHopAnalytics_SameResultAfterRestart(t *testing.T) { db := setupTestDB(t) mustExec(t, db, `CREATE TABLE neighbor_edges (node_a TEXT NOT NULL, node_b TEXT NOT NULL, count INTEGER DEFAULT 1, last_seen TEXT, PRIMARY KEY (node_a, node_b))`) - mustExec(t, db, `INSERT INTO neighbor_edges (node_a, node_b, count, last_seen) VALUES (?, ?, 50, ?)`, + mustExec(t, db, `INSERT INTO neighbor_edges (node_a, node_b, count, last_seen) VALUES ($1, $2, 50, $3)`, hopOther, hopTarget, time.Now().UTC().Format(time.RFC3339)) hopInsertNode(t, db, hopTarget, "Target", 1) hopInsertNode(t, db, hopCollider, "Collider", 500) @@ -403,14 +403,14 @@ func TestHandleNodeHopAnalytics(t *testing.T) { hopInsertNode(t, db, hopOther, "Other", 1) hopInsertNode(t, db, hopCollider, "Collider", 1) - mustExec(t, db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type) VALUES (1, 'AA', 'hop_recent_2', ?, 1, 5)`, recent) - mustExec(t, db, `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp) VALUES (1, NULL, '["77AA","77AA","AB12"]', ?)`, recentEpoch) - mustExec(t, db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type) VALUES (2, 'BB', 'hop_recent_1byte', ?, 0, 4)`, recent) - mustExec(t, db, `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp) VALUES (2, NULL, '["AB"]', ?)`, recentEpoch) - mustExec(t, db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type) VALUES (3, 'CC', 'hop_scoped_advert', ?, 0, 4)`, recent) - mustExec(t, db, `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp) VALUES (3, NULL, '["AB12"]', ?)`, recentEpoch) - mustExec(t, db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type) VALUES (4, 'DD', 'hop_old', ?, 1, 5)`, old) - mustExec(t, db, `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp) VALUES (4, NULL, '["AB12"]', ?)`, oldEpoch) + mustExec(t, db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type) VALUES (1, 'AA', 'hop_recent_2', $1, 1, 5)`, recent) + mustExec(t, db, `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp) VALUES (1, NULL, '["77AA","77AA","AB12"]', $1)`, recentEpoch) + mustExec(t, db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type) VALUES (2, 'BB', 'hop_recent_1byte', $1, 0, 4)`, recent) + mustExec(t, db, `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp) VALUES (2, NULL, '["AB"]', $1)`, recentEpoch) + mustExec(t, db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type) VALUES (3, 'CC', 'hop_scoped_advert', $1, 0, 4)`, recent) + mustExec(t, db, `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp) VALUES (3, NULL, '["AB12"]', $1)`, recentEpoch) + mustExec(t, db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type) VALUES (4, 'DD', 'hop_old', $1, 1, 5)`, old) + mustExec(t, db, `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp) VALUES (4, NULL, '["AB12"]', $1)`, oldEpoch) srv := NewServer(db, &Config{Port: 3000}, NewHub()) router := mux.NewRouter() diff --git a/cmd/server/node_reach.go b/cmd/server/node_reach.go index 0d4ab9257..b756676e8 100644 --- a/cmd/server/node_reach.go +++ b/cmd/server/node_reach.go @@ -5,6 +5,7 @@ import ( "database/sql" "encoding/json" "errors" + "fmt" "log" "net/http" "sort" @@ -15,6 +16,7 @@ import ( "time" "github.com/gorilla/mux" + "github.com/meshcore-analyzer/dbconfig" "github.com/meshcore-analyzer/dbschema" "golang.org/x/sync/singleflight" ) @@ -218,7 +220,7 @@ func (s *Server) reachNodeScopes(ctx context.Context, pubkey string) (string, st args[i] = &vals[i] } row := s.db.conn.QueryRowContext(ctx, - "SELECT "+strings.Join(cols, ", ")+" FROM nodes WHERE public_key = ?", pubkey) + "SELECT "+strings.Join(cols, ", ")+" FROM nodes WHERE public_key = "+s.db.parameter(1), pubkey) if err := row.Scan(args...); err != nil { return "", "", "" } @@ -336,7 +338,7 @@ type reachState struct { // buildSem bounds how many cold-cache reach scans run at the same time. // singleflight only collapses identical keys; distinct (pubkey, days) // keys each start a scan, and the endpoint is unauthenticated, so - // without a cap a handful of requests can hold every SQLite reader. + // without a cap a handful of requests can hold every database reader. buildSemOnce sync.Once buildSem chan struct{} // sf dedups concurrent cold-cache requests for the same key so N @@ -736,24 +738,21 @@ func (s *Server) getDegreeSnapshot(ctx context.Context) *degreeSnapshot { return snap } -// reachScanSQL is the reach scan for nTokens path tokens. Arguments: the -// window start (epoch seconds), one LIKE pattern per token, the row limit. -// -// CROSS JOIN fixes the join order in SQLite: observations is the outer loop, -// read through idx_observations_timestamp for the window. With a plain JOIN -// the planner scanned every transmission and probed observations per row -// (3.2-3.3 s against 1.8-2.0 s on an 18M-observation database, #2146). -func reachScanSQL(nTokens int) string { +// reachScanSQL builds the windowed reach scan. Arguments are the window +// start, one LIKE pattern per token, and the hard row limit. +// SQLite CROSS JOIN keeps the timestamp window as the outer loop (#2155). +// PostgreSQL retains its ordinary JOIN syntax and native planner. +func (db *DB) reachScanSQL(nTokens int) string { likes := make([]string, nTokens) for i := range likes { - likes[i] = "o.path_json LIKE ?" + likes[i] = fmt.Sprintf(db.nativeSQL("o.path_json LIKE %s", "o.path_json ILIKE %s ESCAPE ''"), db.parameter(i+2)) } return `SELECT LOWER(COALESCE(obs.id,'')), LOWER(COALESCE(t.from_pubkey,'')), COALESCE(t.payload_type,0), o.path_json, o.snr FROM observations o - CROSS JOIN transmissions t ON t.id = o.transmission_id + ` + db.nativeSQL("CROSS JOIN", "JOIN") + ` transmissions t ON t.id = o.transmission_id LEFT JOIN observers obs ON obs.rowid = o.observer_idx - WHERE o.timestamp >= ? AND (` + strings.Join(likes, " OR ") + `) - LIMIT ?` + WHERE o.timestamp >= ` + db.parameter(1) + ` AND (` + strings.Join(likes, " OR ") + `) + LIMIT ` + db.parameter(nTokens+2) } // scanReachRows reads windowed observations whose path contains any reliable @@ -797,7 +796,7 @@ func (s *Server) scanReachRows(ctx context.Context, tokens map[string]bool, sinc args = append(args, "%\""+tok+"\"%") } args = append(args, reachScanRowLimit) - return s.readReachRows(ctx, reachScanSQL(len(toks)), args) + return s.readReachRows(ctx, s.db.reachScanSQL(len(toks)), args) } // readReachRows runs a reach scan query (column order fixed by the callers) and @@ -845,7 +844,7 @@ func (s *Server) readReachRows(ctx context.Context, q string, args []interface{} } // reachMaxConcurrentBuilds is the number of cold-cache reach scans allowed at -// once. Two keeps half of the default 4-connection SQLite pool free for +// once. Two keeps half of the default 4-connection PostgreSQL pool free for // every other handler while a scan runs. const reachMaxConcurrentBuilds = 2 @@ -873,6 +872,11 @@ func (s *Server) reachAcquireBuildSlot() (func(), bool) { // internal/dbschema/observation_hops.go): a missing table or coverage row // (index off, older DB, ingestor not yet started) just means "no". func (s *Server) reachHopIndexCovers(ctx context.Context, sinceEpoch int64) bool { + // observation_hops is an optional SQLite-derived accelerator. PostgreSQL + // uses the native bounded scan, even if a similarly named table exists. + if s.db.Backend() != dbconfig.SQLite { + return false + } var covered int64 err := s.db.conn.QueryRowContext(ctx, `SELECT since FROM `+dbschema.ObservationHopsCoverageTable+` WHERE id = 1`).Scan(&covered) diff --git a/cmd/server/node_reach_bench_test.go b/cmd/server/node_reach_bench_test.go index bbf24a347..12db04bbb 100644 --- a/cmd/server/node_reach_bench_test.go +++ b/cmd/server/node_reach_bench_test.go @@ -2,11 +2,8 @@ package main import ( "context" - "database/sql" "fmt" "testing" - - _ "github.com/mattn/go-sqlite3" ) // benchReachDB builds an in-memory DB with nObs observations. matchEvery @@ -24,14 +21,14 @@ func benchReachDB(b *testing.B, nObs, matchEvery int, lowerHops bool) *DB { // path_json is uppercase; this measures the worst case Carmack flagged). matchPath, fillerPath = `["aa","01fa","bb"]`, `["aa","cc","bb"]` } - conn, err := sql.Open("sqlite3", ":memory:") + conn, err := openPostgresTestDB(b) if err != nil { b.Fatal(err) } schema := []string{ - `CREATE TABLE transmissions (id INTEGER PRIMARY KEY, hash TEXT, first_seen TEXT, payload_type INTEGER, from_pubkey TEXT)`, - `CREATE TABLE observers (id TEXT PRIMARY KEY, name TEXT, inactive INTEGER)`, - `CREATE TABLE observations (id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_idx INTEGER, snr REAL, path_json TEXT, timestamp INTEGER)`, + testNativeSQL(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY AUTOINCREMENT, hash TEXT, first_seen TEXT, payload_type INTEGER, from_pubkey TEXT)`, `CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, hash TEXT, first_seen TEXT, payload_type INTEGER, from_pubkey TEXT)`), + testNativeSQL(`CREATE TABLE observers (id TEXT PRIMARY KEY, name TEXT, inactive INTEGER)`, `CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY UNIQUE, id TEXT PRIMARY KEY, name TEXT, inactive INTEGER)`), + testNativeSQL(`CREATE TABLE observations (id INTEGER PRIMARY KEY AUTOINCREMENT, transmission_id INTEGER, observer_idx INTEGER, snr DOUBLE PRECISION, path_json TEXT, timestamp INTEGER)`, `CREATE TABLE observations (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_idx INTEGER, snr DOUBLE PRECISION, path_json TEXT, timestamp INTEGER)`), `CREATE INDEX idx_obs_ts ON observations(timestamp)`, } for _, s := range schema { @@ -47,7 +44,7 @@ func benchReachDB(b *testing.B, nObs, matchEvery int, lowerHops bool) *DB { b.Fatal(err) } for i := 0; i < nObs; i++ { - if _, err := tx.Exec(`INSERT INTO transmissions (id, hash, first_seen, payload_type, from_pubkey) VALUES (?,?,?,5,'')`, + if _, err := tx.Exec(`INSERT INTO transmissions (id, hash, first_seen, payload_type, from_pubkey) VALUES ($1,$2,$3,5,'')`, i, fmt.Sprintf("h%d", i), "2026-06-07T00:00:00Z"); err != nil { b.Fatal(err) } @@ -55,7 +52,7 @@ func benchReachDB(b *testing.B, nObs, matchEvery int, lowerHops bool) *DB { if i%matchEvery == 0 { path = matchPath } - if _, err := tx.Exec(`INSERT INTO observations (id, transmission_id, observer_idx, snr, path_json, timestamp) VALUES (?,?,1,-7.0,?,?)`, + if _, err := tx.Exec(`INSERT INTO observations (id, transmission_id, observer_idx, snr, path_json, timestamp) VALUES ($1,$2,1,-7.0,$3,$4)`, i, i, path, 1000); err != nil { b.Fatal(err) } @@ -63,7 +60,7 @@ func benchReachDB(b *testing.B, nObs, matchEvery int, lowerHops bool) *DB { if err := tx.Commit(); err != nil { b.Fatal(err) } - return &DB{conn: conn} + return &DB{backend: testBackendValue(), conn: conn} } // BenchmarkNodeReachScan measures the windowed scan + path-decode at increasing @@ -155,7 +152,7 @@ func BenchmarkNodeReachAttribute(b *testing.B) { // must surface an error, not a swallowed nil. Lives in this file because // the bench callers in the same file rely on the same signature. func TestScanReachRows_ErrorReturn(t *testing.T) { - conn, err := sql.Open("sqlite3", ":memory:") + conn, err := openPostgresTestDB(t) if err != nil { t.Fatalf("open: %v", err) } @@ -164,7 +161,7 @@ func TestScanReachRows_ErrorReturn(t *testing.T) { t.Fatalf("schema: %v", err) } conn.Close() // force QueryContext to fail - srv := &Server{db: &DB{conn: conn}} + srv := &Server{db: &DB{backend: testBackendValue(), conn: conn}} rows, err := srv.scanReachRows(context.Background(), map[string]bool{"01FA": true}, 0) if err == nil { t.Fatalf("expected error from closed DB, got nil (rows=%d)", len(rows)) diff --git a/cmd/server/node_reach_endpoint_test.go b/cmd/server/node_reach_endpoint_test.go index a7fc6d4aa..14aa8eb87 100644 --- a/cmd/server/node_reach_endpoint_test.go +++ b/cmd/server/node_reach_endpoint_test.go @@ -1,7 +1,6 @@ package main import ( - "database/sql" "encoding/json" "net/http" "net/http/httptest" @@ -11,7 +10,6 @@ import ( "time" "github.com/gorilla/mux" - _ "github.com/mattn/go-sqlite3" ) func serveReach(srv *Server, path string) *httptest.ResponseRecorder { @@ -57,7 +55,7 @@ func resetReachState(t *testing.T, servers ...*Server) { // build the zero-reach case (identifiable node, no matching observations). func newReachIntegrationDB(t *testing.T, obsPath string) (*DB, string) { t.Helper() - conn, err := sql.Open("sqlite3", ":memory:") + conn, err := openPostgresTestDB(t) if err != nil { t.Fatal(err) } @@ -66,10 +64,10 @@ func newReachIntegrationDB(t *testing.T, obsPath string) (*DB, string) { b := pk64("ccdd") // successor → B hears us now := time.Now().Unix() stmts := []string{ - `CREATE TABLE nodes (public_key TEXT, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, advert_count INTEGER)`, - `CREATE TABLE transmissions (id INTEGER PRIMARY KEY, from_pubkey TEXT, payload_type INTEGER)`, - `CREATE TABLE observers (id TEXT, inactive INTEGER)`, - `CREATE TABLE observations (id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_idx INTEGER, snr REAL, path_json TEXT, timestamp INTEGER)`, + `CREATE TABLE nodes (public_key TEXT, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, last_seen TEXT, first_seen TEXT, advert_count INTEGER)`, + testNativeSQL(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY AUTOINCREMENT, from_pubkey TEXT, payload_type INTEGER)`, `CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, from_pubkey TEXT, payload_type INTEGER)`), + testNativeSQL(`CREATE TABLE observers (id TEXT, inactive INTEGER)`, `CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY UNIQUE, id TEXT, inactive INTEGER)`), + testNativeSQL(`CREATE TABLE observations (id INTEGER PRIMARY KEY AUTOINCREMENT, transmission_id INTEGER, observer_idx INTEGER, snr DOUBLE PRECISION, path_json TEXT, timestamp INTEGER)`, `CREATE TABLE observations (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_idx INTEGER, snr DOUBLE PRECISION, path_json TEXT, timestamp INTEGER)`), `CREATE TABLE neighbor_edges (node_a TEXT, node_b TEXT, count INTEGER)`, } for _, s := range stmts { @@ -81,19 +79,19 @@ func newReachIntegrationDB(t *testing.T, obsPath string) (*DB, string) { q string args []interface{} }{ - {`INSERT INTO nodes VALUES (?, 'N', 'repeater', 50.9, 5.4, ?, '2026-06-01T00:00:00Z', 3)`, []interface{}{n, "2026-06-07T00:00:00Z"}}, - {`INSERT INTO nodes VALUES (?, 'A', 'repeater', 51.0, 5.5, ?, '2026-06-01T00:00:00Z', 1)`, []interface{}{a, "2026-06-07T00:00:00Z"}}, - {`INSERT INTO nodes VALUES (?, 'B', 'repeater', 51.1, 5.6, ?, '2026-06-01T00:00:00Z', 1)`, []interface{}{b, "2026-06-07T00:00:00Z"}}, + {`INSERT INTO nodes VALUES ($1, 'N', 'repeater', 50.9, 5.4, $2, '2026-06-01T00:00:00Z', 3)`, []interface{}{n, "2026-06-07T00:00:00Z"}}, + {`INSERT INTO nodes VALUES ($1, 'A', 'repeater', 51.0, 5.5, $2, '2026-06-01T00:00:00Z', 1)`, []interface{}{a, "2026-06-07T00:00:00Z"}}, + {`INSERT INTO nodes VALUES ($1, 'B', 'repeater', 51.1, 5.6, $2, '2026-06-01T00:00:00Z', 1)`, []interface{}{b, "2026-06-07T00:00:00Z"}}, {`INSERT INTO observers (id) VALUES ('OBS1')`, nil}, {`INSERT INTO transmissions (id, from_pubkey, payload_type) VALUES (1, '', 5)`, nil}, - {`INSERT INTO observations (id, transmission_id, observer_idx, snr, path_json, timestamp) VALUES (1,1,1,-7.0,?,?)`, []interface{}{obsPath, now}}, + {`INSERT INTO observations (id, transmission_id, observer_idx, snr, path_json, timestamp) VALUES (1,1,1,-7.0,$1,$2)`, []interface{}{obsPath, now}}, } for _, in := range ins { if _, err := conn.Exec(in.q, in.args...); err != nil { t.Fatal(err) } } - return &DB{conn: conn, isV3: true}, n + return &DB{backend: testBackendValue(), conn: conn, isV3: true}, n } func TestClampDays(t *testing.T) { @@ -232,14 +230,21 @@ func TestNodeReach_ShapeAndClamp(t *testing.T) { // shape (empty — no rows needed for shape/clamp assertions). mustExecDB(t, db, `DROP TABLE observations`) // PREFLIGHT: async=true reason="test-only in-memory schema rebuild; not a production migration" - mustExecDB(t, db, `CREATE TABLE observations ( + mustExecDB(t, db, testNativeSQL(`CREATE TABLE observations ( id INTEGER PRIMARY KEY AUTOINCREMENT, transmission_id INTEGER, observer_idx INTEGER, - snr REAL, + snr DOUBLE PRECISION, path_json TEXT, timestamp INTEGER - )`) + )`, `CREATE TABLE observations ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + transmission_id INTEGER, + observer_idx INTEGER, + snr DOUBLE PRECISION, + path_json TEXT, + timestamp INTEGER + )`)) cfg := &Config{} srv := &Server{store: newTestStoreWithDB(t, db, cfg), db: db, cfg: cfg, perfStats: NewPerfStats()} diff --git a/cmd/server/node_reach_hops_test.go b/cmd/server/node_reach_hops_test.go index dff86a2ee..f22d1bb5c 100644 --- a/cmd/server/node_reach_hops_test.go +++ b/cmd/server/node_reach_hops_test.go @@ -7,6 +7,7 @@ import ( "sort" "testing" + "github.com/meshcore-analyzer/dbconfig" "github.com/meshcore-analyzer/dbschema" ) @@ -22,6 +23,9 @@ func canon(rows []pathRow) []string { func newHopIndexedTestDB(t *testing.T) *DB { t.Helper() + if testBackend(t) != dbconfig.SQLite { + t.Skip("the derived observation hop accelerator is SQLite-only") + } db := newReachScanTestDB(t) db.conn.SetMaxOpenConns(1) // Extra rows: mixed hop widths, a path hitting two tokens, one row outside @@ -43,6 +47,32 @@ func newHopIndexedTestDB(t *testing.T) *DB { return db } +// A similarly named table does not make the SQLite-only accelerator usable on +// PostgreSQL. The native scan must still return complete results. +func TestScanReachRows_PostgresIgnoresSQLiteHopCoverage(t *testing.T) { + if testBackend(t) != dbconfig.Postgres { + t.Skip("PostgreSQL native fallback") + } + db := newReachScanTestDB(t) + defer db.conn.Close() + for _, query := range []string{ + `CREATE TABLE observation_hops_coverage (id INTEGER PRIMARY KEY, since BIGINT NOT NULL)`, + `INSERT INTO observation_hops_coverage VALUES (1, 0)`, + } { + if _, err := db.conn.Exec(query); err != nil { + t.Fatal(err) + } + } + s := &Server{db: db} + if s.reachHopIndexCovers(context.Background(), 500) { + t.Fatal("PostgreSQL must not claim coverage from SQLite's derived hop index") + } + rows, err := s.scanReachRows(context.Background(), map[string]bool{"01fa": true}, 500) + if err != nil || len(rows) != 2 { + t.Fatalf("native fallback rows=%d, err=%v; want both matching observations", len(rows), err) + } +} + // The indexed scan must return exactly what the LIKE scan returns, for upper- // and lower-case tokens alike. func TestScanReachRows_IndexedMatchesLike(t *testing.T) { diff --git a/cmd/server/node_reach_test.go b/cmd/server/node_reach_test.go index fef55896c..4ea0454fd 100644 --- a/cmd/server/node_reach_test.go +++ b/cmd/server/node_reach_test.go @@ -2,26 +2,25 @@ package main import ( "context" - "database/sql" "strconv" "strings" "testing" - _ "github.com/mattn/go-sqlite3" + "github.com/meshcore-analyzer/dbconfig" ) // newReachScanTestDB builds a minimal observer_idx-schema DB with two rows whose // path contains "01FA" and one that does not, for scanReachRows coverage. func newReachScanTestDB(t *testing.T) *DB { t.Helper() - conn, err := sql.Open("sqlite3", ":memory:") + conn, err := openPostgresTestDB(t) if err != nil { t.Fatal(err) } stmts := []string{ - `CREATE TABLE transmissions (id INTEGER PRIMARY KEY, from_pubkey TEXT, payload_type INTEGER)`, - `CREATE TABLE observers (id TEXT, inactive INTEGER)`, - `CREATE TABLE observations (id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_idx INTEGER, snr REAL, path_json TEXT, timestamp INTEGER)`, + testNativeSQL(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY AUTOINCREMENT, from_pubkey TEXT, payload_type INTEGER)`, `CREATE TABLE transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, from_pubkey TEXT, payload_type INTEGER)`), + testNativeSQL(`CREATE TABLE observers (id TEXT, inactive INTEGER)`, `CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY UNIQUE, id TEXT, inactive INTEGER)`), + testNativeSQL(`CREATE TABLE observations (id INTEGER PRIMARY KEY AUTOINCREMENT, transmission_id INTEGER, observer_idx INTEGER, snr DOUBLE PRECISION, path_json TEXT, timestamp INTEGER)`, `CREATE TABLE observations (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_idx INTEGER, snr DOUBLE PRECISION, path_json TEXT, timestamp INTEGER)`), `INSERT INTO observers (id) VALUES ('OBS1')`, // rowid 1 `INSERT INTO transmissions (id, from_pubkey, payload_type) VALUES (1,'FF00',4),(2,'',5),(3,'',5)`, `INSERT INTO observations (id, transmission_id, observer_idx, snr, path_json, timestamp) VALUES @@ -34,7 +33,7 @@ func newReachScanTestDB(t *testing.T) *DB { t.Fatal(err) } } - return &DB{conn: conn} + return &DB{backend: testBackendValue(), conn: conn} } // resolver that only resolves the exact tokens it's told are unique. @@ -291,16 +290,15 @@ func TestScanReachRows_DecodesRows(t *testing.T) { } } -// #2146: the reach scan must be driven from the observations time window. -// With the default join order SQLite picked SCAN transmissions (every row) -// and probed observations per transmission: 3.2-3.3 s on an 18M-observation -// database against 1.8-2.0 s when it starts from idx_observations_timestamp. +// Upstream #2155 / #2146: plan using the ANALYZE statistics captured from an +// 18.3M-observation, 1.5M-transmission database, without building that corpus. func TestReachScanStartsFromObservationsTimeIndex(t *testing.T) { + if testBackend(t) != dbconfig.SQLite { + t.Skip("SQLite planner regression; PostgreSQL uses its native join planner") + } db := newReachScanTestDB(t) - // The planner's choice depends on sqlite_stat1. These are the statistics - // ANALYZE produced on that database (18.3M observations, 1.5M - // transmissions), loaded so the test plans like production does. - for _, s := range []string{ + db.conn.SetMaxOpenConns(1) + for _, query := range []string{ `CREATE INDEX idx_observations_timestamp ON observations(timestamp)`, `CREATE INDEX idx_observations_tx_ts ON observations(transmission_id, timestamp)`, `ANALYZE`, @@ -313,29 +311,49 @@ func TestReachScanStartsFromObservationsTimeIndex(t *testing.T) { ('observers', NULL, '88')`, `ANALYZE sqlite_master`, } { - if _, err := db.conn.Exec(s); err != nil { + if _, err := db.conn.Exec(query); err != nil { t.Fatal(err) } } - args := []interface{}{int64(0), `%"01FA"%`, `%"01FA00"%`, 10} - rows, err := db.conn.Query("EXPLAIN QUERY PLAN "+reachScanSQL(2), args...) + rows, err := db.conn.Query("EXPLAIN QUERY PLAN "+db.reachScanSQL(2), int64(0), `%"01FA"%`, `%"01FA00"%`, 10) if err != nil { t.Fatal(err) } defer rows.Close() var steps []string for rows.Next() { - var id, parent, notused int + var id, parent, unused int var detail string - if err := rows.Scan(&id, &parent, ¬used, &detail); err != nil { + if err := rows.Scan(&id, &parent, &unused, &detail); err != nil { t.Fatal(err) } steps = append(steps, detail) } - if len(steps) == 0 { - t.Fatal("no query plan") + if err := rows.Err(); err != nil { + t.Fatal(err) } - if !strings.Contains(steps[0], "o USING INDEX idx_observations_timestamp") { + if len(steps) == 0 || !strings.Contains(steps[0], "o USING INDEX idx_observations_timestamp") { t.Fatalf("reach scan must start from observations by timestamp; plan: %q", steps) } + t.Logf("reach plan: %q", steps) +} + +func TestScanReachRows_WindowAndMultipleTokens(t *testing.T) { + db := newReachScanTestDB(t) + for _, query := range []string{ + `INSERT INTO transmissions (id, from_pubkey, payload_type) VALUES (4,NULL,5),(5,'',5)`, + `INSERT INTO observations (id, transmission_id, observer_idx, snr, path_json, timestamp) VALUES + (4,4,NULL,NULL,'["cafe"]',2000), (5,5,1,2,'["CAFE"]',999)`, + } { + if _, err := db.conn.Exec(query); err != nil { + t.Fatal(err) + } + } + rows, err := (&Server{db: db}).scanReachRows(context.Background(), map[string]bool{"01FA": true, "CAFE": true}, 1500) + if err != nil { + t.Fatal(err) + } + if len(rows) != 1 || len(rows[0].path) != 1 || rows[0].path[0] != "CAFE" || rows[0].observerPK != "" || rows[0].fromPubkey != "" || rows[0].snrValid { + t.Fatalf("window, token binding or nullable metadata changed: %+v", rows) + } } diff --git a/cmd/server/node_resolve.go b/cmd/server/node_resolve.go index bf5bd5ff6..2dcf43600 100644 --- a/cmd/server/node_resolve.go +++ b/cmd/server/node_resolve.go @@ -41,7 +41,7 @@ func (s *Server) handleResolvePrefix(w http.ResponseWriter, r *http.Request) { } // LIMIT 2: we only need to know unique vs ambiguous. nodes.public_key is the // PK and stored lowercase; pfx is validated hex so the LIKE pattern is safe. - rows, err := s.db.conn.Query(`SELECT public_key, COALESCE(name,'') FROM nodes WHERE public_key LIKE ? LIMIT 2`, pfx+"%") + rows, err := s.db.conn.Query(`SELECT public_key, COALESCE(name,'') FROM nodes WHERE public_key LIKE `+s.db.parameter(1)+` LIMIT 2`, pfx+"%") if err != nil { http.Error(w, "query failed", http.StatusInternalServerError) return diff --git a/cmd/server/nodes_pagination_has_more_test.go b/cmd/server/nodes_pagination_has_more_test.go index de1eada42..9bbb13ce6 100644 --- a/cmd/server/nodes_pagination_has_more_test.go +++ b/cmd/server/nodes_pagination_has_more_test.go @@ -40,7 +40,7 @@ func TestNodesHasMoreSurvivesPostLimitFiltering(t *testing.T) { lastSeen := fmt.Sprintf("2026-06-0%dT00:00:00Z", 7-i) if _, err := srv.db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, ?, 'repeater', 0, 0, ?, '2026-06-01T00:00:00Z', 1)`, + VALUES ($1, $2, 'repeater', 0, 0, $3, '2026-06-01T00:00:00Z', 1)`, fmt.Sprintf("deadbeef0000200%d", i), name, lastSeen); err != nil { t.Fatalf("insert %d: %v", i, err) } diff --git a/cmd/server/nodes_perf_limit2000_test.go b/cmd/server/nodes_perf_limit2000_test.go index e03a7ad38..7ac64d86b 100644 --- a/cmd/server/nodes_perf_limit2000_test.go +++ b/cmd/server/nodes_perf_limit2000_test.go @@ -41,7 +41,7 @@ func TestHandleNodesLimit2000ColdMiss(t *testing.T) { } stmt, err := tx.Prepare(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count, foreign_advert) - VALUES (?, ?, ?, 0, 0, ?, '2026-01-01T00:00:00Z', 1, 0)`) + VALUES ($1, $2, $3, 0, 0, $4, '2026-01-01T00:00:00Z', 1, 0)`) if err != nil { t.Fatal(err) } diff --git a/cmd/server/nodes_perf_test.go b/cmd/server/nodes_perf_test.go index c7e1c5722..b1ac1acb4 100644 --- a/cmd/server/nodes_perf_test.go +++ b/cmd/server/nodes_perf_test.go @@ -43,7 +43,7 @@ func TestHandleNodesPerfLargeFleet(t *testing.T) { } stmt, err := tx.Prepare(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count, foreign_advert) - VALUES (?, ?, ?, 0, 0, ?, '2026-01-01T00:00:00Z', 1, 0)`) + VALUES ($1, $2, $3, 0, 0, $4, '2026-01-01T00:00:00Z', 1, 0)`) if err != nil { t.Fatal(err) } diff --git a/cmd/server/nodes_response_cache_test.go b/cmd/server/nodes_response_cache_test.go index 4fffde0db..abd45e886 100644 --- a/cmd/server/nodes_response_cache_test.go +++ b/cmd/server/nodes_response_cache_test.go @@ -133,7 +133,7 @@ func nodesBenchServer(b *testing.B) http.Handler { } seen := now.Add(-time.Duration(i%720) * time.Hour).Format(time.RFC3339) if _, err := db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`, fmt.Sprintf("%064x", i+1000), fmt.Sprintf("Bench node %d", i), role, 55+float64(i%100)/50, -4+float64(i%80)/40, seen, seen, i%50); err != nil { b.Fatal(err) diff --git a/cmd/server/notify_service_test.go b/cmd/server/notify_service_test.go index 884876c88..9360861c8 100644 --- a/cmd/server/notify_service_test.go +++ b/cmd/server/notify_service_test.go @@ -3,7 +3,6 @@ package main import ( "context" "errors" - "path/filepath" "regexp" "strings" "testing" @@ -98,7 +97,7 @@ type notifyFixture struct { // admin@example.org is a config admin. func newNotifyFixture(t *testing.T, ns notifySettings) *notifyFixture { t.Helper() - a, fake := newTestAuthService(t, "admin@example.org") + a, fake, ownerURL := newTestAuthServiceWithURL(t, postgresTestDSN(t), "admin@example.org") a.set.notify = ns clk := ¬ifyClock{t: time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC)} a.st.SetClock(clk.Now) @@ -110,7 +109,7 @@ func newNotifyFixture(t *testing.T, ns notifySettings) *notifyFixture { srv := &Server{cfg: &Config{APIKey: testAPIKey}, perfStats: NewPerfStats(), auth: a} r := mux.NewRouter() srv.registerAuthRoutes(r) - return ¬ifyFixture{authFixture: &authFixture{srv: srv, router: r, fake: fake, st: a.st}, n: a.notify, src: src, clk: clk} + return ¬ifyFixture{authFixture: &authFixture{srv: srv, router: r, fake: fake, st: a.st, ownerURL: ownerURL}, n: a.notify, src: src, clk: clk} } // watcher registers and activates a user with default prefs watching pubkeys. @@ -319,13 +318,13 @@ func TestNotifierStateWriteFailureSendsNothing(t *testing.T) { f.watcher(t, "pat@example.org", "Pat", evPkA) f.setNode(evPkA, "Alpha", "companion", time.Hour, nil) f.tick() - f.execDB(t, `CREATE TRIGGER notify_state_fail BEFORE UPDATE ON notification_state BEGIN SELECT RAISE(ABORT, 'boom'); END`) + f.execDB(t, testNativeSQL(`CREATE TRIGGER notify_state_fail BEFORE UPDATE ON notification_state BEGIN SELECT RAISE(ABORT,'boom'); END`, `CREATE FUNCTION notify_state_fail() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'boom'; END $$; CREATE TRIGGER notify_state_fail BEFORE UPDATE ON notification_state FOR EACH ROW EXECUTE FUNCTION notify_state_fail()`)) f.clk.Advance(25 * time.Hour) f.tick() if m := f.notifyMails(); len(m) != 0 { t.Fatalf("mailed although the states were not written: %+v", m) } - f.execDB(t, `DROP TRIGGER notify_state_fail`) + f.execDB(t, testNativeSQL(`DROP TRIGGER notify_state_fail`, `DROP TRIGGER notify_state_fail ON notification_state`)) f.tick() if m := f.notifyMails(); len(m) != 1 { t.Fatalf("mails after the write works again = %d; want 1", len(m)) @@ -506,13 +505,16 @@ func TestNotifySkipReason(t *testing.T) { func TestInitUserManagementStartsTheNotifierOnlyWhenEnabled(t *testing.T) { for _, on := range []bool{false, true} { - um := &UserManagementConfig{Enabled: true, PublicBaseURL: testBase, - Mail: UserMailConfig{BrevoAPIKey: "k", FromEmail: "noreply@example.org"}} + a, _ := newTestAuthService(t) + um := validUM() + um.DatabaseURL = a.set.databaseURL + backupOff := false + um.Backup = &UsersBackupConfig{Enabled: &backupOff} if on { um.Notifications = &NotificationsConfig{Enabled: true} } srv := &Server{cfg: &Config{UserManagement: um}} - if err := srv.initUserManagement(filepath.Join(t.TempDir(), "meshcore.db")); err != nil { + if err := srv.initUserManagement(testDatabaseDSN(t)); err != nil { t.Fatal(err) } if (srv.auth.notify != nil) != on { diff --git a/cmd/server/notify_source.go b/cmd/server/notify_source.go index f95f8011c..c346c9c33 100644 --- a/cmd/server/notify_source.go +++ b/cmd/server/notify_source.go @@ -151,7 +151,7 @@ func (db *DB) NotifyNodes(pubkeys []string, withForeign bool) (map[string]notify for j, pk := range chunk { args[j] = pk } - rows, err := db.conn.Query(notifyNodeCols+` WHERE public_key IN (`+sqlPlaceholders(len(chunk))+`)`, args...) + rows, err := db.conn.Query(notifyNodeCols+` WHERE public_key IN (`+db.sqlPlaceholders(len(chunk))+`)`, args...) if err != nil { return nil, err } diff --git a/cmd/server/notify_source_test.go b/cmd/server/notify_source_test.go index 1d5f025a5..46fef0941 100644 --- a/cmd/server/notify_source_test.go +++ b/cmd/server/notify_source_test.go @@ -45,7 +45,7 @@ func TestDBNotifyNodesChunksLongLists(t *testing.T) { keys = append(keys, fmt.Sprintf("%064x", i)) } for _, i := range []int{3, 600, 1199} { - if _, err := db.conn.Exec(`INSERT INTO nodes (public_key, role) VALUES (?, 'companion')`, keys[i]); err != nil { + if _, err := db.conn.Exec(`INSERT INTO nodes (public_key, role) VALUES ($1, 'companion')`, keys[i]); err != nil { t.Fatal(err) } } diff --git a/cmd/server/obs_raw_hex_test.go b/cmd/server/obs_raw_hex_test.go index 45d9a45e0..557e50ffc 100644 --- a/cmd/server/obs_raw_hex_test.go +++ b/cmd/server/obs_raw_hex_test.go @@ -33,11 +33,11 @@ func seedDistinctFrames(t *testing.T, db *DB, hash string) (string, []int) { now := time.Now().UTC().Format(time.RFC3339) if _, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('CANON0000', ?, ?, 1, 5, '{"type":"CHAN"}')`, hash, now); err != nil { + VALUES ('CANON0000', $1, $2, 1, 5, '{"type":"CHAN"}')`, hash, now); err != nil { t.Fatalf("insert transmission: %v", err) } var txID int - if err := db.conn.QueryRow("SELECT id FROM transmissions WHERE hash = ?", hash).Scan(&txID); err != nil { + if err := db.conn.QueryRow("SELECT id FROM transmissions WHERE hash = $1", hash).Scan(&txID); err != nil { t.Fatalf("lookup tx id: %v", err) } @@ -52,14 +52,15 @@ func seedDistinctFrames(t *testing.T, db *DB, hash string) (string, []int) { } var ids []int for i, f := range frames { - res, err := db.conn.Exec(`INSERT INTO observations + var id int64 + err := db.conn.QueryRow(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp, raw_hex) - VALUES (?, 1, 5.0, -90, ?, ?, ?)`, - txID, f.pathJSON, time.Now().Unix()-int64(i), f.rawHex) + VALUES ($1, 1, 5.0, -90, $2, $3, $4) RETURNING id`, + txID, f.pathJSON, time.Now().Unix()-int64(i), f.rawHex).Scan(&id) if err != nil { t.Fatalf("insert observation %d: %v", i, err) } - id, err := res.LastInsertId() + if err != nil { t.Fatalf("last insert id: %v", err) } diff --git a/cmd/server/observer_airtime_test.go b/cmd/server/observer_airtime_test.go index a0c151546..d2b0e0483 100644 --- a/cmd/server/observer_airtime_test.go +++ b/cmd/server/observer_airtime_test.go @@ -55,7 +55,7 @@ func TestGetMetricsSummaryAirtime1h(t *testing.T) { now := time.Now().UTC().Truncate(time.Second) ins := func(obs string, minAgo, tx, rx int) { ts := now.Add(-time.Duration(minAgo) * time.Minute).Format(time.RFC3339) - if _, err := db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs) VALUES (?, ?, ?, ?, ?)", + if _, err := db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs) VALUES ($1, $2, $3, $4, $5)", obs, ts, -110.0, tx, rx); err != nil { t.Fatal(err) } diff --git a/cmd/server/observer_blacklist_test.go b/cmd/server/observer_blacklist_test.go index ec6b2b473..0457e25a7 100644 --- a/cmd/server/observer_blacklist_test.go +++ b/cmd/server/observer_blacklist_test.go @@ -48,8 +48,8 @@ func TestConfigIsObserverBlacklistedNil(t *testing.T) { func TestObserverBlacklistFiltersHandleObservers(t *testing.T) { db := setupTestDB(t) - db.conn.Exec("INSERT OR IGNORE INTO observers (id, name, iata, last_seen) VALUES ('goodobs', 'GoodObs', 'SFO', datetime('now'))") - db.conn.Exec("INSERT OR IGNORE INTO observers (id, name, iata, last_seen) VALUES ('badobs', 'BadObs', 'LAX', datetime('now'))") + db.conn.Exec(testNativeSQL(`INSERT INTO observers (id, name, iata, last_seen) VALUES ('goodobs', 'GoodObs', 'SFO', strftime('%Y-%m-%dT%H:%M:%SZ','now')) ON CONFLICT DO NOTHING`, "INSERT INTO observers (id, name, iata, last_seen) VALUES ('goodobs', 'GoodObs', 'SFO', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')) ON CONFLICT DO NOTHING")) + db.conn.Exec(testNativeSQL(`INSERT INTO observers (id, name, iata, last_seen) VALUES ('badobs', 'BadObs', 'LAX', strftime('%Y-%m-%dT%H:%M:%SZ','now')) ON CONFLICT DO NOTHING`, "INSERT INTO observers (id, name, iata, last_seen) VALUES ('badobs', 'BadObs', 'LAX', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')) ON CONFLICT DO NOTHING")) cfg := &Config{ ObserverBlacklist: []string{"badobs"}, @@ -89,7 +89,7 @@ func TestObserverBlacklistFiltersHandleObservers(t *testing.T) { func TestObserverBlacklistFiltersObserverDetail(t *testing.T) { db := setupTestDB(t) - db.conn.Exec("INSERT OR IGNORE INTO observers (id, name, iata, last_seen) VALUES ('badobs', 'BadObs', 'LAX', datetime('now'))") + db.conn.Exec(testNativeSQL(`INSERT INTO observers (id, name, iata, last_seen) VALUES ('badobs', 'BadObs', 'LAX', strftime('%Y-%m-%dT%H:%M:%SZ','now')) ON CONFLICT DO NOTHING`, "INSERT INTO observers (id, name, iata, last_seen) VALUES ('badobs', 'BadObs', 'LAX', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')) ON CONFLICT DO NOTHING")) cfg := &Config{ ObserverBlacklist: []string{"badobs"}, @@ -108,7 +108,7 @@ func TestObserverBlacklistFiltersObserverDetail(t *testing.T) { func TestNoObserverBlacklistPassesAll(t *testing.T) { db := setupTestDB(t) - db.conn.Exec("INSERT OR IGNORE INTO observers (id, name, iata, last_seen) VALUES ('someobs', 'SomeObs', 'SFO', datetime('now'))") + db.conn.Exec(testNativeSQL(`INSERT INTO observers (id, name, iata, last_seen) VALUES ('someobs', 'SomeObs', 'SFO', strftime('%Y-%m-%dT%H:%M:%SZ','now')) ON CONFLICT DO NOTHING`, "INSERT INTO observers (id, name, iata, last_seen) VALUES ('someobs', 'SomeObs', 'SFO', to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')) ON CONFLICT DO NOTHING")) cfg := &Config{} srv := NewServer(db, cfg, NewHub()) diff --git a/cmd/server/observer_count_1888_test.go b/cmd/server/observer_count_1888_test.go index 89d4bd225..65aa4d29c 100644 --- a/cmd/server/observer_count_1888_test.go +++ b/cmd/server/observer_count_1888_test.go @@ -27,18 +27,18 @@ func seedObserversForCount(t *testing.T, db *DB) { live := []string{"live1", "live2", "live3"} for _, id := range live { if _, err := db.conn.Exec(`INSERT INTO observers (id, name, last_seen, first_seen, packet_count, inactive) - VALUES (?, ?, ?, ?, 10, 0)`, id, "Observer "+id, now, now); err != nil { + VALUES ($1, $2, $3, $4, 10, 0)`, id, "Observer "+id, now, now); err != nil { t.Fatalf("insert %s: %v", id, err) } } // One explicitly soft-deleted, one with a NULL flag — GetObservers treats // NULL as live, so only the inactive=1 row must be excluded. if _, err := db.conn.Exec(`INSERT INTO observers (id, name, last_seen, first_seen, packet_count, inactive) - VALUES ('gone1', 'Gone One', ?, ?, 5, 1)`, now, now); err != nil { + VALUES ('gone1', 'Gone One', $1, $2, 5, 1)`, now, now); err != nil { t.Fatalf("insert gone1: %v", err) } if _, err := db.conn.Exec(`INSERT INTO observers (id, name, last_seen, first_seen, packet_count, inactive) - VALUES ('nullflag', 'Null Flag', ?, ?, 5, NULL)`, now, now); err != nil { + VALUES ('nullflag', 'Null Flag', $1, $2, 5, NULL)`, now, now); err != nil { t.Fatalf("insert nullflag: %v", err) } } diff --git a/cmd/server/observer_naive_clock_1478_test.go b/cmd/server/observer_naive_clock_1478_test.go index e0d369c24..5e749367f 100644 --- a/cmd/server/observer_naive_clock_1478_test.go +++ b/cmd/server/observer_naive_clock_1478_test.go @@ -30,7 +30,7 @@ func TestHandleObservers_Issue1478_SurfacesRecentNaiveSkew(t *testing.T) { _, err := srv.db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count, clock_skew_seconds, clock_skew_count_24h, clock_last_naive_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)`, "naive-obs-1", "California Pi", "SFO", now.Format(time.RFC3339), now.Add(-7*24*time.Hour).Format(time.RFC3339), 42, -28800, 17, recent) @@ -91,7 +91,7 @@ func TestHandleObservers_Issue1478_DecaysAfter24h(t *testing.T) { _, err := srv.db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count, clock_skew_seconds, clock_skew_count_24h, clock_last_naive_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)`, "naive-obs-old", "Fixed Pi", "LAX", now.Format(time.RFC3339), now.Add(-30*24*time.Hour).Format(time.RFC3339), 99, -28800, 5, stale) @@ -136,7 +136,7 @@ func TestHandleObserverDetail_Issue1478_IncludesClockNaiveFields(t *testing.T) { _, err := srv.db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count, clock_skew_seconds, clock_skew_count_24h, clock_last_naive_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)`, "naive-obs-detail", "Detail Pi", "SJC", now.Format(time.RFC3339), now.Add(-2*24*time.Hour).Format(time.RFC3339), 7, 25200, 3, recent) diff --git a/cmd/server/observer_planner_test.go b/cmd/server/observer_planner_test.go new file mode 100644 index 000000000..d49208966 --- /dev/null +++ b/cmd/server/observer_planner_test.go @@ -0,0 +1,211 @@ +package main + +import ( + "context" + "database/sql" + "fmt" + "path/filepath" + "reflect" + "sort" + "strings" + "testing" + "time" + + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/dbschema/legacy" +) + +func TestGetObserverPacketCountsWindowAndIdentity(t *testing.T) { + db := setupTestDB(t) + for _, query := range []string{ + `INSERT INTO observers(rowid,id,inactive) VALUES(1,'obs-a',0),(2,'obs-b',1)`, + `INSERT INTO transmissions(id,raw_hex,hash,first_seen) VALUES(1,'00','counts','2026-01-01')`, + `INSERT INTO observations(transmission_id,observer_idx,timestamp) VALUES + (1,1,100),(1,1,101),(1,1,102),(1,2,101),(1,NULL,101),(1,99,101)`, + } { + if _, err := db.conn.Exec(query); err != nil { + t.Fatal(err) + } + } + // Strict cutoff and inner-join identity semantics are shared by both engines. + want := map[string]int{"obs-a": 2, "obs-b": 1} + if got := db.GetObserverPacketCounts(100); !reflect.DeepEqual(got, want) { + t.Fatalf("observer counts=%v, want=%v", got, want) + } +} + +// Match the B corpus's cardinalities: 128 observers, 16 observations per +// transmission on average, eight days, and one eighth in the recent window. +// SQL-only data exercises the real endpoint pipeline without a PacketStore. +func observerPlannerFixture(t testing.TB, observations int) (*sql.DB, string, int64, map[string]int) { + t.Helper() + if testBackend(t) != dbconfig.SQLite { + t.Skip("SQLite identity migration and planner fixture") + } + path := filepath.Join(t.TempDir(), "observers.db") + dsn, err := legacy.WriterDSN(path) + if err != nil { + t.Fatal(err) + } + w, err := sql.Open("sqlite3", dsn) + if err != nil { + t.Fatal(err) + } + w.SetMaxOpenConns(1) + t.Cleanup(func() { w.Close() }) + if err = legacy.ApplyBase(w); err != nil { + t.Fatal(err) + } + if err = legacy.Apply(w, nil); err != nil { + t.Fatal(err) + } + if err = dbschema.EnsureSQLiteObserverTimeIndex(context.Background(), w); err != nil { + t.Fatal(err) + } + epoch := time.Now().Unix() + tx, err := w.Begin() + if err != nil { + t.Fatal(err) + } + defer tx.Rollback() + for _, query := range []string{ + `WITH RECURSIVE n(i) AS (VALUES(1) UNION ALL SELECT i+1 FROM n WHERE i<128) + INSERT INTO observers(rowid,id,name,iata,last_seen,first_seen,packet_count) + SELECT i,printf('%064x',i),printf('Synthetic observer %d',i),'AAA','2026-01-01','2026-01-01',0 FROM n`, + `INSERT INTO nodes(public_key,name,role,lat,lon) SELECT id,name,'repeater',20,30 FROM observers`, + fmt.Sprintf(`WITH RECURSIVE n(i) AS (VALUES(1) UNION ALL SELECT i+1 FROM n WHERE i<%d) + INSERT INTO transmissions(id,raw_hex,hash,first_seen) SELECT i,'00',printf('fixture-%%d',i),'2026-01-01' FROM n`, observations/16), + fmt.Sprintf(`WITH RECURSIVE n(i) AS (VALUES(0) UNION ALL SELECT i+1 FROM n WHERE i+1<%d) + INSERT INTO observations(transmission_id,observer_idx,timestamp,path_json) + SELECT i/16+1,i%%128+1,CASE WHEN (i/160)%%8=0 THEN %d-2700+(i/16)%%1800 + ELSE %d-((i/160)%%8)*86400-43200+(i/16)%%3600 END,'[]' FROM n`, observations, epoch, epoch), + } { + if _, err = tx.Exec(query); err != nil { + t.Fatal(err) + } + } + if err = tx.Commit(); err != nil { + t.Fatal(err) + } + if _, err = w.Exec(`ANALYZE`); err != nil { + t.Fatal(err) + } + want := make(map[string]int) + for i := 0; i < observations; i++ { + if (i/160)%8 == 0 { + want[fmt.Sprintf("%064x", i%128+1)]++ + } + } + return w, path, epoch - 3600, want +} + +func observerPlannerReader(t testing.TB, path string, since int64, want map[string]int) *DB { + t.Helper() + db, err := OpenDB(path) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { db.Close() }) + if got := db.GetObserverPacketCounts(since); !reflect.DeepEqual(got, want) { + t.Fatalf("recent observation counts changed: got %d observer groups, want %d", len(got), len(want)) + } + return db +} + +func observerAggregatePlan(t testing.TB, db *DB, since int64) []string { + t.Helper() + rows, err := db.conn.Query("EXPLAIN QUERY PLAN "+db.observerPacketCountsSQL(), since) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + var plan []string + for rows.Next() { + var id, parent, unused int + var detail string + if err := rows.Scan(&id, &parent, &unused, &detail); err != nil { + t.Fatal(err) + } + plan = append(plan, detail) + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + return plan +} + +func TestObserverAggregatePlannerAfterSQLiteUpgrade(t *testing.T) { + for _, repair := range []bool{false, true} { + name := "legacy adoption" + if repair { + name = "already rebuilt without statistics" + } + t.Run(name, func(t *testing.T) { + w, path, since, want := observerPlannerFixture(t, 128000) + if repair { + if err := dbschema.ApplySQLite(w, nil); err != nil { + t.Fatal(err) + } + if _, err := w.Exec(`DELETE FROM sqlite_stat1 WHERE tbl='observers'; ANALYZE sqlite_schema`); err != nil { + t.Fatal(err) + } + } + if err := dbschema.ApplySQLite(w, nil); err != nil { + t.Fatal(err) + } + // A new reader is important: the old writer can retain pre-rebuild + // planner estimates and conceal the statistics lost by DROP TABLE. + db := observerPlannerReader(t, path, since, want) + plan := observerAggregatePlan(t, db, since) + if strings.Contains(strings.Join(plan, "\n"), "TEMP B-TREE FOR GROUP BY") { + t.Fatalf("observer statistics lost after upgrade; aggregate sorts the observation window: %q", plan) + } + if !strings.Contains(strings.Join(plan, "\n"), "observer_idx=? AND timestamp>?") { + t.Fatalf("aggregate no longer uses observer/time range lookups: %q", plan) + } + }) + } +} + +// Run explicitly with -benchtime=50x. Reports uncached response-build latency, +// not full concurrent HTTP latency or a replacement for the paired workload. +func BenchmarkObserverListSQLiteUpgrade(b *testing.B) { + w, path, since, want := observerPlannerFixture(b, 2048000) + var reference ObserverListResponse + for _, stage := range []string{"legacy", "upgraded"} { + if stage == "upgraded" { + if err := dbschema.ApplySQLite(w, nil); err != nil { + b.Fatal(err) + } + } + b.Run(stage, func(b *testing.B) { + db := observerPlannerReader(b, path, since, want) + srv := &Server{db: db, cfg: &Config{}} + b.Logf("observations=2048000 transmissions=128000 observers=128 recent=%d; plan=%q", 256000, observerAggregatePlan(b, db, since)) + response, err := srv.buildObserversDefaultResponse() + if err != nil || len(response.Observers) != 128 { + b.Fatalf("response rows=%d err=%v", len(response.Observers), err) + } + response.ServerTime = "" + if stage == "legacy" { + reference = response + } else if !reflect.DeepEqual(reference, response) { + b.Fatal("observer response semantics changed after identity migration") + } + durations := make([]time.Duration, 0, b.N) + b.ResetTimer() + for i := 0; i < b.N; i++ { + start := time.Now() + if _, err := srv.buildObserversDefaultResponse(); err != nil { + b.Fatal(err) + } + durations = append(durations, time.Since(start)) + } + b.StopTimer() + sort.Slice(durations, func(i, j int) bool { return durations[i] < durations[j] }) + b.ReportMetric(float64(durations[(len(durations)-1)/2])/float64(time.Millisecond), "p50-ms") + b.ReportMetric(float64(durations[(len(durations)*95+99)/100-1])/float64(time.Millisecond), "p95-ms") + }) + } +} diff --git a/cmd/server/observer_strings_intern_test.go b/cmd/server/observer_strings_intern_test.go index e1bc1c24e..363f1c87f 100644 --- a/cmd/server/observer_strings_intern_test.go +++ b/cmd/server/observer_strings_intern_test.go @@ -1,10 +1,7 @@ package main import ( - "database/sql" "fmt" - "os" - "path/filepath" "runtime" "strings" "sync" @@ -57,13 +54,11 @@ func assertObserverStringsShared(t *testing.T, store *PacketStore, wantObs int) func openInternTestStore(t *testing.T, numTx int) *PacketStore { t.Helper() - dir := t.TempDir() - dbPath := filepath.Join(dir, "intern.db") + dbPath := postgresTestDSN(t) createTestDBAt(t, dbPath, numTx) - t.Cleanup(func() { os.RemoveAll(dir) }) // The shared fixture leaves observers empty; give each an IATA code so // the loaders read one for every observation. - conn, err := sql.Open("sqlite3", dbPath) + conn, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } @@ -71,7 +66,7 @@ func openInternTestStore(t *testing.T, numTx int) *PacketStore { t.Fatalf("fixture observers: %v", err) } conn.Close() - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatalf("OpenDB: %v", err) } diff --git a/cmd/server/observers_canrelay_tristate_1290_test.go b/cmd/server/observers_canrelay_tristate_1290_test.go index 146f7ae9e..5073edcda 100644 --- a/cmd/server/observers_canrelay_tristate_1290_test.go +++ b/cmd/server/observers_canrelay_tristate_1290_test.go @@ -39,19 +39,19 @@ func TestObservers_CanRelayTriState_Issue1290(t *testing.T) { // Legacy: never received repeat field. can_relay=DEFAULT 1, seen=0. if _, err := srv.store.db.conn.Exec( `INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('legacy-obs', 'Legacy', 'SJC', ?, '2026-01-01T00:00:00Z', 1)`, now); err != nil { + VALUES ('legacy-obs', 'Legacy', 'SJC', $1, '2026-01-01T00:00:00Z', 1)`, now); err != nil { t.Fatalf("seed legacy: %v", err) } // Repeater: ingestor wrote can_relay=1, seen=1. if _, err := srv.store.db.conn.Exec( `INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count, can_relay, can_relay_seen) - VALUES ('rep-obs', 'Repeater', 'SFO', ?, '2026-01-01T00:00:00Z', 1, 1, 1)`, now); err != nil { + VALUES ('rep-obs', 'Repeater', 'SFO', $1, '2026-01-01T00:00:00Z', 1, 1, 1)`, now); err != nil { t.Fatalf("seed repeater: %v", err) } // Listener: ingestor wrote can_relay=0, seen=1. if _, err := srv.store.db.conn.Exec( `INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count, can_relay, can_relay_seen) - VALUES ('lst-obs', 'Listener', 'OAK', ?, '2026-01-01T00:00:00Z', 1, 0, 1)`, now); err != nil { + VALUES ('lst-obs', 'Listener', 'OAK', $1, '2026-01-01T00:00:00Z', 1, 0, 1)`, now); err != nil { t.Fatalf("seed listener: %v", err) } diff --git a/cmd/server/openapi.go b/cmd/server/openapi.go index 153f795f8..0b29daff8 100644 --- a/cmd/server/openapi.go +++ b/cmd/server/openapi.go @@ -50,14 +50,16 @@ func routeDescriptions() map[string]routeMeta { "GET /api/config/geo-filter": {Summary: "Get geo-filter configuration", Tag: "config"}, // Admin / system - "GET /api/health": {Summary: "Health check", Description: "Returns server health, uptime, and memory stats.", Tag: "admin"}, - "GET /api/stats": {Summary: "Network statistics", Description: "Returns aggregate stats (node counts, packet counts, observer counts). Cached for 10s.", Tag: "admin"}, - "GET /api/perf": {Summary: "Performance statistics", Description: "Returns per-endpoint request timing and slow query log.", Tag: "admin"}, - "GET /api/mqtt/status": {Summary: "MQTT source status", Description: "Returns per-MQTT-source connection state and counters (lastConnectUnix, lastPacketUnix, packetsTotal, etc.). Broker URL passwords are masked. Sourced from the ingestor stats file; empty list when unavailable. (#1043)", Tag: "admin"}, - "POST /api/perf/reset": {Summary: "Reset performance stats", Tag: "admin", Auth: true, Session: true}, + "GET /api/health": {Summary: "Health check", Description: "Returns server health, uptime, and memory stats.", Tag: "admin"}, + "GET /api/stats": {Summary: "Network statistics", Description: "Returns aggregate stats (node counts, packet counts, observer counts). Cached for 10s.", Tag: "admin"}, + "GET /api/perf": {Summary: "Performance statistics", Description: "Returns per-endpoint request timing and slow query log.", Tag: "admin"}, + "GET /api/perf/database": {Summary: "Selected database diagnostics", Description: "Native metrics for the selected SQLite or PostgreSQL backend. engine identifies the backend; unavailable metrics stay null. Database and row samples refresh every 30 seconds, while connection pool use remains live. SQLite reports WAL bytes, pages, free space, and planner statistics availability without checkpointing or changing the database.", Tag: "admin"}, + "GET /api/perf/postgres": {Summary: "PostgreSQL database statistics", Description: "Returns databaseBytes and PostgreSQL block-cache hit ratio (cacheHitRate, 0..1 or null without available reads), sampled with table counts once per 30 seconds and shared with /api/perf. sampledAt and sampleIntervalSeconds disclose freshness; stale marks a failed refresh. Before a first successful sample, unavailable size/counts are null and error is set. After a refresh failure, the last storage/count sample remains with stale=true and the unavailable block ratio is null. Pool usage and waits remain live. Block-cache counters are cumulative since the PostgreSQL statistics reset and exclude the operating-system cache.", Tag: "admin"}, + "GET /api/mqtt/status": {Summary: "MQTT source status", Description: "Returns per-MQTT-source connection state and counters (lastConnectUnix, lastPacketUnix, packetsTotal, etc.). Broker URL passwords are masked. Sourced from the ingestor stats file; empty list when unavailable. (#1043)", Tag: "admin"}, + "POST /api/perf/reset": {Summary: "Reset performance stats", Tag: "admin", Auth: true, Session: true}, // "POST /api/admin/prune" removed in #1283 (ingestor owns prune). "GET /api/debug/affinity": {Summary: "Debug neighbor affinity scores", Tag: "admin", Auth: true, Session: true}, - "GET /api/backup": {Summary: "Download SQLite backup", Description: "Streams a consistent SQLite snapshot of the analyzer DB (VACUUM INTO). Response is application/octet-stream with attachment filename corescope-backup-.db.", Tag: "admin", Auth: true, Session: true}, + "GET /api/backup": {Summary: "Download PostgreSQL backup", Description: "Streams a consistent PostgreSQL custom-format pg_dump archive of telemetry. Restore with pg_restore into a separate database. Response is application/octet-stream with attachment filename corescope-backup-.dump.", Tag: "admin", Auth: true, Session: true}, // User management (optional; routes exist only when userManagement.enabled) "POST /api/auth/register": {Summary: "Register an account", Description: "Creates a pending account and mails an activation link. The response is identical whether or not the address is already registered.", Tag: "users"}, @@ -94,7 +96,7 @@ func routeDescriptions() map[string]routeMeta { "POST /api/admin/users/{id}/mail/{mailId}/refresh": {Summary: "Pull delivery events for one mail from the provider (admin)", Tag: "users", Session: true}, "GET /api/admin/audit": {Summary: "Global audit log (admin)", Description: "Newest first, keyset-paginated: {entries, next}. Pass next as before for the following page; next is null on the last page. Entry: {id, at, action, actor, target, detail}; actor and target are {id, displayName, email}, {id, deleted: true} for a removed account, or null for the system. Login rows (user.login, user.login.failed) are kept 90 days. 400 on an invalid parameter.", Tag: "users", Session: true, QueryParams: []paramMeta{{Name: "action", Description: "One action, or a group ending in .* (user.login.* matches user.login and user.login.failed)", Type: "string"}, {Name: "user", Description: "User id, as actor or target", Type: "integer"}, {Name: "from", Description: "RFC 3339 time, inclusive", Type: "string"}, {Name: "to", Description: "RFC 3339 time, inclusive", Type: "string"}, {Name: "before", Description: "Only entries with a smaller id (next of the previous page)", Type: "integer"}, {Name: "limit", Description: "Page size, default 100, capped at 500", Type: "integer"}}}, "GET /api/admin/stats": {Summary: "User figures for the admin overview (admin)", Description: "Accounts by status, active admins, pending accounts older than 24 hours (stuckPending), bouncing addresses, registrations in 7 and 30 days and per UTC day for 30 days (newPerDay, oldest first), users active in 7 and 30 days, logins and failed logins in 24 hours, mail sent in 7 days by latest delivery status (mail7d), and accounts with 5 or more failed logins in 24 hours (guessing: [{userId, displayName, failed}]).", Tag: "users", Session: true}, - "GET /api/admin/users-backup": {Summary: "Download a users.db snapshot (admin)", Description: "A fresh, consistent copy of users.db (VACUUM INTO) as application/octet-stream, attachment corescope-users-.db (UTC). It holds every password hash and address: store it encrypted. Records a user.backup audit row. The analyzer database has its own route, GET /api/backup.", Tag: "users", Session: true}, + "GET /api/admin/users-backup": {Summary: "Download accounts backup (admin)", Description: "A consistent PostgreSQL custom-format pg_dump archive of accounts as application/octet-stream, attachment corescope-users-.dump (UTC). It holds every password hash and address: store it encrypted. Records a user.backup audit row. The analyzer database has its own route, GET /api/backup.", Tag: "users", Session: true}, "POST /api/proposals": {Summary: "Propose a hashtag channel", Description: "Body {kind: \"hashtag_channel\", subject}. subject is trimmed and gets a leading # when missing; at most 31 UTF-8 bytes including the #, no control or invisible characters (ZWJ allowed), not Public. 201 with the proposal {id, kind, subject, status, note, createdAt, decidedAt}. 400 invalid kind or name, 409 already pending, already approved or rejected, or already decrypted on this instance (a name in config.json hashChannels or channelKeys, case-sensitive), 429 when the caller created userManagement.channelProposals.perUserPerDay proposals in 24 hours or maxPending proposals wait for review. Registered only when userManagement.channelProposals.enabled.", Tag: "users", Session: true}, "GET /api/account/proposals": {Summary: "List own proposals", Description: "The caller's proposals, newest first: [{id, kind, subject, status, note, createdAt, decidedAt}]. note is the reviewer's reason.", Tag: "users", Session: true}, "GET /api/admin/proposals": {Summary: "List proposals (admin)", Description: "Newest first, at most 500: the proposal fields plus proposer and reviewer ({id, displayName, email}, or null once the account is deleted). 400 on an invalid filter.", Tag: "users", Session: true, QueryParams: []paramMeta{{Name: "status", Description: "pending | approved | rejected | revoked", Type: "string"}, {Name: "kind", Description: "hashtag_channel", Type: "string"}}}, diff --git a/cmd/server/packet_filter_lock_test.go b/cmd/server/packet_filter_lock_test.go new file mode 100644 index 000000000..d0ef623d8 --- /dev/null +++ b/cmd/server/packet_filter_lock_test.go @@ -0,0 +1,306 @@ +package main + +import ( + "net/http" + "net/http/httptest" + "path/filepath" + "reflect" + "sync" + "sync/atomic" + "testing" + "time" +) + +func packetFilterLockStore(t *testing.T) *PacketStore { + t.Helper() + db := setupTestDB(t) + mustExecDB(t, db, `INSERT INTO observers (id, iata) VALUES ('obs-sjc', 'SJC'), ('obs-lax', 'LAX')`) + mustExecDB(t, db, `INSERT INTO nodes (public_key, lat, lon) VALUES ('inside-node', 50.85, 4.35), ('outside-node', 48.0, 4.35)`) + s := newTestStoreWithDB(t, db, &Config{Areas: map[string]AreaEntry{ + "BEL": {Polygon: [][2]float64{{50, 2.5}, {51.5, 2.5}, {51.5, 6.4}, {50, 6.4}}}, + "EMPTY": {Polygon: [][2]float64{{0, 0}, {1, 0}, {1, 1}, {0, 1}}}, + }}) + pt := PayloadADVERT + s.packets = []*StoreTx{ + {ID: 1, Hash: "inside", FirstSeen: "2026-01-01T00:00:01Z", LatestSeen: "2026-01-01T00:00:01Z", PayloadType: &pt, + DecodedJSON: `{"public_key":"inside-node"}`, Observations: []*StoreObs{{ObserverID: "obs-sjc"}}}, + {ID: 2, Hash: "other-region", FirstSeen: "2026-01-01T00:00:02Z", LatestSeen: "2026-01-01T00:00:02Z", PayloadType: &pt, + DecodedJSON: `{"public_key":"inside-node"}`, Observations: []*StoreObs{{ObserverID: "obs-lax"}}}, + {ID: 3, Hash: "outside-area", FirstSeen: "2026-01-01T00:00:03Z", LatestSeen: "2026-01-01T00:00:03Z", PayloadType: &pt, + DecodedJSON: `{"public_key":"outside-node"}`, Observations: []*StoreObs{{ObserverID: "obs-sjc"}}}, + } + return s +} + +func packetFilterHashes(result *PacketResult) []string { + hashes := make([]string, 0, len(result.Packets)) + for _, packet := range result.Packets { + hashes = append(hashes, packet["hash"].(string)) + } + return hashes +} + +func assertColdFilterAllowsStoreReaders(t *testing.T, s *PacketStore, kind string, query func()) { + t.Helper() + s.clockSkew = &ClockSkewEngine{} + s.signalStartupLoadDone() + hook, _, entered, release := blockingLoadHook(kind) + s.cacheLoadHook = hook + var workers sync.WaitGroup + var releaseOnce sync.Once + releaseSQL := func() { releaseOnce.Do(func() { close(release) }) } + defer func() { releaseSQL(); workers.Wait() }() + oldReadiness := readiness.Swap(1) + t.Cleanup(func() { readiness.Store(oldReadiness) }) + t.Setenv("CORESCOPE_INGESTOR_STATS", filepath.Join(t.TempDir(), "missing.json")) + + queryDone := make(chan struct{}) + workers.Add(1) + go func() { defer workers.Done(); query(); close(queryDone) }() + select { + case <-entered: + case <-time.After(time.Second): + t.Fatal("query did not reach the cold filter lookup") + } + + // Queue the poller writer while the filter's database work is blocked. + // Both the writer and healthz must complete before SQL is released. + writerStarted, writerDone := make(chan struct{}), make(chan struct{}) + workers.Add(1) + go func() { + defer workers.Done() + close(writerStarted) + s.mu.Lock() + s.mu.Unlock() + close(writerDone) + }() + <-writerStarted + healthDone := make(chan int, 1) + workers.Add(1) + go func() { + defer workers.Done() + w := httptest.NewRecorder() + (&Server{store: s}).handleHealthz(w, httptest.NewRequest(http.MethodGet, "/api/healthz", nil)) + healthDone <- w.Code + }() + select { + case <-writerDone: + case <-time.After(time.Second): + t.Error("store writer blocked behind cold filter SQL") + } + select { + case status := <-healthDone: + if status != http.StatusOK { + t.Errorf("healthz status = %d, want 200", status) + } + case <-time.After(time.Second): + t.Error("healthz blocked behind cold filter SQL and the queued writer") + } + releaseSQL() + select { + case <-queryDone: + case <-time.After(time.Second): + t.Fatal("query did not finish after releasing filter SQL") + } +} + +func TestColdPacketFiltersDoNotHoldStoreLock(t *testing.T) { + for _, grouped := range []bool{false, true} { + for _, filter := range []struct { + name, kind string + q PacketQuery + want []string + }{ + {"region", "regionObs", PacketQuery{Region: "SJC"}, []string{"outside-area", "inside"}}, + {"area", "areaNodes", PacketQuery{Area: "BEL"}, []string{"other-region", "inside"}}, + } { + name := "raw/" + filter.name + if grouped { + name = "grouped/" + filter.name + } + t.Run(name, func(t *testing.T) { + s := packetFilterLockStore(t) + query := s.QueryPackets + if grouped { + query = s.QueryGroupedPackets + } + var got *PacketResult + assertColdFilterAllowsStoreReaders(t, s, filter.kind, func() { got = query(filter.q) }) + if hashes := packetFilterHashes(got); !reflect.DeepEqual(hashes, filter.want) || got.Total != len(filter.want) { + t.Errorf("filtered packets = %v, total %d; want %v", hashes, got.Total, filter.want) + } + }) + } + } +} + +func TestColdRegionConsumersDoNotHoldStoreLock(t *testing.T) { + for _, tc := range []struct { + name string + query func(*PacketStore) + }{ + {"channels", func(s *PacketStore) { s.GetChannels("SJC") }}, + {"encrypted-channels", func(s *PacketStore) { s.GetEncryptedChannels("SJC") }}, + {"channel-messages", func(s *PacketStore) { s.GetChannelMessages("public", 10, 0, "SJC") }}, + {"analytics-channels", func(s *PacketStore) { s.computeAnalyticsChannels("SJC", "", TimeWindow{}) }}, + {"analytics-rf", func(s *PacketStore) { s.computeAnalyticsRF("SJC", "", TimeWindow{}) }}, + {"windowed-subpaths", func(s *PacketStore) { s.computeAnalyticsSubpathsWindowed("SJC", 2, 3, 10, TimeWindow{}) }}, + {"subpaths", func(s *PacketStore) { s.computeAnalyticsSubpaths("SJC", 2, 3, 10) }}, + } { + t.Run(tc.name, func(t *testing.T) { + s := packetFilterLockStore(t) + s.nodePM, s.nodeCacheTime = buildPrefixMap(nil), time.Now() + assertColdFilterAllowsStoreReaders(t, s, "regionObs", func() { tc.query(s) }) + }) + } +} + +func TestPacketFilterEmptyAndUnknownSets(t *testing.T) { + all := []string{"outside-area", "other-region", "inside"} + for _, grouped := range []bool{false, true} { + for _, tc := range []struct { + name string + q PacketQuery + want []string + }{ + {"no-filters", PacketQuery{}, all}, + {"unknown-region", PacketQuery{Region: "UNKNOWN"}, []string{}}, + {"unknown-region-with-area", PacketQuery{Region: "UNKNOWN", Area: "BEL"}, []string{}}, + {"unknown-area", PacketQuery{Area: "UNKNOWN"}, all}, + {"empty-area", PacketQuery{Area: "EMPTY"}, all}, + {"combined", PacketQuery{Region: "SJC", Area: "BEL"}, []string{"inside"}}, + } { + name := "raw/" + tc.name + if grouped { + name = "grouped/" + tc.name + } + t.Run(name, func(t *testing.T) { + s := packetFilterLockStore(t) + if tc.q.Region == "UNKNOWN" { + s.cacheLoadHook = func(kind string) { + if kind == "areaNodes" { + t.Error("empty region should short-circuit the area lookup") + } + } + } + query := s.QueryPackets + if grouped { + query = s.QueryGroupedPackets + } + got := query(tc.q) + if hashes := packetFilterHashes(got); !reflect.DeepEqual(hashes, tc.want) || got.Total != len(tc.want) { + t.Errorf("filtered packets = %v, total %d; want %v", hashes, got.Total, tc.want) + } + }) + } + } +} + +func TestPacketFilterUsesResolvedSnapshotAfterCacheEviction(t *testing.T) { + for _, grouped := range []bool{false, true} { + name := "raw" + if grouped { + name = "grouped" + } + t.Run(name, func(t *testing.T) { + s := packetFilterLockStore(t) + query := s.QueryPackets + if grouped { + query = s.QueryGroupedPackets + } + hook, _, entered, release := blockingLoadHook("areaNodes") + var regionLoads atomic.Int32 + s.cacheLoadHook = func(kind string) { + if kind == "regionObs" { + regionLoads.Add(1) + } + hook(kind) + } + var releaseOnce sync.Once + releaseSQL := func() { releaseOnce.Do(func() { close(release) }) } + var worker sync.WaitGroup + defer func() { releaseSQL(); worker.Wait() }() + result := make(chan *PacketResult, 1) + worker.Add(1) + go func() { + defer worker.Done() + result <- query(PacketQuery{Region: "SJC", Area: "BEL"}) + }() + select { + case <-entered: + case <-time.After(time.Second): + t.Fatal("query did not resolve its region before the area lookup") + } + // The refresh may evict a region between its lookup and the + // locked packet scan. Retain the set; do not query it again. + s.regionObsMu.Lock() + delete(s.regionObsCache, "SJC") + s.regionObsMu.Unlock() + releaseSQL() + got := <-result + if hashes := packetFilterHashes(got); !reflect.DeepEqual(hashes, []string{"inside"}) { + t.Errorf("snapshot-filtered packets = %v, want [inside]", hashes) + } + if loads := regionLoads.Load(); loads != 1 { + t.Errorf("region loaded %d times; query must retain its first snapshot", loads) + } + }) + } +} + +func TestPacketFiltersPreserveWarmAndStaleCaches(t *testing.T) { + for _, grouped := range []bool{false, true} { + for _, stale := range []bool{false, true} { + for _, kind := range []string{"regionObs", "areaNodes"} { + name := "raw/warm/" + kind + if grouped { + name = "grouped/warm/" + kind + } + if stale { + name += "/stale" + } + t.Run(name, func(t *testing.T) { + s := packetFilterLockStore(t) + query := s.QueryPackets + if grouped { + query = s.QueryGroupedPackets + } + cachedAt := time.Now() + if stale { + cachedAt = cachedAt.Add(-time.Hour) + } + q, want, refreshKey := PacketQuery{Region: "SJC"}, "other-region", "regionObs" + // Deliberately differ from the database, proving the cached + // set is served while the refresh is blocked. + s.regionObsCache["SJC"] = map[string]bool{"obs-lax": true} + s.regionObsCacheTime = cachedAt + if kind == "areaNodes" { + q, want, refreshKey = PacketQuery{Area: "BEL"}, "outside-area", "areaNodes:BEL" + s.areaNodeCache["BEL"] = map[string]bool{"outside-node": true} + s.areaNodeCacheTimes["BEL"] = cachedAt + } + hook, calls, entered, release := blockingLoadHook(kind) + s.cacheLoadHook = hook + defer func() { + close(release) + waitUntil(t, "filter refresh to finish", func() bool { return !s.cacheRefresh.running(refreshKey) }) + }() + var got *PacketResult + returnsWithin(t, time.Second, "cached packet filter", func() { got = query(q) }) + if hashes := packetFilterHashes(got); !reflect.DeepEqual(hashes, []string{want}) { + t.Errorf("cached packets = %v, want [%s]", hashes, want) + } + if stale { + select { + case <-entered: + case <-time.After(time.Second): + t.Fatal("stale filter did not start a background refresh") + } + } else if atomic.LoadInt64(calls) != 0 { + t.Fatal("warm filter queried the database") + } + }) + } + } + } +} diff --git a/cmd/server/packet_load.go b/cmd/server/packet_load.go new file mode 100644 index 000000000..d75c4fd3b --- /dev/null +++ b/cmd/server/packet_load.go @@ -0,0 +1,356 @@ +package main + +import ( + "database/sql" + "fmt" + "strconv" + "strings" + "time" +) + +type packetLoadCursor struct { + id int + recency string +} + +type loadedPacket struct { + tx *StoreTx + cursor packetLoadCursor + bytes int64 + hops int + resolved map[string]bool + nodes map[string]bool +} + +// loadPacketRange is shared by the hot and background loaders. The SQL cursor +// walks newest-first for a bounded cache, including timestamp ties and ID zero. +// Unlimited loads keep the indexed ID walk; they do not need recency sorting. +// Memory is charged +// per observation, not from an assumed average fanout. An over-budget packet is +// discarded before publishing any of its indexes; SQL still owns its full data. +func (s *PacketStore) loadPacketRange(from, to time.Time, chunkSize int, onChunk func(int, int)) (int, error) { + _, relayPM := s.getCachedNodesAndPM() + var cursor packetLoadCursor + total := 0 + for { + s.mu.RLock() + limited := s.startupMemoryLimited + s.mu.RUnlock() + if limited { + return total, nil + } + count, next, err := s.loadPacketChunk(from, to, cursor, chunkSize, relayPM) + if err != nil { + return total, err + } + if count == 0 { + return total, nil + } + total += count + if onChunk != nil { + onChunk(count, total) + } + if count < chunkSize { + return total, nil + } + cursor = next + } +} + +func (s *PacketStore) packetLoadQuery(from, to time.Time, cursor packetLoadCursor, limit int) (string, []any) { + axis := "first_seen" + if s.db.hasLastSeen { + axis = "last_seen" + } + var conditions []string + var args []any + parameter := func(value any) string { + args = append(args, value) + return s.db.parameter(len(args)) + } + timestamp := func(t time.Time) any { + if s.db.hasLastSeen { + return t.Unix() + } + return t.UTC().Format(time.RFC3339) + } + if !from.IsZero() { + conditions = append(conditions, "t2."+axis+" >= "+parameter(timestamp(from))) + } + if !to.IsZero() { + conditions = append(conditions, "t2."+axis+" < "+parameter(timestamp(to))) + } + if cursor.recency != "" { + if s.maxMemoryMB > 0 { + // Separate bind slots also work with SQLite's positional '?' syntax. + conditions = append(conditions, "(t2."+axis+" < "+parameter(cursor.recency)+" OR (t2."+axis+" = "+parameter(cursor.recency)+" AND t2.id < "+parameter(cursor.id)+"))") + } else { + conditions = append(conditions, "t2.id > "+parameter(cursor.id)) + } + } + order := "id ASC" + if s.maxMemoryMB > 0 { + order = axis + " DESC, %s.id DESC" + } + innerOrder := strings.ReplaceAll("t2."+order, "%s", "t2") + outerOrder := strings.ReplaceAll("t."+order, "%s", "t") + where := "" + if len(conditions) > 0 { + where = " WHERE " + strings.Join(conditions, " AND ") + } + query := `SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type, + t.payload_type, t.decoded_json, o.id, ` + if s.db.isV3 { + query += s.db.nativeSQL(`obs.id, obs.name, COALESCE(obs.iata, ''), o.direction, + o.snr, o.rssi, o.score, o.path_json, strftime('%Y-%m-%dT%H:%M:%fZ', o.timestamp, 'unixepoch')`, + `obs.id, obs.name, COALESCE(obs.iata, ''), o.direction, + o.snr, o.rssi, o.score, o.path_json, to_char(to_timestamp(o.timestamp) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')`) + } else { + query += `o.observer_id, o.observer_name, COALESCE(obs.iata, ''), o.direction, + o.snr, o.rssi, o.score, o.path_json, o.timestamp` + } + if s.db.hasResolvedPath { + query += ", o.resolved_path" + } + if s.db.hasScopeName { + query += ", t.scope_name" + } + query += ", t." + axis + ` FROM (SELECT * FROM transmissions t2` + where + + " ORDER BY " + innerOrder + " LIMIT " + strconv.Itoa(limit) + `) AS t + LEFT JOIN observations o ON o.transmission_id = t.id` + if s.db.isV3 { + query += " LEFT JOIN observers obs ON obs.rowid = o.observer_idx" + } else { + query += " LEFT JOIN observers obs ON obs.id = o.observer_id" + } + query += " ORDER BY " + outerOrder + ", o.timestamp DESC, o.id DESC" + + return query, args +} + +func (s *PacketStore) loadPacketChunk(from, to time.Time, cursor packetLoadCursor, limit int, relayPM *prefixMap) (int, packetLoadCursor, error) { + query, args := s.packetLoadQuery(from, to, cursor, limit) + + // Serialize with live advert ingestion before opening a cursor, but do not + // hold the packet-store lock while doing SQL or decoding observations. + s.advertEvidenceMu.Lock() + defer s.advertEvidenceMu.Unlock() + rows, err := s.db.conn.Query(query, args...) + if err != nil { + return 0, cursor, err + } + defer rows.Close() + s.mu.RLock() + remaining := int64(s.maxMemoryMB)*1048576 - s.trackedBytes + s.mu.RUnlock() + var packets []loadedPacket + var current *loadedPacket + var currentCursor packetLoadCursor + var bytes int64 + limited := false + seen := 0 + skippedHops := 0 + finish := func() { + if current == nil { + return + } + pickBestObservation(current.tx) + current.bytes += rechargeTx(current.tx) + packets = append(packets, *current) + bytes += current.bytes + cursor = currentCursor + current = nil + } + var id int + var raw, hash, first, decoded, recency sql.NullString + var route, payload, obsID sql.NullInt64 + var observer, name, iata, direction, path, stamp, resolved, scope sql.NullString + var snr, rssi, score sql.NullFloat64 + scan := []any{&id, &raw, &hash, &first, &route, &payload, &decoded, + &obsID, &observer, &name, &iata, &direction, &snr, &rssi, &score, &path, &stamp} + if s.db.hasResolvedPath { + scan = append(scan, &resolved) + } + if s.db.hasScopeName { + scan = append(scan, &scope) + } + scan = append(scan, &recency) + for rows.Next() { + if err := rows.Scan(scan...); err != nil { + return 0, cursor, err + } + if current == nil || current.tx.ID != id { + finish() + seen++ + currentCursor = packetLoadCursor{id: id, recency: recency.String} + current = &loadedPacket{cursor: currentCursor, tx: &StoreTx{ + ID: id, RawHex: raw.String, Hash: hash.String, FirstSeen: first.String, LatestSeen: first.String, + RouteType: nullIntPtr(route), PayloadType: nullIntPtr(payload), DecodedJSON: decoded.String, + ScopeName: nullStrPtr(scope), + }, resolved: make(map[string]bool), nodes: make(map[string]bool)} + current.bytes = rechargeTx(current.tx) + } + tx := current.tx + if obsID.Valid { + if tx.hasObservation(observer.String, path.String) { + continue + } + obs := &StoreObs{ID: int(obsID.Int64), TransmissionID: id, + ObserverID: internStr(observer.String), ObserverName: internStr(name.String), ObserverIATA: internStr(iata.String), + Direction: internStr(direction.String), SNR: nullFloatPtr(snr), RSSI: nullFloatPtr(rssi), Score: nullFloatPtr(score), + PathJSON: path.String, Timestamp: normalizeTimestamp(stamp.String)} + tx.addObservation(obs) + if obs.Timestamp > tx.LatestSeen { + tx.LatestSeen = obs.Timestamp + } + current.bytes += estimateStoreObsBytes(obs) + if hops := pathLen(path.String); len(tx.Observations) == 1 || hops > current.hops { + current.hops = hops + tx.PathJSON = path.String + tx.pathParsed = false + current.bytes += rechargeTx(tx) + } + if resolved.String != "" { + for _, pk := range extractResolvedPubkeys(unmarshalResolvedPath(resolved.String)) { + current.resolved[pk] = true + } + } else if relayPM != nil && path.String != "" && path.String != "[]" { + for _, pk := range extractResolvedPubkeys(resolvePathForObsColdLoad(path.String, observer.String, tx, relayPM, &skippedHops)) { + current.nodes[pk] = true + } + } + } + if s.maxMemoryMB > 0 && bytes+current.bytes > remaining { + limited = true + break + } + } + if err := rows.Err(); err != nil { + return 0, cursor, err + } + rows.Close() + if !limited { + finish() + } + ids := make([]int, 0, len(packets)) + for _, packet := range packets { + if packet.tx.PayloadType != nil && *packet.tx.PayloadType == PayloadADVERT { + ids = append(ids, packet.tx.ID) + } + } + masks, err := s.db.advertEvidenceForIDs(ids) + if err != nil { + return 0, cursor, err + } + + s.mu.Lock() + defer s.mu.Unlock() + accepted := make([]*StoreTx, 0, len(packets)) + hopsSeen := make(map[string]bool) + changed := false + for _, packet := range packets { + tx := packet.tx + existing := s.byHash[tx.Hash] + observations := tx.Observations + charge := packet.bytes + if existing != nil { + observations = make([]*StoreObs, 0, len(tx.Observations)) + charge = 0 + for _, obs := range tx.Observations { + if !existing.hasObservation(obs.ObserverID, obs.PathJSON) { + observations = append(observations, obs) + charge += estimateStoreObsBytes(obs) + } + } + // Estimate the winning path before mutating the live pointer. Copy + // only the fields the estimator needs, never the decode sync.Once. + candidate := StoreTx{RawHex: existing.RawHex, Hash: existing.Hash, + DecodedJSON: existing.DecodedJSON, PathJSON: existing.PathJSON} + if (len(existing.Observations) == 0 && len(observations) > 0) || packet.hops > pathLen(existing.PathJSON) { + candidate.PathJSON = tx.PathJSON + candidate.pathParsed = false + } + charge += estimateStoreTxBytes(&candidate) - existing.accountedBytes + } + // Live observations can consume budget while the SQL cursor is open. + if s.maxMemoryMB > 0 && s.trackedBytes+charge > int64(s.maxMemoryMB)*1048576 { + limited = true + currentCursor = packet.cursor + break + } + if existing == nil { + s.byHash[tx.Hash], s.byTxID[tx.ID] = tx, tx + accepted = append(accepted, tx) + if tx.PayloadType != nil { + s.byPayloadType[*tx.PayloadType] = append(s.byPayloadType[*tx.PayloadType], tx) + } + s.trackAdvertPubkey(tx) + } else { + tx = existing + for _, obs := range observations { + tx.addObservation(obs) + if obs.Timestamp > tx.LatestSeen { + tx.LatestSeen = obs.Timestamp + } + } + pickBestObservation(tx) + // The preflight charge above includes this same path delta. + rechargeTx(tx) + } + changed = unionAdvertEvidence(tx, masks[tx.ID]) || changed || len(observations) > 0 + for _, obs := range observations { + s.byObsID[obs.ID] = obs + if obs.ObserverID != "" { + s.byObserver[obs.ObserverID] = append(s.byObserver[obs.ObserverID], obs) + } + if s.maxMemoryMB <= 0 && obs.ID > s.maxObsID { + s.maxObsID = obs.ID + } + } + if s.maxMemoryMB <= 0 && tx.ID > s.maxTxID { + s.maxTxID = tx.ID + } + s.indexByNode(tx) + pks := make([]string, 0, len(packet.resolved)) + for pk := range packet.resolved { + pks = append(pks, pk) + } + s.indexResolvedPathHops(tx, pks, hopsSeen) + for pk := range packet.nodes { + s.addToByNode(tx, pk) + } + s.totalObs += len(observations) + s.trackedBytes += charge + } + if len(accepted) > 0 { + s.packets = mergeChunkIntoPackets(accepted, s.packets) + } + if changed { + s.invalidateAdvertEvidence() + } + if limited { + s.startupMemoryLimited = true + // A partial timestamp group is not complete coverage. Use the next + // second as a conservative boundary, including last_seen's second + // precision when first_seen itself contains fractional seconds. + var omitted time.Time + if s.db.hasLastSeen { + unix, err := strconv.ParseInt(currentCursor.recency, 10, 64) + if err != nil { + return 0, cursor, fmt.Errorf("load cursor timestamp: %w", err) + } + omitted = time.Unix(unix, 0) + } else { + omitted, err = time.Parse(time.RFC3339Nano, currentCursor.recency) + if err != nil { + return 0, cursor, fmt.Errorf("load cursor timestamp: %w", err) + } + } + s.oldestLoaded = omitted.Add(time.Second).UTC().Format(time.RFC3339) + // Newest-first admission stops at the first omitted group, including + // timestamp ties and concurrent live-data budget growth. The known + // boundary preserves useful history without claiming coverage of a hole. + s.loadCapEdge = s.oldestLoaded + } + return seen, cursor, nil +} diff --git a/cmd/server/packet_load_merge_test.go b/cmd/server/packet_load_merge_test.go new file mode 100644 index 000000000..24def22a2 --- /dev/null +++ b/cmd/server/packet_load_merge_test.go @@ -0,0 +1,45 @@ +package main + +import ( + "fmt" + "testing" + "time" + "unsafe" +) + +// The bounded loader must use the same compact observation representation as +// live ingest and the unlimited loader, including both sides of the lazy-map +// threshold. A second window load merges no duplicate observations. +func TestPacketRangeUsesSharedObservationRepresentation(t *testing.T) { + for _, count := range []int{obsIndexThreshold - 1, obsIndexThreshold + 1} { + t.Run(fmt.Sprint(count), func(t *testing.T) { + db := startupMemoryFixture(t, 2, count, true) + s := NewPacketStore(db, &PacketStoreConfig{MaxMemoryMB: 4}) + for pass := 0; pass < 2; pass++ { + if _, err := s.loadPacketRange(time.Time{}, time.Time{}, 1, nil); err != nil { + t.Fatal(err) + } + if len(s.packets) != 2 || s.totalObs != 2*count { + t.Fatalf("pass %d loaded %d packets and %d observations", pass, len(s.packets), s.totalObs) + } + for _, tx := range s.packets { + if tx.ObservationCount != count || tx.UniqueObserverCount != count { + t.Fatalf("counts: %d/%d, want %d", tx.ObservationCount, tx.UniqueObserverCount, count) + } + if (tx.obsKeys != nil) != (count >= obsIndexThreshold) { + t.Fatal("bounded loader bypassed lazy dedup maps") + } + for _, obs := range tx.Observations { + if !tx.hasObservation(obs.ObserverID, obs.PathJSON) { + t.Fatal("loaded observation missing from dedup lookup") + } + } + } + } + a, b := s.packets[0].Observations[0], s.packets[1].Observations[0] + if a.ObserverName != b.ObserverName || unsafe.StringData(a.ObserverName) != unsafe.StringData(b.ObserverName) { + t.Fatal("bounded loader did not share repeated observer names") + } + }) + } +} diff --git a/cmd/server/packet_scope_name_test.go b/cmd/server/packet_scope_name_test.go index 27f6ae2c7..4150d0293 100644 --- a/cmd/server/packet_scope_name_test.go +++ b/cmd/server/packet_scope_name_test.go @@ -71,15 +71,15 @@ func TestPacketDetailExposesScopeName(t *testing.T) { now := time.Now().UTC().Format(time.RFC3339) // route_type 1 = FLOOD (never transport-scoped → NULL); 0 = TRANSPORT_FLOOD. if _, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type) - VALUES ('AABB', 'aaaaaaaaaaaaaaa1', ?, 1, 4)`, now); err != nil { + VALUES ('AABB', 'aaaaaaaaaaaaaaa1', $1, 1, 4)`, now); err != nil { t.Fatalf("insert unscoped: %v", err) } if _, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, scope_name) - VALUES ('AABB', 'aaaaaaaaaaaaaaa2', ?, 0, 4, '')`, now); err != nil { + VALUES ('AABB', 'aaaaaaaaaaaaaaa2', $1, 0, 4, '')`, now); err != nil { t.Fatalf("insert unknown-scope: %v", err) } if _, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, scope_name) - VALUES ('AABB', 'aaaaaaaaaaaaaaa3', ?, 0, 4, '#belgium')`, now); err != nil { + VALUES ('AABB', 'aaaaaaaaaaaaaaa3', $1, 0, 4, '#belgium')`, now); err != nil { t.Fatalf("insert matched-scope: %v", err) } @@ -169,15 +169,15 @@ func TestGroupedPacketsEndpointExposesScopeName(t *testing.T) { now := time.Now().UTC().Format(time.RFC3339) if _, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type) - VALUES ('AABB', 'bbbbbbbbbbbbbbb1', ?, 1, 4)`, now); err != nil { + VALUES ('AABB', 'bbbbbbbbbbbbbbb1', $1, 1, 4)`, now); err != nil { t.Fatalf("insert unscoped: %v", err) } if _, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, scope_name) - VALUES ('AABB', 'bbbbbbbbbbbbbbb2', ?, 0, 4, '')`, now); err != nil { + VALUES ('AABB', 'bbbbbbbbbbbbbbb2', $1, 0, 4, '')`, now); err != nil { t.Fatalf("insert unknown-scope: %v", err) } if _, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, scope_name) - VALUES ('AABB', 'bbbbbbbbbbbbbbb3', ?, 0, 4, '#belgium')`, now); err != nil { + VALUES ('AABB', 'bbbbbbbbbbbbbbb3', $1, 0, 4, '#belgium')`, now); err != nil { t.Fatalf("insert matched-scope: %v", err) } diff --git a/cmd/server/packets_order_test.go b/cmd/server/packets_order_test.go index 8d62efb99..853eaf803 100644 --- a/cmd/server/packets_order_test.go +++ b/cmd/server/packets_order_test.go @@ -32,11 +32,11 @@ func TestQueryPacketsOrdersByIngestID(t *testing.T) { bufferedFirstSeen := now.Add(-6 * time.Hour).Format(time.RFC3339) if _, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, payload_type) - VALUES ('AA', 'hashfresh00000001', ?, 4)`, freshFirstSeen); err != nil { + VALUES ('AA', 'hashfresh00000001', $1, 4)`, freshFirstSeen); err != nil { t.Fatal(err) } if _, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, payload_type) - VALUES ('BB', 'hashbuffered00002', ?, 4)`, bufferedFirstSeen); err != nil { + VALUES ('BB', 'hashbuffered00002', $1, 4)`, bufferedFirstSeen); err != nil { t.Fatal(err) } @@ -78,24 +78,24 @@ func TestQueryPacketsSinceFilterUsesFirstSeen(t *testing.T) { oldEpoch := now.Add(-6 * time.Hour).Unix() if _, err := db.conn.Exec(`INSERT INTO observers (id, name, last_seen, first_seen, packet_count) - VALUES ('obs1', 'Obs1', ?, ?, 1)`, recent, recent); err != nil { + VALUES ('obs1', 'Obs1', $1, $2, 1)`, recent, recent); err != nil { t.Fatal(err) } if _, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, payload_type) - VALUES ('AA', 'recentrx00000001', ?, 4)`, recent); err != nil { + VALUES ('AA', 'recentrx00000001', $1, 4)`, recent); err != nil { t.Fatal(err) } // Buffered upload — ingested SECOND, but rxTime is 6h ago. if _, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, payload_type) - VALUES ('BB', 'oldrxbuffered001', ?, 4)`, old); err != nil { + VALUES ('BB', 'oldrxbuffered001', $1, 4)`, old); err != nil { t.Fatal(err) } if _, err := db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 10, -90, '[]', ?)`, recentEpoch); err != nil { + VALUES (1, 1, 10, -90, '[]', $1)`, recentEpoch); err != nil { t.Fatal(err) } if _, err := db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (2, 1, 10, -90, '[]', ?)`, oldEpoch); err != nil { + VALUES (2, 1, 10, -90, '[]', $1)`, oldEpoch); err != nil { t.Fatal(err) } diff --git a/cmd/server/parity_test.go b/cmd/server/parity_test.go index 3abe15e70..e4ea2cf2b 100644 --- a/cmd/server/parity_test.go +++ b/cmd/server/parity_test.go @@ -1,8 +1,9 @@ package main // parity_test.go — Golden fixture shape tests. -// Validates that Go API responses match the shape of Node.js API responses. -// Shapes were captured from the production Node.js server and stored in +// Validates the public API shape contract captured from the original server. +// PostgreSQL diagnostics deliberately replace SQLite-only metrics; telemetry +// and account response contracts remain unchanged. Shapes are stored in // testdata/golden/shapes.json. import ( @@ -41,6 +42,12 @@ func loadShapes(t *testing.T) map[string]shapeSpec { if err := json.Unmarshal(data, &shapes); err != nil { t.Fatalf("cannot parse shapes.json: %v", err) } + if testBackendValue() == "sqlite" { + perf := shapes["perf"] + perf.Keys["sqlite"] = perf.Keys["postgres"] + delete(perf.Keys, "postgres") + shapes["perf"] = perf + } return shapes } @@ -200,7 +207,7 @@ func TestParityShapes(t *testing.T) { mismatches := validateShape(body, spec, ep.path) if len(mismatches) > 0 { - t.Errorf("Go %s has %d shape mismatches vs Node.js golden:\n %s", + t.Errorf("Go %s has %d shape mismatches vs golden contract:\n %s", ep.path, len(mismatches), strings.Join(mismatches, "\n ")) } }) @@ -231,7 +238,7 @@ func TestParityNodeDetail(t *testing.T) { mismatches := validateShape(body, spec, "/api/nodes/{pubkey}") if len(mismatches) > 0 { - t.Errorf("Go node detail has %d shape mismatches vs Node.js golden:\n %s", + t.Errorf("Go node detail has %d shape mismatches vs golden contract:\n %s", len(mismatches), strings.Join(mismatches, "\n ")) } } @@ -441,7 +448,7 @@ func TestParityWSMultiObserverGolden(t *testing.T) { now := time.Now().UTC().Format(time.RFC3339) if _, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('BEEF', 'goldenstarburst237', ?, 1, 4, '{"pubKey":"aabbccdd11223344","type":"ADVERT"}')`, now); err != nil { + VALUES ('BEEF', 'goldenstarburst237', $1, 1, 4, '{"pubKey":"aabbccdd11223344","type":"ADVERT"}')`, now); err != nil { t.Fatalf("insert tx failed: %v", err) } var txID int @@ -450,9 +457,9 @@ func TestParityWSMultiObserverGolden(t *testing.T) { } ts := time.Now().Unix() if _, err := db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (?, 1, 11.0, -88, '["p1"]', ?), - (?, 2, 9.0, -92, '["p1","p2"]', ?), - (?, 1, 7.0, -96, '["p1","p2","p3"]', ?)`, + VALUES ($1, 1, 11.0, -88, '["p1"]', $2), + ($3, 2, 9.0, -92, '["p1","p2"]', $4), + ($5, 1, 7.0, -96, '["p1","p2","p3"]', $6)`, txID, ts, txID, ts+1, txID, ts+2); err != nil { t.Fatalf("insert obs failed: %v", err) } diff --git a/cmd/server/pathhop_dedupe_2108_test.go b/cmd/server/pathhop_dedupe_2108_test.go index b77b2fe5c..c9c6d326a 100644 --- a/cmd/server/pathhop_dedupe_2108_test.go +++ b/cmd/server/pathhop_dedupe_2108_test.go @@ -62,11 +62,11 @@ func phdSeed(t testing.TB, db *DB, observers int) { t.Helper() ts := time.Now().UTC().Add(-time.Hour).Format(time.RFC3339) for i, pk := range phdRelays { - phdExec(t, db, `INSERT INTO nodes (public_key, name, role, last_seen, first_seen, advert_count) VALUES (?, ?, 'repeater', ?, '2026-01-01', 1)`, + phdExec(t, db, `INSERT INTO nodes (public_key, name, role, last_seen, first_seen, advert_count) VALUES ($1, $2, 'repeater', $3, '2026-01-01', 1)`, pk, fmt.Sprintf("phdRelay-%d", i), ts) } for i := 1; i <= observers; i++ { - phdExec(t, db, `INSERT INTO observers (rowid, id, name, iata) VALUES (?, ?, ?, 'TST')`, + phdExec(t, db, `INSERT INTO observers (rowid, id, name, iata) VALUES ($1, $2, $3, 'TST')`, i, fmt.Sprintf("phdObs-%02d", i), fmt.Sprintf("Observer %d", i)) } } @@ -82,7 +82,7 @@ func phdExec(t testing.TB, db *DB, q string, args ...any) { func phdInsertTx(t testing.TB, db *DB, i int, firstSeen time.Time) { t.Helper() phdExec(t, db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES (?, 'CAFE', ?, ?, 1, 2, '{"type":"TXT_MSG"}')`, i+1, fmt.Sprintf("phdTx-%03d", i), firstSeen.Format(time.RFC3339)) + VALUES ($1, 'CAFE', $2, $3, 1, 2, '{"type":"TXT_MSG"}')`, i+1, fmt.Sprintf("phdTx-%03d", i), firstSeen.Format(time.RFC3339)) } // phdInsertObs inserts one observation of transmission i through its path, @@ -102,7 +102,7 @@ func phdInsertObsVia(t testing.TB, db *DB, i int, path []int, obsIdx int, ts tim rp = phdResolvedJSON(path) } phdExec(t, db, `INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp, resolved_path) - VALUES (?, ?, 5, -90, ?, ?, ?)`, i+1, obsIdx, phdRawHopsJSON(path), ts.Unix(), rp) + VALUES ($1, $2, 5, -90, $3, $4, $5)`, i+1, obsIdx, phdRawHopsJSON(path), ts.Unix(), rp) } func phdMaxObsID(t testing.TB, db *DB) int { @@ -580,7 +580,7 @@ func TestPathHopConsumersIgnoreDuplicateEntries_2108(t *testing.T) { func TestMultiByteCapabilityIgnoresDuplicateEntries_2108(t *testing.T) { db := setupCapabilityTestDB(t) defer db.conn.Close() - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ($1, $2, $3, $4)", "aabbccdd11223344", "RepB", "repeater", recentTS(48)) store := NewPacketStore(db, nil) pt := 1 diff --git a/cmd/server/paths_anchor_bias_test.go b/cmd/server/paths_anchor_bias_test.go index 96d9b209e..3f3cadd9a 100644 --- a/cmd/server/paths_anchor_bias_test.go +++ b/cmd/server/paths_anchor_bias_test.go @@ -43,23 +43,23 @@ func TestHandleNodePaths_AnchorBiasInconsistency_Issue1278(t *testing.T) { nodeGPSPK := "c0ffeec700000000000000000000000000000000000000000000000000000001" if _, err := db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, 'NodeNoGPS', 'repeater', 0, 0, ?, '2026-01-01', 1)`, nodeNoGPSPK, recent); err != nil { + VALUES ($1, 'NodeNoGPS', 'repeater', 0, 0, $2, '2026-01-01', 1)`, nodeNoGPSPK, recent); err != nil { t.Fatalf("insert nodeNoGPS: %v", err) } if _, err := db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, 'NodeGPS', 'repeater', 37.5, -122.0, ?, '2026-01-01', 1)`, nodeGPSPK, recent); err != nil { + VALUES ($1, 'NodeGPS', 'repeater', 37.5, -122.0, $2, '2026-01-01', 1)`, nodeGPSPK, recent); err != nil { t.Fatalf("insert nodeGPS: %v", err) } if _, err := db.conn.Exec(`INSERT INTO transmissions (id, raw_hex, hash, first_seen) - VALUES (100, 'AA', 'hash_collision', ?)`, recent); err != nil { + VALUES (100, 'AA', 'hash_collision', $1)`, recent); err != nil { t.Fatalf("insert tx: %v", err) } // obs1: SHORTER path_json (single hop), resolved → nodeNoGPS. // (Without this row, the membership index wouldn't list nodeNoGPS at all // and the tx would be cleanly excluded — the bug needs the index hit.) if _, err := db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp, resolved_path) - VALUES (100, NULL, '["c0"]', ?, ?)`, recentEpoch, `["`+nodeNoGPSPK+`"]`); err != nil { + VALUES (100, NULL, '["c0"]', $1, $2)`, recentEpoch, `["`+nodeNoGPSPK+`"]`); err != nil { t.Fatalf("insert obs1: %v", err) } // obs2: LONGER path_json (two hops, first hop is what packets page shows @@ -67,7 +67,7 @@ func TestHandleNodePaths_AnchorBiasInconsistency_Issue1278(t *testing.T) { // canonical because it has the longer path_json. Its resolved_path // picks nodeGPS for "c0", NOT nodeNoGPS. if _, err := db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp, resolved_path) - VALUES (100, NULL, '["c0","ee"]', ?, ?)`, recentEpoch, `["`+nodeGPSPK+`","ee00000000000000000000000000000000000000000000000000000000000000"]`); err != nil { + VALUES (100, NULL, '["c0","ee"]', $1, $2)`, recentEpoch, `["`+nodeGPSPK+`","ee00000000000000000000000000000000000000000000000000000000000000"]`); err != nil { t.Fatalf("insert obs2: %v", err) } diff --git a/cmd/server/paths_batch_test.go b/cmd/server/paths_batch_test.go index 5009cbf8d..a6fb4cca1 100644 --- a/cmd/server/paths_batch_test.go +++ b/cmd/server/paths_batch_test.go @@ -47,7 +47,7 @@ func newPathsBatchStore(t *testing.T) (*PacketStore, map[int][]rpObs) { {11, 7, `["a","b","c"]`, `["cc33cc33","aa11aa11","dd44dd44"]`}, } for _, r := range rows { - if _, err := conn.Exec(`INSERT INTO observations (id, transmission_id, path_json, resolved_path) VALUES (?,?,?,?)`, + if _, err := conn.Exec(`INSERT INTO observations (id, transmission_id, path_json, resolved_path) VALUES ($1,$2,$3,$4)`, r.id, r.tx, r.path, r.rp); err != nil { t.Fatal(err) } @@ -102,7 +102,7 @@ func TestLoadCanonicalResolvedPathsBatchesQueries(t *testing.T) { const n = 2*resolvedPathBatchSize + 1 snapshots := make(map[int][]rpObs, n) for i := 1; i <= n; i++ { - conn.Exec(`INSERT INTO observations (id, transmission_id, path_json, resolved_path) VALUES (?,?,?,?)`, + conn.Exec(`INSERT INTO observations (id, transmission_id, path_json, resolved_path) VALUES ($1,$2,$3,$4)`, i, i, `["a"]`, `["aa11aa11"]`) snapshots[i] = []rpObs{{i, `["a"]`}} } @@ -132,11 +132,11 @@ func TestLoadCanonicalResolvedPathsBatchesQueries(t *testing.T) { // both A and B are index hits, only A is on the canonical path. func TestHandleNodePaths_CanonicalPathDecidesIndexHits(t *testing.T) { sc := setupCollisionScenario(t, false) - mustExec(t, sc.db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (43, 'BEEF', 'hash_2146', ?)`, sc.recent) + mustExec(t, sc.db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (43, 'BEEF', 'hash_2146', $1)`, sc.recent) mustExec(t, sc.db, `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp, resolved_path) - VALUES (43, NULL, '["c0","aa"]', ?, ?)`, sc.recentEpoch, `["`+sc.nodeAPK+`","aaaa0000aaaa0000"]`) + VALUES (43, NULL, '["c0","aa"]', $1, $2)`, sc.recentEpoch, `["`+sc.nodeAPK+`","aaaa0000aaaa0000"]`) mustExec(t, sc.db, `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp, resolved_path) - VALUES (43, NULL, '["c0"]', ?, ?)`, sc.recentEpoch-10, `["`+sc.nodeBPK+`"]`) + VALUES (43, NULL, '["c0"]', $1, $2)`, sc.recentEpoch-10, `["`+sc.nodeBPK+`"]`) sc.reloadStore(t) queries := 0 diff --git a/cmd/server/paths_hop_name_test.go b/cmd/server/paths_hop_name_test.go index 56d6408e1..a1f13780f 100644 --- a/cmd/server/paths_hop_name_test.go +++ b/cmd/server/paths_hop_name_test.go @@ -29,14 +29,14 @@ func TestHandleNodePaths_HopName_CanonicalPathShowsTarget_1144(t *testing.T) { siblingPK := "37bb000011112222" // has GPS — biased resolver picks this without fix mustExec(t, db, `INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, 'CJS SF Mission', 'repeater', 0, 0, ?, '2026-01-01', 1)`, targetPK, recent) + VALUES ($1, 'CJS SF Mission', 'repeater', 0, 0, $2, '2026-01-01', 1)`, targetPK, recent) mustExec(t, db, `INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, 'Templeton Hills', 'repeater', 35.5, -120.7, ?, '2026-01-01', 1)`, siblingPK, recent) + VALUES ($1, 'Templeton Hills', 'repeater', 35.5, -120.7, $2, '2026-01-01', 1)`, siblingPK, recent) // TX: resolved_path = [targetPK] → canonical path (Option A) → lookupNode(targetPK) - mustExec(t, db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (1, 'AA', 'hash1144', ?)`, recent) + mustExec(t, db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (1, 'AA', 'hash1144', $1)`, recent) mustExec(t, db, `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp, resolved_path) - VALUES (1, NULL, '["37"]', ?, ?)`, recentEpoch, `["`+targetPK+`"]`) + VALUES (1, NULL, '["37"]', $1, $2)`, recentEpoch, `["`+targetPK+`"]`) cfg := &Config{Port: 3000} hub := NewHub() diff --git a/cmd/server/paths_observations_race_test.go b/cmd/server/paths_observations_race_test.go index c4fbe840c..d491785c5 100644 --- a/cmd/server/paths_observations_race_test.go +++ b/cmd/server/paths_observations_race_test.go @@ -4,6 +4,7 @@ import ( "net/http/httptest" "sync" "testing" + "time" "github.com/gorilla/mux" ) @@ -13,6 +14,9 @@ import ( // ingest appends to under the write lock. Run with -race. func TestNodePathsDoesNotReadObservationsUnlocked(t *testing.T) { store := newResolvedPathStore(t) + if !store.WaitIndexesReady(5 * time.Second) { + t.Fatal("background indexes not ready within 5s") + } srv := NewServer(store.db, &Config{Port: 3000}, NewHub()) srv.store = store router := mux.NewRouter() @@ -26,6 +30,10 @@ func TestNodePathsDoesNotReadObservationsUnlocked(t *testing.T) { stop := make(chan struct{}) var wg sync.WaitGroup wg.Add(1) + defer func() { + close(stop) + wg.Wait() + }() go func() { // stands in for ingest adding an observation to the tx defer wg.Done() for i := 0; ; i++ { @@ -50,6 +58,4 @@ func TestNodePathsDoesNotReadObservationsUnlocked(t *testing.T) { t.Fatalf("paths: status %d: %s", w.Code, w.Body.String()) } } - close(stop) - wg.Wait() } diff --git a/cmd/server/paths_sort_test.go b/cmd/server/paths_sort_test.go index aad84c1f5..8dbf5c53a 100644 --- a/cmd/server/paths_sort_test.go +++ b/cmd/server/paths_sort_test.go @@ -39,7 +39,7 @@ func TestHandleNodePaths_SortByRecency_1145(t *testing.T) { // Only the target node needs to be in the nodes table. // Relay pubkeys appear only in resolved_path; they don't need a nodes row. mustExec(t, db, `INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, 'Target', 'repeater', 0, 0, '2026-05-01T00:00:00Z', '2026-01-01T00:00:00Z', 1)`, targetPK) + VALUES ($1, 'Target', 'repeater', 0, 0, '2026-05-01T00:00:00Z', '2026-01-01T00:00:00Z', 1)`, targetPK) // -- Path A (via relay1): 3 txs, last seen 2026-01-03 → group sig "relay1PK→targetPK" -- for txID, ts := range map[int]string{ @@ -47,17 +47,17 @@ func TestHandleNodePaths_SortByRecency_1145(t *testing.T) { 2: "2026-01-02T00:00:00Z", 3: "2026-01-03T00:00:00Z", } { - mustExec(t, db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (?, 'AA', ?, ?)`, + mustExec(t, db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES ($1, 'AA', $2, $3)`, txID, "hashA"+string(rune('0'+txID)), ts) mustExec(t, db, `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp, resolved_path) - VALUES (?, NULL, '["11", "aa"]', ?, ?)`, + VALUES ($1, NULL, '["11", "aa"]', $2, $3)`, txID, epoch(ts), `["`+relay1PK+`", "`+targetPK+`"]`) } // -- Path B (via relay2): 1 tx, last seen 2026-05-01 → group sig "relay2PK→targetPK" -- mustExec(t, db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (4, 'BB', 'hashB1', '2026-05-01T00:00:00Z')`) mustExec(t, db, `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp, resolved_path) - VALUES (4, NULL, '["22", "aa"]', ?, ?)`, + VALUES (4, NULL, '["22", "aa"]', $1, $2)`, epoch("2026-05-01T00:00:00Z"), `["`+relay2PK+`", "`+targetPK+`"]`) // -- Path C (direct — target is sole hop): 2 txs, last seen 2026-03-02 -- @@ -65,10 +65,10 @@ func TestHandleNodePaths_SortByRecency_1145(t *testing.T) { 5: "2026-03-01T00:00:00Z", 6: "2026-03-02T00:00:00Z", } { - mustExec(t, db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (?, 'CC', ?, ?)`, + mustExec(t, db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES ($1, 'CC', $2, $3)`, txID, "hashC"+string(rune('0'+txID)), ts) mustExec(t, db, `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp, resolved_path) - VALUES (?, NULL, '["aa"]', ?, ?)`, + VALUES ($1, NULL, '["aa"]', $2, $3)`, txID, epoch(ts), `["`+targetPK+`"]`) } @@ -130,7 +130,7 @@ func TestHandleNodePaths_SortCountTiebreaker_1145(t *testing.T) { } mustExec(t, db, `INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, 'Tgt', 'repeater', 0, 0, ?, '2026-01-01T00:00:00Z', 1)`, targetPK, sameTS) + VALUES ($1, 'Tgt', 'repeater', 0, 0, $2, '2026-01-01T00:00:00Z', 1)`, targetPK, sameTS) // Path X: 3 txs, all at sameTS → higher count for txID, ts := range map[int]string{ @@ -138,17 +138,17 @@ func TestHandleNodePaths_SortCountTiebreaker_1145(t *testing.T) { 11: "2026-04-15T11:30:00Z", 12: sameTS, } { - mustExec(t, db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (?, 'XX', ?, ?)`, + mustExec(t, db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES ($1, 'XX', $2, $3)`, txID, "hashX"+string(rune('0'+txID)), ts) mustExec(t, db, `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp, resolved_path) - VALUES (?, NULL, '["aa", "cc"]', ?, ?)`, + VALUES ($1, NULL, '["aa", "cc"]', $2, $3)`, txID, epoch(ts), `["`+relay1PK+`", "`+targetPK+`"]`) } // Path Y: 1 tx, at sameTS → lower count - mustExec(t, db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (20, 'YY', 'hashY1', ?)`, sameTS) + mustExec(t, db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (20, 'YY', 'hashY1', $1)`, sameTS) mustExec(t, db, `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp, resolved_path) - VALUES (20, NULL, '["bb", "cc"]', ?, ?)`, + VALUES (20, NULL, '["bb", "cc"]', $1, $2)`, epoch(sameTS), `["`+relay2PK+`", "`+targetPK+`"]`) cfg := &Config{Port: 3000} diff --git a/cmd/server/paths_through_collision_1352_test.go b/cmd/server/paths_through_collision_1352_test.go index 00545bde8..52bfaa7cb 100644 --- a/cmd/server/paths_through_collision_1352_test.go +++ b/cmd/server/paths_through_collision_1352_test.go @@ -67,11 +67,11 @@ func setupCollisionScenario(t *testing.T, withGPS bool) *collisionScenario { } mustExec(t, db, `INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, 'NodeA', 'repeater', ?, ?, ?, '2026-01-01', 1)`, sc.nodeAPK, aLat, aLon, recent) + VALUES ($1, 'NodeA', 'repeater', $2, $3, $4, '2026-01-01', 1)`, sc.nodeAPK, aLat, aLon, recent) mustExec(t, db, `INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, 'NodeB', 'repeater', ?, ?, ?, '2026-01-01', 1)`, sc.nodeBPK, bLat, bLon, recent) + VALUES ($1, 'NodeB', 'repeater', $2, $3, $4, '2026-01-01', 1)`, sc.nodeBPK, bLat, bLon, recent) mustExec(t, db, `INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, 'NodeC', 'repeater', ?, ?, ?, '2026-01-01', 1)`, sc.nodeCPK, cLat, cLon, recent) + VALUES ($1, 'NodeC', 'repeater', $2, $3, $4, '2026-01-01', 1)`, sc.nodeCPK, cLat, cLon, recent) cfg := &Config{Port: 3000} hub := NewHub() @@ -133,9 +133,9 @@ func (sc *collisionScenario) query(t *testing.T, pk string) NodePathsResponse { // must include the tx. paths-through-A and paths-through-C must exclude it. func TestHandleNodePaths_PrefixCollision_1352(t *testing.T) { sc := setupCollisionScenario(t, false /* only B has GPS */) - mustExec(t, sc.db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (42, 'DEAD', 'hash_1352', ?)`, sc.recent) + mustExec(t, sc.db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (42, 'DEAD', 'hash_1352', $1)`, sc.recent) mustExec(t, sc.db, `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp, resolved_path) - VALUES (42, NULL, '["c0"]', ?, ?)`, sc.recentEpoch, `["`+sc.nodeBPK+`"]`) + VALUES (42, NULL, '["c0"]', $1, $2)`, sc.recentEpoch, `["`+sc.nodeBPK+`"]`) sc.reloadStore(t) respA := sc.query(t, sc.nodeAPK) @@ -173,9 +173,9 @@ func TestHandleNodePaths_PrefixCollision_1352(t *testing.T) { // all three = 3. Fixed: ≤1, and we tighten further to ≤1 explicitly. func TestHandleNodePaths_PrefixCollision_1352_FallbackBranch(t *testing.T) { sc := setupCollisionScenario(t, true /* all three have GPS */) - mustExec(t, sc.db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (43, 'BEEF', 'hash_1352_fb', ?)`, sc.recent) + mustExec(t, sc.db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (43, 'BEEF', 'hash_1352_fb', $1)`, sc.recent) mustExec(t, sc.db, `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp, resolved_path) - VALUES (43, NULL, '["c0"]', ?, NULL)`, sc.recentEpoch) + VALUES (43, NULL, '["c0"]', $1, NULL)`, sc.recentEpoch) sc.reloadStore(t) a := sc.query(t, sc.nodeAPK).TotalTransmissions @@ -213,10 +213,10 @@ func TestHandleNodePaths_FallbackUniquePrefix_1352(t *testing.T) { pk := "abcdef0123456789" mustExec(t, db, `INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, 'UniqueNode', 'repeater', 37.78, -122.4, ?, '2026-01-01', 1)`, pk, recent) - mustExec(t, db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (44, 'CAFE', 'hash_1352_unique', ?)`, recent) + VALUES ($1, 'UniqueNode', 'repeater', 37.78, -122.4, $2, '2026-01-01', 1)`, pk, recent) + mustExec(t, db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (44, 'CAFE', 'hash_1352_unique', $1)`, recent) mustExec(t, db, `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp, resolved_path) - VALUES (44, NULL, '["ab"]', ?, NULL)`, recentEpoch) + VALUES (44, NULL, '["ab"]', $1, NULL)`, recentEpoch) cfg := &Config{Port: 3000} hub := NewHub() @@ -274,14 +274,14 @@ func TestHandleNodePaths_FallbackPreconfirmed_1352(t *testing.T) { // tx 50: best obs has NULL resolved_path (fallback branch). A SECOND // obs persists resolved_path = [B] which populates the byPathHop index // for B's full pubkey AND lets confirmedBySQL hit via INSTR. - mustExec(t, sc.db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (50, 'F00D', 'hash_1352_pre', ?)`, sc.recent) + mustExec(t, sc.db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (50, 'F00D', 'hash_1352_pre', $1)`, sc.recent) mustExec(t, sc.db, `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp, resolved_path) - VALUES (50, NULL, '["c0"]', ?, NULL)`, sc.recentEpoch) + VALUES (50, NULL, '["c0"]', $1, NULL)`, sc.recentEpoch) // Second observation (different observer) — same tx, persisted resolved_path = [B]. // This populates byPathHop[B] during Load(), so confirmedByFullKey is true // when paths-through-B is queried. mustExec(t, sc.db, `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp, resolved_path) - VALUES (50, 1, '["c0"]', ?, ?)`, sc.recentEpoch+1, `["`+sc.nodeBPK+`"]`) + VALUES (50, 1, '["c0"]', $1, $2)`, sc.recentEpoch+1, `["`+sc.nodeBPK+`"]`) sc.reloadStore(t) respA := sc.query(t, sc.nodeAPK) @@ -324,9 +324,9 @@ func TestHandleNodePaths_FallbackPreconfirmed_1352(t *testing.T) { func TestHandleNodePaths_FallbackUnresolvableHop_1352(t *testing.T) { sc := setupCollisionScenario(t, false /* only B has GPS */) - mustExec(t, sc.db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (60, 'FEED', 'hash_1352_unres', ?)`, sc.recent) + mustExec(t, sc.db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (60, 'FEED', 'hash_1352_unres', $1)`, sc.recent) mustExec(t, sc.db, `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp, resolved_path) - VALUES (60, NULL, '["c0"]', ?, NULL)`, sc.recentEpoch) + VALUES (60, NULL, '["c0"]', $1, NULL)`, sc.recentEpoch) sc.reloadStore(t) // Query A (no GPS): biased resolver in the fallback branch picks B via diff --git a/cmd/server/paths_through_test.go b/cmd/server/paths_through_test.go index ee049b850..43c4aadbf 100644 --- a/cmd/server/paths_through_test.go +++ b/cmd/server/paths_through_test.go @@ -28,19 +28,19 @@ func TestHandleNodePaths_PrefixCollisionExclusion(t *testing.T) { nodeBPK := "7aff2222ccccdddd" // same "7a" prefix, has GPS so resolveHop("7a") picks B db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, 'NodeA', 'repeater', 0, 0, ?, '2026-01-01', 1)`, nodeAPK, recent) + VALUES ($1, 'NodeA', 'repeater', 0, 0, $2, '2026-01-01', 1)`, nodeAPK, recent) db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, 'NodeB', 'repeater', 37.5, -122.0, ?, '2026-01-01', 1)`, nodeBPK, recent) + VALUES ($1, 'NodeB', 'repeater', 37.5, -122.0, $2, '2026-01-01', 1)`, nodeBPK, recent) // tx1: no resolved_path — should be excluded by hop-level check - db.conn.Exec(`INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (10, 'AA', 'hash_fp', ?)`, recent) + db.conn.Exec(`INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (10, 'AA', 'hash_fp', $1)`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp, resolved_path) - VALUES (10, NULL, '["7a"]', ?, NULL)`, recentEpoch) + VALUES (10, NULL, '["7a"]', $1, NULL)`, recentEpoch) // tx2: resolved_path confirms nodeA — must be included - db.conn.Exec(`INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (11, 'BB', 'hash_tp', ?)`, recent) + db.conn.Exec(`INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (11, 'BB', 'hash_tp', $1)`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp, resolved_path) - VALUES (11, NULL, '["7a"]', ?, ?)`, recentEpoch, `["`+nodeAPK+`"]`) + VALUES (11, NULL, '["7a"]', $1, $2)`, recentEpoch, `["`+nodeAPK+`"]`) cfg := &Config{Port: 3000} hub := NewHub() diff --git a/cmd/server/perf_io.go b/cmd/server/perf_io.go index dcf1372eb..a9acc7c77 100644 --- a/cmd/server/perf_io.go +++ b/cmd/server/perf_io.go @@ -5,10 +5,12 @@ import ( "encoding/json" "net/http" "os" + "path/filepath" "sync" "sync/atomic" "time" + "github.com/meshcore-analyzer/dbconfig" "github.com/meshcore-analyzer/perfio" ) @@ -36,14 +38,27 @@ type PerfIOResponse struct { // drift between the publisher (ingestor) and the consumer (server) (#1167). type PerfIOSample = perfio.Sample -// PerfSqliteResponse holds SQLite-specific perf metrics. -type PerfSqliteResponse struct { - WalSizeMB float64 `json:"walSizeMB"` - WalSize int64 `json:"walSize"` - PageCount int64 `json:"pageCount"` - PageSize int64 `json:"pageSize"` - CacheSize int64 `json:"cacheSize"` - CacheHitRate float64 `json:"cacheHitRate"` +// PerfPostgresResponse combines sampled database metrics with live pool usage. +type PerfPostgresResponse struct { + WalSize *int64 `json:"walSize,omitempty"` + WalSizeMB *float64 `json:"walSizeMB,omitempty"` + PageCount *int64 `json:"pageCount,omitempty"` + PageSize *int64 `json:"pageSize,omitempty"` + CacheSize *int64 `json:"cacheSize,omitempty"` + JournalMode *string `json:"journalMode,omitempty"` + PlannerStats *bool `json:"plannerStats,omitempty"` + Engine string `json:"engine"` + DatabaseBytes *int64 `json:"databaseBytes"` + OpenConnections int `json:"openConnections"` + InUseConnections int `json:"inUseConnections"` + IdleConnections int `json:"idleConnections"` + ConnectionWaitCount int64 `json:"connectionWaitCount"` + ConnectionWaitMs float64 `json:"connectionWaitMs"` + CacheHitRate *float64 `json:"cacheHitRate"` + SampledAt string `json:"sampledAt,omitempty"` + SampleIntervalSeconds int `json:"sampleIntervalSeconds"` + Stale bool `json:"stale"` + Error string `json:"error,omitempty"` } // procIOSample is a snapshot of /proc/self/io counters. @@ -246,36 +261,31 @@ func readIngestorIOSample() *PerfIOSample { return st.ProcIO } -// handlePerfSqlite returns SQLite WAL size + cache hit-rate stats. -func (s *Server) handlePerfSqlite(w http.ResponseWriter, r *http.Request) { - resp := PerfSqliteResponse{} +// handlePerfPostgres reports read-only, database-scoped PostgreSQL metrics. +func (s *Server) handlePerfPostgres(w http.ResponseWriter, r *http.Request) { + resp := PerfPostgresResponse{Engine: "postgresql", SampleIntervalSeconds: int(postgresStatsTTL / time.Second), Stale: true, Error: "database diagnostics unavailable"} if s.db != nil && s.db.conn != nil { - var pageCount, pageSize int64 - _ = s.db.conn.QueryRow("PRAGMA page_count").Scan(&pageCount) - _ = s.db.conn.QueryRow("PRAGMA page_size").Scan(&pageSize) - var cacheSize int64 - _ = s.db.conn.QueryRow("PRAGMA cache_size").Scan(&cacheSize) - resp.PageCount = pageCount - resp.PageSize = pageSize - resp.CacheSize = cacheSize - - // Cache hit rate: derived from PacketStore cache (rw_cache). We don't - // have a direct SQLite cache counter through the driver, so we - // surface the closest available proxy — the in-process row cache. - if s.store != nil { - cs := s.store.GetCacheStatsTyped() - total := cs.Hits + cs.Misses - if total > 0 { - resp.CacheHitRate = float64(cs.Hits) / float64(total) - } + if s.db.Backend() == dbconfig.SQLite { + resp.Engine = "sqlite" + } + sample, err := s.db.postgresStats(r.Context()) + if sample.SQLite != nil { + native := sample.SQLite + walMB := float64(native.WalSize) / 1048576 + resp.WalSize, resp.WalSizeMB = &native.WalSize, &walMB + resp.PageCount, resp.PageSize, resp.CacheSize = &native.PageCount, &native.PageSize, &native.CacheSize + resp.JournalMode, resp.PlannerStats = &native.JournalMode, &native.PlannerStats } - if s.db.path != "" && s.db.path != ":memory:" { - if info, err := os.Stat(s.db.path + "-wal"); err == nil { - resp.WalSize = info.Size() - resp.WalSizeMB = float64(info.Size()) / 1048576 - } + resp.CacheHitRate = sample.CacheHitRate + resp.SampledAt, resp.Stale = postgresSampleStamp(sample), err != nil + if !sample.SampledAt.IsZero() { + resp.DatabaseBytes, resp.Error = &sample.DatabaseBytes, "" } + pool := s.db.conn.Stats() + resp.OpenConnections, resp.InUseConnections, resp.IdleConnections = pool.OpenConnections, pool.InUse, pool.Idle + resp.ConnectionWaitCount = pool.WaitCount + resp.ConnectionWaitMs = float64(pool.WaitDuration) / float64(time.Millisecond) } writeJSON(w, resp) } @@ -297,7 +307,7 @@ type IngestorStats struct { // ProcIO is the ingestor's own /proc/self/io rates (since its previous // sample). Optional — older ingestor builds don't publish this. See #1120. ProcIO *PerfIOSample `json:"procIO,omitempty"` - // WriterPerf is the per-component SQLite writer-lock latency + // WriterPerf is the per-component serialized-writer latency // snapshot (#1340). Optional — older ingestor builds don't // publish this. Surfaced under .writer_perf by // handlePerfWriteSources. @@ -340,7 +350,7 @@ func IngestorStatsPath() string { if p := os.Getenv("CORESCOPE_INGESTOR_STATS"); p != "" { return p } - return "/tmp/corescope-ingestor-stats.json" + return filepath.Join(runtimeStateDir, "ingestor-stats.json") } // readIngestorSourceLiveness returns the per-source receipt/write-path @@ -482,7 +492,7 @@ func (s *Server) handlePerfWriteSources(w http.ResponseWriter, r *http.Request) } out["sources"] = sources out["sampleAt"] = st.SampledAt - // Surface per-component SQLite writer-lock latency histograms + // Surface per-component serialized-writer latency histograms // (#1340) under .writer_perf so operators can see when a // component (e.g. neighbor_builder) is starving the writer. // Empty map when the ingestor is too old to publish this field. diff --git a/cmd/server/perf_io_test.go b/cmd/server/perf_io_test.go index ecb4f884f..ed944f1c0 100644 --- a/cmd/server/perf_io_test.go +++ b/cmd/server/perf_io_test.go @@ -9,6 +9,48 @@ import ( "testing" ) +func TestPerfPostgresCacheIsNotPacketStoreCache(t *testing.T) { + srv, _ := setupTestServer(t) + get := func() map[string]any { + w := httptest.NewRecorder() + srv.handlePerfPostgres(w, httptest.NewRequest("GET", "/api/perf/postgres", nil)) + var body map[string]any + if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil { + t.Fatal(err) + } + return body + } + srv.store.cacheMu.Lock() + srv.store.cacheHits, srv.store.cacheMisses = 0, 1 + srv.store.cacheMu.Unlock() + first := get() + srv.store.cacheMu.Lock() + srv.store.cacheHits, srv.store.cacheMisses = 1, 0 + srv.store.cacheMu.Unlock() + second := get() + if first["cacheHitRate"] != second["cacheHitRate"] { + t.Fatalf("database cache metric changed with the PacketStore cache: %v -> %v", first["cacheHitRate"], second["cacheHitRate"]) + } + if first["sampledAt"] == nil || first["sampleIntervalSeconds"] != float64(30) { + t.Fatalf("diagnostic sample freshness is not disclosed: %v", first) + } +} + +func TestPerfPostgresUnavailableCacheIsNull(t *testing.T) { + w := httptest.NewRecorder() + (&Server{}).handlePerfPostgres(w, httptest.NewRequest("GET", "/api/perf/postgres", nil)) + var body map[string]any + if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil { + t.Fatal(err) + } + if v, present := body["cacheHitRate"]; !present || v != nil { + t.Fatalf("unavailable database cache metric must be null, got %v", v) + } + if v, present := body["databaseBytes"]; !present || v != nil { + t.Fatalf("unavailable database size must be null, got %v", v) + } +} + func TestPerfIOEndpoint_ReturnsValidJSON(t *testing.T) { _, router := setupTestServer(t) @@ -55,10 +97,10 @@ func TestPerfIOEndpoint_ReturnsValidJSON(t *testing.T) { } } -func TestPerfSqliteEndpoint_ReturnsValidJSON(t *testing.T) { +func TestPerfPostgresEndpoint_ReturnsValidJSON(t *testing.T) { _, router := setupTestServer(t) - req := httptest.NewRequest("GET", "/api/perf/sqlite", nil) + req := httptest.NewRequest("GET", "/api/perf/postgres", nil) w := httptest.NewRecorder() router.ServeHTTP(w, req) @@ -69,14 +111,14 @@ func TestPerfSqliteEndpoint_ReturnsValidJSON(t *testing.T) { if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil { t.Fatalf("invalid JSON: %v", err) } - for _, field := range []string{"walSize", "pageCount", "pageSize", "cacheHitRate"} { + for _, field := range []string{"engine", "databaseBytes", "openConnections", "cacheHitRate"} { if _, ok := body[field]; !ok { t.Errorf("missing field %q", field) } } - // pageSize must be > 0 for any open SQLite DB - if v, ok := body["pageSize"].(float64); !ok || v <= 0 { - t.Errorf("expected pageSize > 0, got %v", body["pageSize"]) + // A PostgreSQL database has a positive physical size. + if v, ok := body["databaseBytes"].(float64); !ok || v <= 0 { + t.Errorf("expected databaseBytes > 0, got %v", body["databaseBytes"]) } } diff --git a/cmd/server/poller_timestamp_test.go b/cmd/server/poller_timestamp_test.go new file mode 100644 index 000000000..95bae7cc0 --- /dev/null +++ b/cmd/server/poller_timestamp_test.go @@ -0,0 +1,23 @@ +package main + +import "testing" + +func TestGetNewTransmissionsSinceIncludesTimestamp(t *testing.T) { + db := setupTestDB(t) + defer db.Close() + const firstSeen = "2026-01-16T10:00:00Z" + if _, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen) VALUES ('AABB', 'poller-timestamp', $1)`, firstSeen); err != nil { + t.Fatal(err) + } + + packets, err := db.GetNewTransmissionsSince(0, 1) + if err != nil || len(packets) != 1 { + t.Fatalf("poller query returned %d packets: %v", len(packets), err) + } + if got := packets[0]["timestamp"]; got != firstSeen { + t.Errorf("fallback packet timestamp=%v, want first_seen %q", got, firstSeen) + } + if got := packets[0]["first_seen"]; got != firstSeen { + t.Errorf("first_seen=%v, want %q", got, firstSeen) + } +} diff --git a/cmd/server/postgres_fixture_test.go b/cmd/server/postgres_fixture_test.go new file mode 100644 index 000000000..46ae13d55 --- /dev/null +++ b/cmd/server/postgres_fixture_test.go @@ -0,0 +1,159 @@ +package main + +import ( + "context" + "database/sql" + "net/url" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/stdlib" + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/pgutil" + "github.com/meshcore-analyzer/pgutil/pgtest" +) + +// copyTestRows streams a bounded synthetic fixture through native PostgreSQL +// COPY. Each COPY remains an atomic durable statement; no test durability +// settings are changed and there is no INSERT round trip per fixture row. +func copyTestRows(t testing.TB, db *sql.DB, table string, columns []string, n int, row func(int) []any) { + t.Helper() + if testBackend(t) == dbconfig.SQLite { + tx, err := db.Begin() + if err != nil { + t.Fatal(err) + } + defer tx.Rollback() + slots := make([]string, len(columns)) + for i := range slots { + slots[i] = dbconfig.SQLite.Parameter(i + 1) + } + stmt, err := tx.Prepare("INSERT INTO " + table + "(" + strings.Join(columns, ",") + ") VALUES(" + strings.Join(slots, ",") + ")") + if err != nil { + t.Fatal(err) + } + defer stmt.Close() + for i := 0; i < n; i++ { + if _, err = stmt.Exec(row(i)...); err != nil { + t.Fatal(err) + } + } + if err = tx.Commit(); err != nil { + t.Fatal(err) + } + return + } + ctx := context.Background() + conn, err := db.Conn(ctx) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + err = conn.Raw(func(driver any) error { + count, err := driver.(*stdlib.Conn).Conn().CopyFrom(ctx, pgx.Identifier{table}, columns, + pgx.CopyFromSlice(n, func(i int) ([]any, error) { return row(i), nil })) + if err == nil && count != int64(n) { + t.Errorf("copied %d/%d rows into %s", count, n, table) + } + return err + }) + if err != nil { + t.Fatalf("seed %s: %v", table, err) + } +} + +func postgresTestDSN(t testing.TB) string { + t.Helper() + if testBackend(t) == dbconfig.SQLite { + return filepath.Join(t.TempDir(), "fixture.db") + } + return pgtest.NewSchema(t) +} + +func openFixtureSQL(dsn string) (*sql.DB, error) { + if strings.HasPrefix(dsn, "postgres://") || strings.HasPrefix(dsn, "postgresql://") { + return pgutil.Open(dsn, false) + } + uri, err := dbconfig.SQLiteURI(dsn, url.Values{"_journal_mode": {"WAL"}, "_synchronous": {"FULL"}, "_busy_timeout": {"5000"}}) + if err != nil { + return nil, err + } + return sql.Open("sqlite3", uri) +} + +func openPostgresTestDB(t testing.TB) (*sql.DB, error) { + t.Helper() + db, err := openFixtureSQL(postgresTestDSN(t)) + if err == nil { + t.Cleanup(func() { db.Close() }) + } + return db, err +} + +func openFixtureReader(t testing.TB, dsn string) (*DB, error) { + t.Helper() + if testBackend(t) == dbconfig.SQLite { + return OpenDB(dsn) + } + return OpenDB(pgtest.ReadOnly(t, dsn)) +} + +func restorePostgresTestBackup(t testing.TB, path string) *sql.DB { + if testBackend(t) == dbconfig.SQLite { + db, err := openFixtureSQL(path) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { db.Close() }) + return db + } + t.Helper() + dsn := pgtest.NewDatabase(t) + config, err := pgutil.ParseConfig(dsn) + if err != nil { + t.Fatal(err) + } + env, err := pgutil.CommandEnv(dsn) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + cmd := exec.CommandContext(ctx, "pg_restore", "--exit-on-error", "--no-owner", "--no-privileges", "--dbname="+config.Database, path) + cmd.Env = env + if err := cmd.Run(); err != nil { + t.Fatal("restore native PostgreSQL backup failed") + } + db, err := openFixtureSQL(dsn) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { db.Close() }) + return db +} + +func testDatabaseDSN(t testing.TB) string { + t.Helper() + if testBackend(t) == dbconfig.SQLite { + return filepath.Join(t.TempDir(), "fixture.db") + } + return pgtest.NewDatabase(t) +} +func postgresOnly(t testing.TB) { + t.Helper() + if testBackend(t) != dbconfig.Postgres { + t.Skip("PostgreSQL matrix not selected") + } +} +func applyTestSchema(t testing.TB, db *sql.DB) error { + t.Helper() + if testBackend(t) == dbconfig.SQLite { + return dbschema.ApplySQLite(db, nil) + } + return dbschema.ApplyPostgres(db, nil) +} diff --git a/cmd/server/postgres_stats.go b/cmd/server/postgres_stats.go new file mode 100644 index 000000000..3d19b7b36 --- /dev/null +++ b/cmd/server/postgres_stats.go @@ -0,0 +1,119 @@ +package main + +import ( + "context" + "errors" + "github.com/meshcore-analyzer/dbconfig" + "os" + "time" +) + +const postgresStatsTTL = 30 * time.Second +const postgresBlockCacheRateSQL = `blks_hit::double precision / NULLIF(blks_hit::double precision + blks_read, 0)` + +// Storage traversal and full row counts are diagnostic samples, independent of +// ingest and PacketStore invalidation. Keep one bounded immutable sample per DB. +type sqliteDatabaseSample struct { + WalSize int64 + PageCount, PageSize, CacheSize, FreelistCount int64 + JournalMode string + PlannerStats bool +} + +type postgresDatabaseSample struct { + SQLite *sqliteDatabaseSample + DatabaseBytes int64 + CacheHitRate *float64 + Rows DatabaseRowCounts + SampledAt time.Time +} + +func (db *DB) cachedPostgresStats() (postgresDatabaseSample, bool) { + db.pgStatsMu.Lock() + sample, valid := db.pgStatsCache, time.Now().Before(db.pgStatsExpires) + db.pgStatsMu.Unlock() + return sample, valid +} + +func (db *DB) postgresStats(ctx context.Context) (postgresDatabaseSample, error) { + cached, valid := db.cachedPostgresStats() + if valid { + return cached, nil + } + if db.pgStatsMissHook != nil { + db.pgStatsMissHook() + } + flight := db.pgStatsSF.DoChan("database", func() (any, error) { + if sample, valid := db.cachedPostgresStats(); valid { + return sample, nil + } + if db.pgStatsQueryHook != nil { + db.pgStatsQueryHook() + } + // One canceled dashboard request must not cancel the shared sample. + // The work itself is bounded, and no cache mutex is held during SQL. + queryCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + var sample postgresDatabaseSample + + var err error + if db.Backend() == dbconfig.SQLite { + native := &sqliteDatabaseSample{} + err = db.conn.QueryRowContext(queryCtx, `SELECT + (SELECT page_count FROM pragma_page_count),(SELECT page_size FROM pragma_page_size), + (SELECT cache_size FROM pragma_cache_size),(SELECT freelist_count FROM pragma_freelist_count), + (SELECT journal_mode FROM pragma_journal_mode),EXISTS(SELECT 1 FROM sqlite_master WHERE name='sqlite_stat1'), + (SELECT COUNT(*) FROM transmissions),(SELECT COUNT(*) FROM observations),(SELECT COUNT(*) FROM nodes),(SELECT COUNT(*) FROM observers)`).Scan( + &native.PageCount, &native.PageSize, &native.CacheSize, &native.FreelistCount, &native.JournalMode, &native.PlannerStats, + &sample.Rows.Transmissions, &sample.Rows.Observations, &sample.Rows.Nodes, &sample.Rows.Observers) + if err == nil { + var info os.FileInfo + info, err = os.Stat(db.path) + if err == nil { + sample.DatabaseBytes = info.Size() + } + } + if err == nil { + if info, statErr := os.Stat(db.path + "-wal"); statErr == nil { + native.WalSize = info.Size() + } else if !errors.Is(statErr, os.ErrNotExist) { + err = statErr + } + } + sample.SQLite = native + } else { + err = db.conn.QueryRowContext(queryCtx, `SELECT pg_database_size(current_database()), `+postgresBlockCacheRateSQL+`, + (SELECT COUNT(*) FROM transmissions), (SELECT COUNT(*) FROM observations), + (SELECT COUNT(*) FROM nodes), (SELECT COUNT(*) FROM observers) + FROM pg_catalog.pg_stat_database WHERE datname=current_database()`).Scan( + &sample.DatabaseBytes, &sample.CacheHitRate, &sample.Rows.Transmissions, + &sample.Rows.Observations, &sample.Rows.Nodes, &sample.Rows.Observers) + } + if err != nil { + return nil, err + } + sample.SampledAt = time.Now() + db.pgStatsMu.Lock() + db.pgStatsCache, db.pgStatsExpires = sample, sample.SampledAt.Add(postgresStatsTTL) + db.pgStatsMu.Unlock() + return sample, nil + }) + select { + case result := <-flight: + if result.Err == nil { + return result.Val.(postgresDatabaseSample), nil + } + cached.CacheHitRate = nil + return cached, result.Err + case <-ctx.Done(): + cached.CacheHitRate = nil + return cached, ctx.Err() + } +} + +func postgresSampleStamp(sample postgresDatabaseSample) string { + if sample.SampledAt.IsZero() { + return "" + } + return sample.SampledAt.UTC().Format(time.RFC3339Nano) +} diff --git a/cmd/server/postgres_stats_test.go b/cmd/server/postgres_stats_test.go new file mode 100644 index 000000000..428df39a0 --- /dev/null +++ b/cmd/server/postgres_stats_test.go @@ -0,0 +1,187 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "math" + "net/http/httptest" + "sync" + "sync/atomic" + "testing" + "time" +) + +func TestPostgresInitialDiagnosticFailureIsExplicit(t *testing.T) { + db := setupTestDB(t) + db.conn.Close() + bytes, err := json.Marshal(db.GetDBSizeStatsTyped()) + if err != nil { + t.Fatal(err) + } + var body map[string]any + if err := json.Unmarshal(bytes, &body); err != nil { + t.Fatal(err) + } + if body["error"] == nil || body["error"] == "" || body["rows"] != nil || body["dbSizeMB"] != nil { + t.Fatalf("unknown database size/counts must not become zeros: %s", bytes) + } + w := httptest.NewRecorder() + (&Server{db: db}).handlePerfPostgres(w, httptest.NewRequest("GET", "/api/perf/postgres", nil)) + if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil { + t.Fatal(err) + } + if body["error"] == nil || body["databaseBytes"] != nil || body["cacheHitRate"] != nil { + t.Fatalf("unknown database measurements must be null: %s", w.Body.String()) + } +} + +func TestPostgresBlockCacheRatioNullAndZero(t *testing.T) { + postgresOnly(t) + db := setupTestDB(t) + nineTenths, zero, one := 0.9, 0.0, 1.0 + for _, tc := range []struct { + name string + hits, reads any + want *float64 + }{ + {"hits", 9, 1, &nineTenths}, {"misses", 0, 9, &zero}, + {"all cached", 9, 0, &one}, {"no reads", 0, 0, nil}, + {"unavailable", nil, 9, nil}, + } { + t.Run(tc.name, func(t *testing.T) { + var got *float64 + err := db.conn.QueryRow(`SELECT `+postgresBlockCacheRateSQL+` FROM (VALUES ($1::bigint,$2::bigint)) AS counters(blks_hit,blks_read)`, tc.hits, tc.reads).Scan(&got) + if err != nil { + t.Fatal(err) + } + if (got == nil) != (tc.want == nil) { + t.Fatalf("ratio = %v, want %v", got, tc.want) + } + if got != nil && math.Abs(*got-*tc.want) > 1e-12 { + t.Fatalf("ratio = %g, want %g", *got, *tc.want) + } + }) + } +} + +func TestPostgresDiagnosticRequestsCoalesce(t *testing.T) { + db := setupTestDB(t) + seedTestData(t, db) + const requests = 12 + misses := make(chan struct{}, requests) + release := make(chan struct{}) + var queries atomic.Int64 + db.pgStatsMissHook = func() { misses <- struct{}{} } + db.pgStatsQueryHook = func() { queries.Add(1); <-release } + var wg sync.WaitGroup + for i := 0; i < requests; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + if i%2 == 0 { + if got := db.GetDBSizeStatsTyped(); got.Rows.Transmissions != 3 { + t.Errorf("sample rows = %d", got.Rows.Transmissions) + } + } else { + w := httptest.NewRecorder() + (&Server{db: db}).handlePerfPostgres(w, httptest.NewRequest("GET", "/api/perf/postgres", nil)) + if w.Code != 200 { + t.Errorf("status = %d", w.Code) + } + } + }(i) + } + // Every request must have observed the empty cache before the first query + // completes. This proves coalescing without sleeps or timing assumptions. + for i := 0; i < requests; i++ { + <-misses + } + close(release) + wg.Wait() + if got := queries.Load(); got != 1 { + t.Fatalf("expensive sampling queries = %d, want 1", got) + } +} + +func TestPostgresDiagnosticTTLDoesNotFollowIngestInvalidation(t *testing.T) { + db := setupTestDB(t) + seedTestData(t, db) + store := NewPacketStore(db, nil) + if err := store.Load(); err != nil { + t.Fatal(err) + } + queries := 0 + db.pgStatsQueryHook = func() { + queries++ + if !db.pgStatsMu.TryLock() { + t.Error("expensive sampling ran under the cache lock") + } else { + db.pgStatsMu.Unlock() + } + } + first := db.GetDBSizeStatsTyped() + if first.Rows.Transmissions != 3 || first.SampledAt == "" || first.SampleIntervalSeconds != 30 { + t.Fatalf("initial sample = %+v", first) + } + if _, err := db.conn.Exec(`INSERT INTO transmissions(raw_hex,hash,first_seen,payload_type,decoded_json) VALUES('AA','new-sample-row',$1,5,'{}')`, time.Now().UTC().Format(time.RFC3339)); err != nil { + t.Fatal(err) + } + store.IngestNewFromDB(3, 100) + if got := db.GetDBSizeStatsTyped(); got.Rows.Transmissions != 3 || queries != 1 { + t.Fatalf("ingest invalidated diagnostic sample: rows=%d queries=%d", got.Rows.Transmissions, queries) + } + db.pgStatsMu.Lock() + db.pgStatsExpires = time.Time{} + db.pgStatsMu.Unlock() + if got := db.GetDBSizeStatsTyped(); got.Rows.Transmissions != 4 || queries != 2 { + t.Fatalf("expiry did not refresh: rows=%d queries=%d", got.Rows.Transmissions, queries) + } +} + +func TestPostgresDiagnosticFailureMarksOldSampleStale(t *testing.T) { + db := setupTestDB(t) + seedTestData(t, db) + first, err := db.postgresStats(context.Background()) + if err != nil { + t.Fatal(err) + } + db.pgStatsMu.Lock() + db.pgStatsExpires = time.Time{} + db.pgStatsMu.Unlock() + db.conn.Close() + got := db.GetDBSizeStatsTyped() + if !got.Stale || got.SampledAt != first.SampledAt.UTC().Format(time.RFC3339Nano) || got.Rows.Transmissions != 3 { + t.Fatalf("failed refresh did not disclose stale sample: %+v", got) + } +} + +func TestPostgresDiagnosticCanceledCallerKeepsSharedSample(t *testing.T) { + db := setupTestDB(t) + seedTestData(t, db) + misses, entered, release := make(chan struct{}, 2), make(chan struct{}), make(chan struct{}) + var once sync.Once + var queries atomic.Int64 + db.pgStatsMissHook = func() { misses <- struct{}{} } + db.pgStatsQueryHook = func() { queries.Add(1); once.Do(func() { close(entered) }); <-release } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + first := make(chan error, 1) + go func() { _, err := db.postgresStats(ctx); first <- err }() + <-misses + <-entered + cancel() + if err := <-first; !errors.Is(err, context.Canceled) { + t.Fatalf("caller cancellation: %v", err) + } + second := make(chan error, 1) + go func() { _, err := db.postgresStats(context.Background()); second <- err }() + <-misses + close(release) + if err := <-second; err != nil { + t.Fatalf("shared sample was canceled: %v", err) + } + if queries.Load() != 1 { + t.Fatalf("queries = %d, want 1", queries.Load()) + } +} diff --git a/cmd/server/postgres_test.go b/cmd/server/postgres_test.go new file mode 100644 index 000000000..5d671f783 --- /dev/null +++ b/cmd/server/postgres_test.go @@ -0,0 +1,260 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "path/filepath" + "strings" + "testing" + + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/pgutil" + "github.com/meshcore-analyzer/pgutil/pgtest" +) + +func TestPostgresCursorStopsAtLimitWithoutSorting(t *testing.T) { + postgresOnly(t) + db := setupTestDB(t) + copyTestRows(t, db.conn, "transmissions", []string{"id", "raw_hex", "hash", "first_seen"}, 20000, func(i int) []any { + return []any{i + 1, "AA", fmt.Sprintf("cursor-%d", i), "2026-01-01T00:00:00Z"} + }) + if _, err := db.conn.Exec(`ANALYZE transmissions`); err != nil { + t.Fatal(err) + } + var raw string + if err := db.conn.QueryRow(`EXPLAIN (ANALYZE, FORMAT JSON) `+newTransmissionsSinceSQL, 10000, 100).Scan(&raw); err != nil { + t.Fatal(err) + } + type planNode struct { + Type string `json:"Node Type"` + Rows float64 `json:"Actual Rows"` + Plans []json.RawMessage `json:"Plans"` + } + var plans []struct{ Plan json.RawMessage } + if err := json.Unmarshal([]byte(raw), &plans); err != nil { + t.Fatal(err) + } + pending := []json.RawMessage{plans[0].Plan} + for len(pending) > 0 { + var node planNode + if err := json.Unmarshal(pending[0], &node); err != nil { + t.Fatal(err) + } + pending = append(pending[1:], node.Plans...) + if strings.Contains(node.Type, "Sort") || (strings.Contains(node.Type, "Scan") && node.Rows > 100) { + t.Fatalf("cursor scanned or sorted more than its bounded page: %s rows=%g", node.Type, node.Rows) + } + } + rows, err := db.GetNewTransmissionsSince(10000, 100) + if err != nil || len(rows) != 100 || rows[0]["id"] != 10001 { + t.Fatalf("cursor page: len=%d, %v", len(rows), err) + } +} + +func TestPostgresNegativeOffsetsPreserveFirstPage(t *testing.T) { + db := setupTestDB(t) + seedTestData(t, db) + for name, query := range map[string]func() (*PacketResult, error){ + "packets": func() (*PacketResult, error) { return db.QueryPackets(PacketQuery{Limit: 2, Offset: -1}) }, + "groups": func() (*PacketResult, error) { return db.QueryGroupedPackets(PacketQuery{Limit: 2, Offset: -1}) }, + "multiple nodes": func() (*PacketResult, error) { + return db.QueryMultiNodePackets([]string{"aabbccdd11223344"}, 2, -1, "DESC", "", "") + }, + } { + t.Run(name, func(t *testing.T) { + result, err := query() + if err != nil || len(result.Packets) != 2 { + t.Fatalf("negative offset must return the first page: %+v, %v", result, err) + } + }) + } + nodes, _, _, err := db.GetNodes(NodeQuery{Limit: 2, Offset: -1}) + if err != nil || len(nodes) != 2 { + t.Fatalf("negative node offset must return the first page: %d, %v", len(nodes), err) + } +} + +func TestPostgresCanonicalSchemaAPIs(t *testing.T) { + postgresOnly(t) + dsn := pgtest.NewSchema(t) + writer, err := openFixtureSQL(dsn) + if err != nil { + t.Fatal(err) + } + defer writer.Close() + if err := dbschema.Apply(writer, nil); err != nil { + t.Fatal(err) + } + seedTestData(t, &DB{backend: testBackendValue(), conn: writer}) + if _, err := writer.Exec(`UPDATE transmissions SET last_seen=EXTRACT(EPOCH FROM first_seen::timestamptz)::bigint`); err != nil { + t.Fatal(err) + } + db, err := openFixtureReader(t, dsn) + if err != nil { + t.Fatal(err) + } + defer db.Close() + if err := dbschema.AssertReady(db.conn); err != nil { + t.Fatal(err) + } + stats, err := db.GetStats() + if err != nil || stats.TotalTransmissions != 3 || stats.TotalObservations != 4 { + t.Fatalf("canonical statistics: %+v, %v", stats, err) + } + pt := 4 + q := PacketQuery{Limit: 5, Type: &pt, Observer: "obs1,obs2", Region: "SJC", Since: "2020-01-01", Until: "2099-01-01", Order: "DESC"} + packets, err := db.QueryPackets(q) + if err != nil || len(packets.Packets) != 2 { + t.Fatalf("filtered packets: %+v, %v", packets, err) + } + grouped, err := db.QueryGroupedPackets(q) + if err != nil || len(grouped.Packets) != 2 { + t.Fatalf("filtered groups: %+v, %v", grouped, err) + } + nodes, count, _, err := db.GetNodes(NodeQuery{Limit: 5, Role: "repeater", Search: "test", RegionPubkeys: []string{"aabbccdd11223344"}}) + if err != nil || count != 1 || len(nodes) != 1 { + t.Fatalf("filtered nodes: %d/%d, %v", len(nodes), count, err) + } + messages, count, err := db.GetChannelMessages("#test", 5, 0, "SJC,SFO") + if err != nil || count != 1 || len(messages) != 1 { + t.Fatalf("channel messages: %d/%d, %v", len(messages), count, err) + } + store := NewPacketStore(db, nil) + if err := store.Load(); err != nil { + t.Fatal(err) + } + if len(store.packets) != 3 || store.totalObs != 4 { + t.Fatalf("canonical load: %d transmissions, %d observations", len(store.packets), store.totalObs) + } +} + +func TestPostgresSelectionCannotOpenLegacyFileWithoutURL(t *testing.T) { + legacy := filepath.Join(t.TempDir(), "private-measurements.db") + db, err := OpenStorage(dbconfig.Storage{Backend: dbconfig.Postgres, DBPath: legacy}) + if db != nil { + db.Close() + } + if err == nil || !strings.Contains(err.Error(), "PostgreSQL") { + t.Fatalf("expected actionable PostgreSQL migration error, got %v", err) + } + if strings.Contains(err.Error(), legacy) { + t.Fatal("database input leaked in connection error") + } +} + +func TestPostgresReaderRejectsWriterCredential(t *testing.T) { + postgresOnly(t) + db, err := OpenDB(pgtest.NewSchema(t)) + if db != nil { + db.Close() + } + if err == nil || !strings.Contains(err.Error(), "reader role") { + t.Fatalf("elevated telemetry credential was not refused: %v", err) + } +} + +func TestPostgresScopeAuditSnapshotKeepsConcurrentCommitOut(t *testing.T) { + postgresOnly(t) + dsn := pgtest.NewSchema(t) + writer, err := openFixtureSQL(dsn) + if err != nil { + t.Fatal(err) + } + defer writer.Close() + if _, err := writer.Exec(`CREATE TABLE transmissions(id BIGINT PRIMARY KEY, first_seen TEXT, scope_name TEXT, route_type INTEGER); + CREATE TABLE observations(id BIGINT PRIMARY KEY, transmission_id BIGINT, path_json TEXT); + INSERT INTO transmissions VALUES(1,'2026-01-02T00:00:00Z','scope',1); + INSERT INTO observations VALUES(1,1,'["aabb"]')`); err != nil { + t.Fatal(err) + } + conn, err := pgutil.Open(pgtest.ReadOnly(t, dsn), true) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + db := &DB{backend: testBackendValue(), conn: conn} + tx, err := db.beginScopeAuditSnapshot(context.Background()) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback() + meta, err := scopeAuditWindowMeta(tx, "2026-01-01T00:00:00Z") + if err != nil || len(meta) != 1 { + t.Fatalf("initial metadata: %d, %v", len(meta), err) + } + if _, err := writer.Exec(`INSERT INTO transmissions VALUES(2,'2026-01-03T00:00:00Z','scope',1); + INSERT INTO observations VALUES(2,2,'["aabb"]')`); err != nil { + t.Fatal(err) + } + rows, err := tx.Query(scopeAuditForwarderScanQuery, "2026-01-01T00:00:00Z") + if err != nil { + t.Fatal(err) + } + count := 0 + for rows.Next() { + var id int64 + var hop string + if err := rows.Scan(&id, &hop); err != nil { + t.Fatal(err) + } + if id != 1 { + t.Fatal("scope snapshot included a later commit absent from metadata") + } + count++ + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + rows.Close() + if count != 1 { + t.Fatalf("hop rows: %d, want 1", count) + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + var visible int + if err := conn.QueryRow(`SELECT COUNT(*) FROM transmissions`).Scan(&visible); err != nil || visible != 2 { + t.Fatalf("fresh snapshot did not see committed row: %d, %v", visible, err) + } +} + +func TestPostgresReaderUsesRestrictedRole(t *testing.T) { + postgresOnly(t) + dsn := pgtest.NewSchema(t) + writer, err := pgutil.Open(dsn, false) + if err != nil { + t.Fatal(err) + } + defer writer.Close() + _, err = writer.Exec(testNativeSQL(`CREATE TABLE transmissions (id BIGINT PRIMARY KEY, hash TEXT); + CREATE TABLE observations (id BIGINT PRIMARY KEY, timestamp BIGINT, observer_idx BIGINT); + CREATE TABLE nodes (public_key TEXT, name TEXT, role TEXT, last_seen TEXT); + CREATE TABLE observers (id TEXT, inactive INTEGER); + INSERT INTO transmissions VALUES (7, 'stable-hash')`, `CREATE TABLE transmissions (id BIGINT PRIMARY KEY, hash TEXT); + CREATE TABLE observations (id BIGINT PRIMARY KEY, timestamp BIGINT, observer_idx BIGINT); + CREATE TABLE nodes (public_key TEXT, name TEXT, role TEXT, last_seen TEXT); + CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY UNIQUE, id TEXT, inactive INTEGER); + INSERT INTO transmissions VALUES (7, 'stable-hash')`)) + if err != nil { + t.Fatal(err) + } + reader, err := OpenDB(pgtest.ReadOnly(t, dsn)) + if err != nil { + t.Fatalf("open PostgreSQL reader: %v", err) + } + defer reader.Close() + if got := reader.GetMaxTransmissionID(); got != 7 { + t.Fatalf("max transmission = %d, want 7", got) + } + if _, err := reader.conn.Exec(`INSERT INTO transmissions VALUES (8, 'forbidden')`); err == nil { + t.Fatal("telemetry reader accepted a write") + } + if _, err := reader.conn.Exec(`SET default_transaction_read_only = off`); err != nil { + t.Fatal(err) + } + if _, err := reader.conn.Exec(`DELETE FROM transmissions`); err == nil { + t.Fatal("reader can write after changing its session flag") + } +} diff --git a/cmd/server/postgres_value_parity_test.go b/cmd/server/postgres_value_parity_test.go new file mode 100644 index 000000000..11d01d04a --- /dev/null +++ b/cmd/server/postgres_value_parity_test.go @@ -0,0 +1,198 @@ +package main + +import ( + "database/sql" + "encoding/json" + "fmt" + "net/http/httptest" + "testing" + "time" + + "github.com/gorilla/mux" +) + +func openPostgresValueFixture(t *testing.T) (*DB, *sql.DB) { + t.Helper() + dsn := postgresTestDSN(t) + writer, err := openFixtureSQL(dsn) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { writer.Close() }) + if err := applyTestSchema(t, writer); err != nil { + t.Fatal(err) + } + reader, err := openFixtureReader(t, dsn) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { reader.Close() }) + return reader, writer +} + +// SQLite INTEGER affinity permits fractional values, and PostgreSQL float8 +// returns large integral values in a form that NullInt64 cannot scan. Neither +// case may drop an observation or its joined transmission from the server. +func TestPostgresObservationScoreLoadPaths(t *testing.T) { + for _, path := range []string{"full", "chunked", "older", "transmission_poll", "observation_poll"} { + t.Run(path, func(t *testing.T) { + db, writer := openPostgresValueFixture(t) + stamp := time.Now().UTC().Truncate(time.Second).Add(-2 * time.Hour) + if _, err := writer.Exec(`INSERT INTO observers(rowid,id,name,iata) VALUES(1,'score-observer','Score observer','AAA')`); err != nil { + t.Fatal(err) + } + scores := []any{1000000, 3.5, nil} + seedTx := func() { + for i := range scores { + if _, err := writer.Exec(`INSERT INTO transmissions(id,raw_hex,hash,first_seen,last_seen,payload_type) VALUES($1,'AA',$2,$3,$4,0)`, i+1, fmt.Sprintf("score-%d", i), stamp.Format(time.RFC3339), stamp.Unix()); err != nil { + t.Fatal(err) + } + } + } + seedObs := func() { + for i, score := range scores { + if _, err := writer.Exec(`INSERT INTO observations(id,transmission_id,observer_idx,score,snr,rssi,path_json,timestamp) VALUES($1,$1,1,$2,3.5,-90.5,'[]',$3)`, i+1, score, stamp.Unix()); err != nil { + t.Fatal(err) + } + } + } + if path != "transmission_poll" { + seedTx() + if path != "observation_poll" { + seedObs() + } + } + cfg := &PacketStoreConfig{RetentionHours: 24} + if path == "older" { + cfg.HotStartupHours = 1 + } + store := NewPacketStore(db, cfg) + if path == "chunked" { + if err := store.LoadChunked(1); err != nil { + t.Fatal(err) + } + } else if err := store.Load(); err != nil { + t.Fatal(err) + } + switch path { + case "older": + if len(store.packets) != 0 { + t.Fatal("older-history fixture unexpectedly loaded during hot startup") + } + if err := store.loadChunk(stamp.Add(-time.Hour), stamp.Add(time.Hour)); err != nil { + t.Fatal(err) + } + case "transmission_poll": + seedTx() + seedObs() + packets, cursor := store.IngestNewFromDB(0, 10) + if len(packets) != 3 || cursor != 3 { + t.Errorf("poll returned %d transmissions, cursor %d; want 3 and 3", len(packets), cursor) + } + case "observation_poll": + seedObs() + if packets := store.IngestNewObservations(0, 10); len(packets) != 3 { + t.Errorf("observation poll updated %d transmissions; want 3", len(packets)) + } + } + if len(store.packets) != 3 || store.totalObs != 3 { + t.Errorf("loaded %d transmissions/%d observations; want 3/3", len(store.packets), store.totalObs) + } + want := map[string]string{"score-0": "1000000", "score-1": "3.5", "score-2": "null"} + for hash, value := range want { + observations := store.GetObservationsForHash(hash) + if len(observations) != 1 { + t.Errorf("%s loaded %d observations; want 1", hash, len(observations)) + continue + } + raw, err := json.Marshal(observations[0]["score"]) + if err != nil || string(raw) != value { + t.Errorf("%s score JSON=%s, %v; want %s", hash, raw, err, value) + } + } + // Exercise the existing HTTP field that exposes enriched scores. + server := &Server{db: db, store: store} + req := mux.SetURLVars(httptest.NewRequest("GET", "/api/observers/score-observer/analytics?days=1", nil), map[string]string{"id": "score-observer"}) + response := httptest.NewRecorder() + server.handleObserverAnalytics(response, req) + var body struct { + RecentPackets []struct { + Hash string `json:"hash"` + Score json.RawMessage `json:"score"` + } `json:"recentPackets"` + } + if response.Code != 200 || json.Unmarshal(response.Body.Bytes(), &body) != nil { + t.Fatalf("observer analytics response: %d %s", response.Code, response.Body.String()) + } + if len(body.RecentPackets) != len(want) { + t.Errorf("HTTP API returned %d observations; want %d", len(body.RecentPackets), len(want)) + } + for _, packet := range body.RecentPackets { + if expected, ok := want[packet.Hash]; !ok || string(packet.Score) != expected { + t.Errorf("HTTP score %s=%s; want JSON number/null %s", packet.Hash, packet.Score, expected) + } + } + }) + } +} + +func TestPostgresPacketRowScore(t *testing.T) { + db, writer := openPostgresValueFixture(t) + for i, score := range []any{1000000, 3.5, nil} { + if _, err := writer.Exec(`INSERT INTO transmissions(id,raw_hex,hash,first_seen) VALUES($1,'AA',$2,'2026-01-01T00:00:00Z')`, i+1, fmt.Sprintf("score-%d", i)); err != nil { + t.Fatal(err) + } + if _, err := writer.Exec(`INSERT INTO observations(id,transmission_id,score,timestamp) VALUES($1,$1,$2,1)`, i+1, score); err != nil { + t.Fatal(err) + } + } + rows, err := db.conn.Query(`SELECT id,raw_hex,timestamp,observer_id,observer_name,direction,snr,rssi,score,hash,route_type,payload_type,payload_version,path_json,decoded_json,created_at FROM packets_v ORDER BY id`) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + want := []string{"1000000", "3.5", "null"} + index := 0 + for rows.Next() { + packet := scanPacketRow(rows) + if packet == nil { + t.Errorf("row %d silently dropped its score", index) + } else if raw, err := json.Marshal(packet["score"]); err != nil || string(raw) != want[index] { + t.Errorf("row %d score=%s, %v; want %s", index, raw, err, want[index]) + } + index++ + } + if rows.Err() != nil || index != len(want) { + t.Fatalf("scanned %d rows, %v", index, rows.Err()) + } +} + +func TestPostgresNodeSearchKeepsLiteralBackslashAndWildcards(t *testing.T) { + db, writer := openPostgresValueFixture(t) + for i, name := range []string{`node\path`, "nodepath", "nodeXpath"} { + if _, err := writer.Exec(`INSERT INTO nodes(public_key,name) VALUES($1,$2)`, fmt.Sprintf("%064x", i+1), name); err != nil { + t.Fatal(err) + } + } + for _, tc := range []struct { + query string + count int + }{{`node\path`, 1}, {`NODE\PATH`, 1}, {`node\%`, 1}, {"node_path", 2}, {"node%path", 3}} { + t.Run(tc.query, func(t *testing.T) { + nodes, total, _, err := db.GetNodes(NodeQuery{Search: tc.query}) + if err != nil || len(nodes) != tc.count || total != tc.count { + t.Errorf("GetNodes(%q): %d rows,total=%d,%v; want %d", tc.query, len(nodes), total, err, tc.count) + } + if tc.count == 1 && len(nodes) == 1 && nodes[0]["name"] != `node\path` { + t.Errorf("GetNodes(%q) matched %v instead of the literal backslash name", tc.query, nodes[0]["name"]) + } + nodes, err = db.SearchNodes(tc.query, 10) + if err != nil || len(nodes) != tc.count { + t.Errorf("SearchNodes(%q): %d rows,%v; want %d", tc.query, len(nodes), err, tc.count) + } + if tc.count == 1 && len(nodes) == 1 && nodes[0]["name"] != `node\path` { + t.Errorf("SearchNodes(%q) matched %v instead of the literal backslash name", tc.query, nodes[0]["name"]) + } + }) + } +} diff --git a/cmd/server/preparable_schema_test.go b/cmd/server/preparable_schema_test.go index 0540091d7..1f75f09a5 100644 --- a/cmd/server/preparable_schema_test.go +++ b/cmd/server/preparable_schema_test.go @@ -2,27 +2,18 @@ package main import ( "database/sql" - "strings" + "github.com/meshcore-analyzer/dbschema/legacy" "testing" ) -// The statements compiled by prepareStatements reference a set of tables and -// columns. Under modernc.org/sqlite that never mattered for test fixtures: -// newStmt only stored the SQL string and compiled it lazily on first use, so a -// fixture could omit half the schema and still open. github.com/mattn/go-sqlite3 -// calls sqlite3_prepare_v2 inside Prepare, so OpenDB now fails on an incomplete -// schema — which is the production behaviour we want (#1901), but it means -// fixtures have to declare what they are prepared against. -// -// ensurePreparable is the seam. Fixtures that deliberately build a partial -// schema (to exercise schema detection, say) call it on the writable connection -// just before OpenDB, and it fills in whatever is missing without disturbing the -// shapes the test actually cares about. +// Deliberately partial fixtures declare the table shapes they test, then +// ensurePreparable supplies the minimum surface required by OpenDB's prepared +// statements. It never creates or modifies a runtime database. var preparableTables = []string{ - `CREATE TABLE IF NOT EXISTS transmissions (id INTEGER PRIMARY KEY, hash TEXT)`, - `CREATE TABLE IF NOT EXISTS observations (id INTEGER PRIMARY KEY, timestamp TEXT)`, + testNativeSQL(`CREATE TABLE IF NOT EXISTS transmissions (id INTEGER PRIMARY KEY AUTOINCREMENT, hash TEXT)`, `CREATE TABLE IF NOT EXISTS transmissions (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, hash TEXT)`), + testNativeSQL(`CREATE TABLE IF NOT EXISTS observations (id INTEGER PRIMARY KEY AUTOINCREMENT, timestamp TEXT)`, `CREATE TABLE IF NOT EXISTS observations (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, timestamp TEXT)`), `CREATE TABLE IF NOT EXISTS nodes (public_key TEXT PRIMARY KEY)`, - `CREATE TABLE IF NOT EXISTS observers (id TEXT)`, + testNativeSQL(`CREATE TABLE IF NOT EXISTS observers (id TEXT)`, `CREATE TABLE IF NOT EXISTS observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY UNIQUE, id TEXT)`), } // preparableColumns are added when absent. A fixture that already declared the @@ -46,9 +37,20 @@ func ensurePreparable(tb testing.TB, conn *sql.DB) { } } for _, c := range preparableColumns { - _, err := conn.Exec("ALTER TABLE " + c.table + " ADD COLUMN " + c.column + " " + c.decl) + var err error + if testBackendValue() == "sqlite" { + exists, e := legacy.TableHasColumn(conn, c.table, c.column) + if e != nil { + tb.Fatal(e) + } + if !exists { + _, err = conn.Exec("ALTER TABLE " + c.table + " ADD COLUMN " + c.column + " " + c.decl) + } + } else { + _, err = conn.Exec("ALTER TABLE " + c.table + " ADD COLUMN IF NOT EXISTS " + c.column + " " + c.decl) + } // "duplicate column name" means the fixture already declared it. - if err != nil && !strings.Contains(err.Error(), "duplicate column name") { + if err != nil { tb.Fatalf("ensurePreparable: add %s.%s: %v", c.table, c.column, err) } } @@ -59,7 +61,7 @@ func ensurePreparable(tb testing.TB, conn *sql.DB) { // create, it fails here rather than as a confusing OpenDB error in whichever // fixture happens to be thinnest. func TestEnsurePreparableMatchesPrepareStatements(t *testing.T) { - conn, err := sql.Open("sqlite3", ":memory:") + conn, err := openPostgresTestDB(t) if err != nil { t.Fatal(err) } @@ -69,7 +71,7 @@ func TestEnsurePreparableMatchesPrepareStatements(t *testing.T) { conn.SetMaxOpenConns(1) ensurePreparable(t, conn) - db := &DB{conn: conn} + db := &DB{backend: testBackendValue(), conn: conn} if err := db.prepareStatements(); err != nil { t.Fatalf("prepareStatements against the minimal preparable schema: %v\n"+ "Add the missing table/column to preparableTables or preparableColumns.", err) diff --git a/cmd/server/proposal_handlers_test.go b/cmd/server/proposal_handlers_test.go index 4a4e6ebe4..b4aeee46b 100644 --- a/cmd/server/proposal_handlers_test.go +++ b/cmd/server/proposal_handlers_test.go @@ -26,12 +26,12 @@ func newProposalFixture(t *testing.T, ps proposalSettings) *authFixture { func newProposalFixtureWithConfig(t *testing.T, ps proposalSettings, cfg *Config) *authFixture { t.Helper() - a, fake := newTestAuthService(t, "admin@example.org") + a, fake, ownerURL := newTestAuthServiceWithURL(t, postgresTestDSN(t), "admin@example.org") a.set.proposals = ps srv := &Server{cfg: cfg, perfStats: NewPerfStats(), auth: a} r := mux.NewRouter() srv.registerAuthRoutes(r) - return &authFixture{srv: srv, router: r, fake: fake, st: a.st} + return &authFixture{srv: srv, router: r, fake: fake, st: a.st, ownerURL: ownerURL} } func proposalPeople(t *testing.T, f *authFixture) (admin, user *client) { diff --git a/cmd/server/readonly_invariant_test.go b/cmd/server/readonly_invariant_test.go index 3e265655e..3effbc33e 100644 --- a/cmd/server/readonly_invariant_test.go +++ b/cmd/server/readonly_invariant_test.go @@ -1,7 +1,6 @@ package main import ( - "database/sql" "fmt" "os" "path/filepath" @@ -9,8 +8,6 @@ import ( "regexp" "strings" "testing" - - _ "github.com/mattn/go-sqlite3" ) // TestServerSourceHasNoCachedRWCalls enforces issue #1287: after the @@ -47,8 +44,8 @@ func TestServerSourceHasNoCachedRWCalls(t *testing.T) { // server publishes via internal/mbcapqueue). // Shapes are normalised before matching (see nodeTableWritePattern): // optional OR-conflict clause, optional quoting, optional alias. - nodeTableWritePattern(`UPDATE(\s+OR\s+\w+)?`, `SET`), - nodeTableWritePattern(`INSERT\s+(OR\s+\w+\s+)?INTO`, ``), + nodeTableWritePattern(`UPDATE(\s+OR\s+\w+)$1`, `SET`), + nodeTableWritePattern(`INSERT\s+(OR\s+\w+\s+)$1INTO`, ``), nodeTableWritePattern(`REPLACE\s+INTO`, ``), nodeTableWritePattern(`DELETE\s+FROM`, ``), regexp.MustCompile(`\bpersistMultibyteCapability\s*\(`), @@ -124,8 +121,7 @@ func TestServerDBHasNoWriteMethods(t *testing.T) { // second RW handle. After the fix, server-side writes are impossible // because there is no helper to open a writable connection. func TestServerDBConnIsReadOnly(t *testing.T) { - dir := t.TempDir() - path := dir + "/ro_invariant.db" + path := postgresTestDSN(t) // Bootstrap a minimal DB with the ingestor-style WAL opener so the // server can attach in read-only mode. @@ -133,7 +129,7 @@ func TestServerDBConnIsReadOnly(t *testing.T) { t.Fatalf("bootstrap: %v", err) } - d, err := OpenDB(path) + d, err := openFixtureReader(t, path) if err != nil { t.Fatalf("OpenDB: %v", err) } @@ -150,8 +146,8 @@ func TestServerDBConnIsReadOnly(t *testing.T) { // Kept in *_test.go so it does NOT add any write capability to the // production server binary. func bootstrapMinimalDB(tb testing.TB, path string) error { - dsn := fmt.Sprintf("file:%s?_journal_mode=WAL&_busy_timeout=5000", path) - rw, err := sql.Open("sqlite3", dsn) + dsn := path + rw, err := openFixtureSQL(dsn) if err != nil { return err } @@ -258,8 +254,8 @@ func TestUsersOpenIsTheOnlyServerWritePath(t *testing.T) { if err != nil { t.Fatal(err) } - call := regexp.MustCompile(`users\.Open\(`) - want := regexp.MustCompile(`users\.Open\(set\.dbPath, measurementDBPath\)`) + call := regexp.MustCompile(`users\.OpenStorage\(`) + want := regexp.MustCompile(`users\.OpenStorage\(storage\)`) calls, good := 0, 0 for _, e := range entries { name := e.Name() diff --git a/cmd/server/region_nodes_test.go b/cmd/server/region_nodes_test.go index 9ee63bb78..223b7281d 100644 --- a/cmd/server/region_nodes_test.go +++ b/cmd/server/region_nodes_test.go @@ -36,7 +36,7 @@ func newRegionTestStore(t testing.TB, observers map[string]string) *PacketStore t.Helper() db := setupTestDB(t) for id, iata := range observers { - if _, err := db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) VALUES (?, ?, ?, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z', 1)`, id, id, iata); err != nil { + if _, err := db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) VALUES ($1, $2, $3, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z', 1)`, id, id, iata); err != nil { t.Fatal(err) } } diff --git a/cmd/server/repeater_liveness_test.go b/cmd/server/repeater_liveness_test.go index 07f108812..e84e1043b 100644 --- a/cmd/server/repeater_liveness_test.go +++ b/cmd/server/repeater_liveness_test.go @@ -13,7 +13,7 @@ func TestRepeaterRelayActivity_Active(t *testing.T) { defer db.conn.Close() pubkey := "aabbccdd11223344" - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ($1, $2, $3, $4)", pubkey, "RepActive", "repeater", recentTS(1)) store := NewPacketStore(db, nil) @@ -60,7 +60,7 @@ func seedUnscopedRelayFixture(t *testing.T, hashPrefix string) (*PacketStore, st t.Helper() db := setupCapabilityTestDB(t) pubkey := "aabbccdd11223344" - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ($1, $2, $3, $4)", pubkey, "RepUnscoped", "repeater", recentTS(1)) store := NewPacketStore(db, nil) pt := 1 // non-advert (TXT_MSG) @@ -109,7 +109,7 @@ func TestRepeaterRelayActivity_Idle(t *testing.T) { defer db.conn.Close() pubkey := "ccddeeff55667788" - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ($1, $2, $3, $4)", pubkey, "RepIdle", "repeater", recentTS(1)) store := NewPacketStore(db, nil) @@ -134,7 +134,7 @@ func TestRepeaterRelayActivity_Stale(t *testing.T) { defer db.conn.Close() pubkey := "1122334455667788" - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ($1, $2, $3, $4)", pubkey, "RepStale", "repeater", recentTS(1)) store := NewPacketStore(db, nil) @@ -176,7 +176,7 @@ func TestRepeaterRelayActivity_IgnoresAdverts(t *testing.T) { defer db.conn.Close() pubkey := "deadbeef00000001" - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ($1, $2, $3, $4)", pubkey, "RepAdvertOnly", "repeater", recentTS(1)) store := NewPacketStore(db, nil) @@ -227,7 +227,7 @@ func TestRepeaterRelayActivity_PrefixHop(t *testing.T) { defer db.conn.Close() pubkey := "a36a21290d9c25a158130fe7c489541210d5f09f25fab997db5e942fb7680510" - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ($1, $2, $3, $4)", pubkey, "RepPrefix", "repeater", recentTS(1)) store := NewPacketStore(db, nil) @@ -274,7 +274,7 @@ func TestRepeaterRelayActivity_DedupAcrossPrefixAndFullKey(t *testing.T) { defer db.conn.Close() pubkey := "a36a21290d9c25a158130fe7c489541210d5f09f25fab997db5e942fb7680510" - db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)", + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES ($1, $2, $3, $4)", pubkey, "RepDedup", "repeater", recentTS(1)) store := NewPacketStore(db, nil) diff --git a/cmd/server/resolve_context_test.go b/cmd/server/resolve_context_test.go index c05018503..b6bfed123 100644 --- a/cmd/server/resolve_context_test.go +++ b/cmd/server/resolve_context_test.go @@ -164,7 +164,7 @@ func TestResolveHopsAPI_UniquePrefix(t *testing.T) { _ = srv // Insert a unique node - srv.db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, lat, lon, role) VALUES (?, ?, ?, ?, ?)", + srv.db.conn.Exec("INSERT INTO nodes (public_key, name, lat, lon, role) VALUES ($1, $2, $3, $4, $5) ON CONFLICT DO NOTHING", "ff11223344", "UniqueNode", 37.0, -122.0, "repeater") srv.store.InvalidateNodeCache() @@ -189,9 +189,9 @@ func TestResolveHopsAPI_UniquePrefix(t *testing.T) { func TestResolveHopsAPI_AmbiguousNoContext(t *testing.T) { srv, router := setupTestServer(t) - srv.db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, lat, lon, role) VALUES (?, ?, ?, ?, ?)", + srv.db.conn.Exec("INSERT INTO nodes (public_key, name, lat, lon, role) VALUES ($1, $2, $3, $4, $5) ON CONFLICT DO NOTHING", "ee1aaaaaaa", "Node-E1", 37.0, -122.0, "repeater") - srv.db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, lat, lon, role) VALUES (?, ?, ?, ?, ?)", + srv.db.conn.Exec("INSERT INTO nodes (public_key, name, lat, lon, role) VALUES ($1, $2, $3, $4, $5) ON CONFLICT DO NOTHING", "ee1bbbbbbb", "Node-E2", 38.0, -121.0, "repeater") srv.store.InvalidateNodeCache() @@ -224,11 +224,11 @@ func TestResolveHopsAPI_AmbiguousNoContext(t *testing.T) { func TestResolveHopsAPI_WithAffinityContext(t *testing.T) { srv, router := setupTestServer(t) - srv.db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, lat, lon, role) VALUES (?, ?, ?, ?, ?)", + srv.db.conn.Exec("INSERT INTO nodes (public_key, name, lat, lon, role) VALUES ($1, $2, $3, $4, $5) ON CONFLICT DO NOTHING", "dd1aaaaaaa", "Node-D1", 37.0, -122.0, "repeater") - srv.db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, lat, lon, role) VALUES (?, ?, ?, ?, ?)", + srv.db.conn.Exec("INSERT INTO nodes (public_key, name, lat, lon, role) VALUES ($1, $2, $3, $4, $5) ON CONFLICT DO NOTHING", "dd1bbbbbbb", "Node-D2", 38.0, -121.0, "repeater") - srv.db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, lat, lon, role) VALUES (?, ?, ?, ?, ?)", + srv.db.conn.Exec("INSERT INTO nodes (public_key, name, lat, lon, role) VALUES ($1, $2, $3, $4, $5) ON CONFLICT DO NOTHING", "c0c0c0c0c0", "Context", 37.1, -122.1, "repeater") // Invalidate node cache so the PM includes newly inserted nodes. @@ -279,7 +279,7 @@ func TestResolveHopsAPI_WithAffinityContext(t *testing.T) { func TestResolveHopsAPI_ResponseShape(t *testing.T) { srv, router := setupTestServer(t) - srv.db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, lat, lon, role) VALUES (?, ?, ?, ?, ?)", + srv.db.conn.Exec("INSERT INTO nodes (public_key, name, lat, lon, role) VALUES ($1, $2, $3, $4, $5) ON CONFLICT DO NOTHING", "bb1aaaaaaa", "Node-B1", 37.0, -122.0, "repeater") req := httptest.NewRequest("GET", "/api/resolve-hops?hops=bb1a", nil) diff --git a/cmd/server/resolved_index.go b/cmd/server/resolved_index.go index 940733dd5..2bc925708 100644 --- a/cmd/server/resolved_index.go +++ b/cmd/server/resolved_index.go @@ -159,7 +159,7 @@ func (s *PacketStore) confirmResolvedPathContains(txID int, pubkey string) bool needle := `"` + strings.ToLower(pubkey) + `"` var count int err := s.db.conn.QueryRow( - `SELECT COUNT(*) FROM observations WHERE transmission_id = ? AND INSTR(LOWER(resolved_path), ?) > 0`, + `SELECT COUNT(*) FROM observations WHERE transmission_id = `+s.db.parameter(1)+` AND `+s.db.nativeSQL("instr", "strpos")+`(LOWER(resolved_path), `+s.db.parameter(2)+`) > 0`, txID, needle, ).Scan(&count) if err != nil { @@ -219,8 +219,7 @@ func (s *PacketStore) loadCanonicalResolvedPaths(snapshots map[int][]rpObs) map[ args[i] = id } s.beforeCacheLoad("resolvedPathBatch") - rows, err := s.db.conn.Query(`SELECT id, resolved_path FROM observations WHERE id IN (?`+ - strings.Repeat(",?", len(chunk)-1)+`)`, args...) + rows, err := s.db.conn.Query(`SELECT id, resolved_path FROM observations WHERE id IN (`+s.db.sqlPlaceholders(len(chunk))+`)`, args...) if err != nil { for _, id := range chunk { fallback = append(fallback, longestTx[id]) @@ -252,7 +251,7 @@ func (s *PacketStore) loadCanonicalResolvedPaths(snapshots map[int][]rpObs) map[ return out } -// fetchResolvedPathsForTx fetches resolved_path from SQLite for all observations +// fetchResolvedPathsForTx fetches resolved_path from the selected database for all observations // of a transmission. Used for on-demand API responses and eviction cleanup. func (s *PacketStore) fetchResolvedPathsForTx(txID int) map[int][]*string { if s.db == nil || s.db.conn == nil { @@ -260,7 +259,7 @@ func (s *PacketStore) fetchResolvedPathsForTx(txID int) map[int][]*string { } s.beforeCacheLoad("resolvedPath") rows, err := s.db.conn.Query( - `SELECT id, resolved_path FROM observations WHERE transmission_id = ? AND resolved_path IS NOT NULL`, + `SELECT id, resolved_path FROM observations WHERE transmission_id = `+s.db.parameter(1)+` AND resolved_path IS NOT NULL`, txID, ) if err != nil { @@ -302,7 +301,7 @@ func (s *PacketStore) fetchResolvedPathForObs(obsID int) []*string { s.beforeCacheLoad("resolvedPath") var rpJSON sql.NullString err := s.db.conn.QueryRow( - `SELECT resolved_path FROM observations WHERE id = ?`, obsID, + `SELECT resolved_path FROM observations WHERE id = `+s.db.parameter(1), obsID, ).Scan(&rpJSON) if err != nil || !rpJSON.Valid { return nil @@ -482,14 +481,14 @@ func (s *PacketStore) lruDelete(obsID int) { // resolvedPubkeysForEvictionBatch fetches resolved pubkeys for multiple txIDs // from SQL in a single batched query. Returns a map from txID to unique pubkeys. // MUST be called WITHOUT holding s.mu — this is the whole point of the batch approach. -// Chunks queries to stay under SQLite's 500-parameter limit. +// Chunks queries to bound each lookup and its result set. func (s *PacketStore) resolvedPubkeysForEvictionBatch(txIDs []int) map[int][]string { result := make(map[int][]string, len(txIDs)) if len(txIDs) == 0 || s.db == nil || s.db.conn == nil { return result } - const chunkSize = 499 // SQLite SQLITE_MAX_VARIABLE_NUMBER default is 999; stay well under + const chunkSize = 499 // Retain the existing bounded lookup batch. for start := 0; start < len(txIDs); start += chunkSize { end := start + chunkSize if end > len(txIDs) { @@ -497,19 +496,12 @@ func (s *PacketStore) resolvedPubkeysForEvictionBatch(txIDs []int) map[int][]str } chunk := txIDs[start:end] - // Build query with placeholders - placeholders := make([]byte, 0, len(chunk)*2) args := make([]interface{}, len(chunk)) for i, id := range chunk { - if i > 0 { - placeholders = append(placeholders, ',') - } - placeholders = append(placeholders, '?') args[i] = id } - query := "SELECT transmission_id, resolved_path FROM observations WHERE transmission_id IN (" + - string(placeholders) + ") AND resolved_path IS NOT NULL" + s.db.sqlPlaceholders(len(chunk)) + ") AND resolved_path IS NOT NULL" rows, err := s.db.conn.Query(query, args...) if err != nil { diff --git a/cmd/server/resolved_index_test.go b/cmd/server/resolved_index_test.go index 9ff385020..43b6eba0b 100644 --- a/cmd/server/resolved_index_test.go +++ b/cmd/server/resolved_index_test.go @@ -1,7 +1,6 @@ package main import ( - "database/sql" "fmt" "reflect" "runtime" @@ -9,8 +8,6 @@ import ( "sync" "testing" "time" - - _ "github.com/mattn/go-sqlite3" ) // --- Unit tests --- @@ -85,9 +82,9 @@ func TestResolvedPubkeyIndex_HashCollision(t *testing.T) { db.conn.Exec("INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (1, '', 'h1', '2026-01-01T00:00:00Z')") db.conn.Exec("INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (2, '', 'h2', '2026-01-01T00:00:00Z')") now := time.Now().Unix() - db.conn.Exec("INSERT INTO observations (id, transmission_id, observer_idx, path_json, timestamp, resolved_path) VALUES (1, 1, 1, ?, ?, ?)", + db.conn.Exec("INSERT INTO observations (id, transmission_id, observer_idx, path_json, timestamp, resolved_path) VALUES (1, 1, 1, $1, $2, $3)", `["aa"]`, now, `["real_match"]`) - db.conn.Exec("INSERT INTO observations (id, transmission_id, observer_idx, path_json, timestamp, resolved_path) VALUES (2, 2, 1, ?, ?, ?)", + db.conn.Exec("INSERT INTO observations (id, transmission_id, observer_idx, path_json, timestamp, resolved_path) VALUES (2, 2, 1, $1, $2, $3)", `["aa"]`, now, `["false_match"]`) // Simulate collision: manually insert both txIDs under the same hash @@ -307,9 +304,9 @@ func TestEviction_ByNodeCleanup_OnDemandSQL(t *testing.T) { relayPK := "relay_evict_test" // Insert DB data for on-demand SQL fetch during eviction - db.conn.Exec("INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (1, '', 'h1', ?)", + db.conn.Exec("INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (1, '', 'h1', $1)", now.Add(-48*time.Hour).Format(time.RFC3339)) - db.conn.Exec("INSERT INTO observations (id, transmission_id, observer_idx, path_json, timestamp, resolved_path) VALUES (1, 1, 1, ?, ?, ?)", + db.conn.Exec("INSERT INTO observations (id, transmission_id, observer_idx, path_json, timestamp, resolved_path) VALUES (1, 1, 1, $1, $2, $3)", `["aa"]`, now.Add(-48*time.Hour).Unix(), `["`+relayPK+`"]`) tx := &StoreTx{ @@ -382,9 +379,9 @@ func TestRunEviction_NoLockDuringSQLFetch(t *testing.T) { // Create 10 stale packets with resolved_path data in DB for i := 1; i <= 10; i++ { staleTime := now.Add(-48 * time.Hour).Format(time.RFC3339) - db.conn.Exec("INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (?, '', ?, ?)", + db.conn.Exec("INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES ($1, '', $2, $3)", i, fmt.Sprintf("h%d", i), staleTime) - db.conn.Exec("INSERT INTO observations (id, transmission_id, observer_idx, path_json, timestamp, resolved_path) VALUES (?, ?, 1, ?, ?, ?)", + db.conn.Exec("INSERT INTO observations (id, transmission_id, observer_idx, path_json, timestamp, resolved_path) VALUES ($1, $2, 1, $3, $4, $5)", i, i, `["aa"]`, now.Add(-48*time.Hour).Unix(), `["pk`+fmt.Sprintf("%d", i)+`"]`) tx := &StoreTx{ @@ -480,7 +477,7 @@ func TestPacketsAPI_OnDemandResolvedPath(t *testing.T) { now := time.Now().Unix() db.conn.Exec("INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (1, '', 'h1', '2026-01-01')") - db.conn.Exec("INSERT INTO observations (id, transmission_id, observer_idx, path_json, timestamp, resolved_path) VALUES (1, 1, 1, ?, ?, ?)", + db.conn.Exec("INSERT INTO observations (id, transmission_id, observer_idx, path_json, timestamp, resolved_path) VALUES (1, 1, 1, $1, $2, $3)", `["aa"]`, now, `["aabbccdd"]`) store := &PacketStore{ @@ -505,7 +502,7 @@ func TestPacketsAPI_OnDemandResolvedPath_LRUHit(t *testing.T) { now := time.Now().Unix() db.conn.Exec("INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (1, '', 'h1', '2026-01-01')") - db.conn.Exec("INSERT INTO observations (id, transmission_id, observer_idx, path_json, timestamp, resolved_path) VALUES (1, 1, 1, ?, ?, ?)", + db.conn.Exec("INSERT INTO observations (id, transmission_id, observer_idx, path_json, timestamp, resolved_path) VALUES (1, 1, 1, $1, $2, $3)", `["aa"]`, now, `["aabbccdd"]`) store := &PacketStore{ @@ -1094,17 +1091,21 @@ func TestLRU_CapacityAfterBulkDelete(t *testing.T) { // TestConfirmResolvedPathContains_SpecialChars verifies that pubkeys containing // SQL LIKE wildcards (%, _) don't cause false positives with the INSTR approach. func TestConfirmResolvedPathContains_SpecialChars(t *testing.T) { - db, err := sql.Open("sqlite3", ":memory:") + db, err := openPostgresTestDB(t) if err != nil { t.Fatal(err) } defer db.Close() - _, err = db.Exec(`CREATE TABLE observations ( - id INTEGER PRIMARY KEY, + _, err = db.Exec(testNativeSQL(`CREATE TABLE observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + transmission_id INTEGER, + resolved_path TEXT + )`, `CREATE TABLE observations ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, resolved_path TEXT - )`) + )`)) if err != nil { t.Fatal(err) } @@ -1117,7 +1118,7 @@ func TestConfirmResolvedPathContains_SpecialChars(t *testing.T) { } store := &PacketStore{ - db: &DB{conn: db}, + db: &DB{backend: testBackendValue(), conn: db}, } // Normal match should work @@ -1173,9 +1174,9 @@ func TestResolvedPubkeyIndex_BoundedByEviction(t *testing.T) { pk := fmt.Sprintf("pk_%04d", i) firstSeen := now.Add(-2 * time.Hour).Format(time.RFC3339) - db.conn.Exec("INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (?, '', ?, ?)", + db.conn.Exec("INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES ($1, '', $2, $3)", i, hash, firstSeen) - db.conn.Exec("INSERT INTO observations (id, transmission_id, observer_idx, path_json, timestamp, resolved_path) VALUES (?, ?, 1, ?, ?, ?)", + db.conn.Exec("INSERT INTO observations (id, transmission_id, observer_idx, path_json, timestamp, resolved_path) VALUES ($1, $2, 1, $3, $4, $5)", i, i, `["aa"]`, now.Add(-2*time.Hour).Unix(), `["`+pk+`"]`) tx := &StoreTx{ID: i, Hash: hash, FirstSeen: firstSeen} @@ -1293,8 +1294,8 @@ func TestRunEviction_TriggersCompaction(t *testing.T) { // Add one old tx with a large-cap slice in the index pk := "compaction_pk" firstSeen := now.Add(-2 * time.Hour).Format(time.RFC3339) - db.conn.Exec("INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (1, '', 'h1', ?)", firstSeen) - db.conn.Exec("INSERT INTO observations (id, transmission_id, observer_idx, path_json, timestamp, resolved_path) VALUES (1, 1, 1, ?, ?, ?)", + db.conn.Exec("INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (1, '', 'h1', $1)", firstSeen) + db.conn.Exec("INSERT INTO observations (id, transmission_id, observer_idx, path_json, timestamp, resolved_path) VALUES (1, 1, 1, $1, $2, $3)", `["aa"]`, now.Add(-2*time.Hour).Unix(), `["`+pk+`"]`) tx := &StoreTx{ID: 1, Hash: "h1", FirstSeen: firstSeen} @@ -1310,8 +1311,8 @@ func TestRunEviction_TriggersCompaction(t *testing.T) { // Add a fresh tx (won't be evicted) with pk shared with old tx pk2 := "shared_pk" freshSeen := now.Format(time.RFC3339) - db.conn.Exec("INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (2, '', 'h2', ?)", freshSeen) - db.conn.Exec("INSERT INTO observations (id, transmission_id, observer_idx, path_json, timestamp, resolved_path) VALUES (2, 2, 1, ?, ?, ?)", + db.conn.Exec("INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (2, '', 'h2', $1)", freshSeen) + db.conn.Exec("INSERT INTO observations (id, transmission_id, observer_idx, path_json, timestamp, resolved_path) VALUES (2, 2, 1, $1, $2, $3)", `["bb"]`, now.Unix(), `["`+pk2+`"]`) tx2 := &StoreTx{ID: 2, Hash: "h2", FirstSeen: freshSeen} @@ -1386,9 +1387,9 @@ func TestResolvedPubkeyIndex_MemoryStableThroughEvictionCycles(t *testing.T) { pk := fmt.Sprintf("pk_%d", txCounter) firstSeen := now.Add(-2 * time.Hour).Format(time.RFC3339) - db.conn.Exec("INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (?, '', ?, ?)", + db.conn.Exec("INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES ($1, '', $2, $3)", txCounter, hash, firstSeen) - db.conn.Exec("INSERT INTO observations (id, transmission_id, observer_idx, path_json, timestamp, resolved_path) VALUES (?, ?, 1, ?, ?, ?)", + db.conn.Exec("INSERT INTO observations (id, transmission_id, observer_idx, path_json, timestamp, resolved_path) VALUES ($1, $2, 1, $3, $4, $5)", txCounter, txCounter, `["aa"]`, now.Add(-2*time.Hour).Unix(), `["`+pk+`"]`) tx := &StoreTx{ID: txCounter, Hash: hash, FirstSeen: firstSeen} diff --git a/cmd/server/resolved_path_unlocked_test.go b/cmd/server/resolved_path_unlocked_test.go index ac7ab5d7a..2f332b104 100644 --- a/cmd/server/resolved_path_unlocked_test.go +++ b/cmd/server/resolved_path_unlocked_test.go @@ -16,15 +16,21 @@ func newResolvedPathStore(t *testing.T) *PacketStore { t.Helper() db := setupTestDB(t) t.Cleanup(func() { db.Close() }) + exec := func(query string, args ...any) { + t.Helper() + if _, err := db.conn.Exec(query, args...); err != nil { + t.Fatalf("seed resolved path fixture: %v", err) + } + } now := time.Now().UTC() recent := now.Add(-1 * time.Hour).Format(time.RFC3339) recentEpoch := now.Add(-1 * time.Hour).Unix() - db.conn.Exec(`INSERT INTO observers (id, name, last_seen, first_seen, packet_count) VALUES ('obs1','O1',?, '2026-01-01T00:00:00Z', 100)`, recent) - db.conn.Exec(`INSERT INTO observers (id, name, last_seen, first_seen, packet_count) VALUES ('obs2','O2',?, '2026-01-01T00:00:00Z', 100)`, recent) - db.conn.Exec(`INSERT INTO nodes (public_key, name, role, last_seen, first_seen, advert_count) VALUES ('aabbccdd11223344','R','repeater',?, '2026-01-01T00:00:00Z', 1)`, recent) - db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) VALUES ('AABB','testhash00000001',?,1,4,'{"pubKey":"aabbccdd11223344","type":"ADVERT"}')`, recent) - db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) VALUES (1,1,12.5,-90,'["aa","bb","cc"]',?)`, recentEpoch) - db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp, resolved_path) VALUES (1,2,8.0,-95,'["aa","bb"]',?,'["aabbccdd11223344","eeff00112233aabb"]')`, recentEpoch-100) + exec(`INSERT INTO observers (id, name, last_seen, first_seen, packet_count) VALUES ('obs1','O1',`+db.parameter(1)+`, '2026-01-01T00:00:00Z', 100)`, recent) + exec(`INSERT INTO observers (id, name, last_seen, first_seen, packet_count) VALUES ('obs2','O2',`+db.parameter(1)+`, '2026-01-01T00:00:00Z', 100)`, recent) + exec(`INSERT INTO nodes (public_key, name, role, last_seen, first_seen, advert_count) VALUES ('aabbccdd11223344','R','repeater',`+db.parameter(1)+`, '2026-01-01T00:00:00Z', 1)`, recent) + exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) VALUES ('AABB','testhash00000001',`+db.parameter(1)+`,1,4,'{"pubKey":"aabbccdd11223344","type":"ADVERT"}')`, recent) + exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) VALUES (1,1,12.5,-90,'["aa","bb","cc"]',`+db.parameter(1)+`)`, recentEpoch) + exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp, resolved_path) VALUES (1,2,8.0,-95,'["aa","bb"]',`+db.parameter(1)+`,'["aabbccdd11223344","eeff00112233aabb"]')`, recentEpoch-100) store := NewPacketStore(db, nil) if err := store.Load(); err != nil { diff --git a/cmd/server/rf_noise.go b/cmd/server/rf_noise.go index ef49f8f19..ac91a7c14 100644 --- a/cmd/server/rf_noise.go +++ b/cmd/server/rf_noise.go @@ -146,11 +146,11 @@ func (s *Server) requireClientRfSamples(w http.ResponseWriter, r *http.Request) // queryRfNoiseRows returns raw RF-environment samples within a bbox, over a // time window (days; 0 = all time). Read-only (server RO connection). func (s *Server) queryRfNoiseRows(days int, b bbox) ([]rfNoiseRow, error) { - where := []string{"lat BETWEEN ? AND ?", "lon BETWEEN ? AND ?", "noise_floor IS NOT NULL"} + where := []string{"lat BETWEEN " + s.db.parameter(1) + " AND " + s.db.parameter(2), "lon BETWEEN " + s.db.parameter(3) + " AND " + s.db.parameter(4), "noise_floor IS NOT NULL"} args := []interface{}{b.MinLat, b.MaxLat, b.MinLon, b.MaxLon} if days > 0 { since := time.Now().UTC().AddDate(0, 0, -days).Format(time.RFC3339) - where = append(where, "sampled_at >= ?") + where = append(where, "sampled_at >= "+s.db.parameter(5)) args = append(args, since) } rows, err := s.db.conn.Query(` diff --git a/cmd/server/rf_noise_test.go b/cmd/server/rf_noise_test.go index a3c1f3204..6b1e5e486 100644 --- a/cmd/server/rf_noise_test.go +++ b/cmd/server/rf_noise_test.go @@ -187,13 +187,19 @@ func TestRfNoiseRouteGatedOn(t *testing.T) { func seedRfNoiseDB(t *testing.T) *DB { db := setupTestDBv2(t) - mustExecDB(t, db, `CREATE TABLE client_rf_samples ( + mustExecDB(t, db, testNativeSQL(`CREATE TABLE client_rf_samples ( id INTEGER PRIMARY KEY AUTOINCREMENT, rx_pubkey TEXT, sampled_at TEXT, ingested_at TEXT, - lat REAL, lon REAL, pos_acc_m REAL, stationary INTEGER NOT NULL DEFAULT 0, + lat DOUBLE PRECISION, lon DOUBLE PRECISION, pos_acc_m DOUBLE PRECISION, stationary INTEGER NOT NULL DEFAULT 0, uptime_secs INTEGER, battery_mv INTEGER, queue_len INTEGER, errors INTEGER, - noise_floor INTEGER, last_rssi INTEGER, last_snr REAL, tx_air_secs INTEGER, + noise_floor INTEGER, last_rssi INTEGER, last_snr DOUBLE PRECISION, tx_air_secs INTEGER, rx_air_secs INTEGER, recv INTEGER, sent INTEGER, flood_rx INTEGER, direct_rx INTEGER, - flood_tx INTEGER, direct_tx INTEGER, recv_errors INTEGER)`) + flood_tx INTEGER, direct_tx INTEGER, recv_errors INTEGER)`, `CREATE TABLE client_rf_samples ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, rx_pubkey TEXT, sampled_at TEXT, ingested_at TEXT, + lat DOUBLE PRECISION, lon DOUBLE PRECISION, pos_acc_m DOUBLE PRECISION, stationary INTEGER NOT NULL DEFAULT 0, + uptime_secs INTEGER, battery_mv INTEGER, queue_len INTEGER, errors INTEGER, + noise_floor INTEGER, last_rssi INTEGER, last_snr DOUBLE PRECISION, tx_air_secs INTEGER, + rx_air_secs INTEGER, recv INTEGER, sent INTEGER, flood_rx INTEGER, direct_rx INTEGER, + flood_tx INTEGER, direct_tx INTEGER, recv_errors INTEGER)`)) return db } diff --git a/cmd/server/routes.go b/cmd/server/routes.go index f84adf2ab..09c6af754 100644 --- a/cmd/server/routes.go +++ b/cmd/server/routes.go @@ -297,7 +297,9 @@ func (s *Server) RegisterRoutes(r *mux.Router) { r.HandleFunc("/api/scope-audit", s.handleScopeAudit).Methods("GET") // #1975 r.HandleFunc("/api/perf", s.handlePerf).Methods("GET") r.HandleFunc("/api/perf/io", s.handlePerfIO).Methods("GET") - r.HandleFunc("/api/perf/sqlite", s.handlePerfSqlite).Methods("GET") + r.HandleFunc("/api/perf/database", s.handlePerfPostgres).Methods("GET") + r.HandleFunc("/api/perf/postgres", s.handlePerfPostgres).Methods("GET") + r.HandleFunc("/api/perf/sqlite", s.handlePerfPostgres).Methods("GET") // legacy diagnostic route alias r.HandleFunc("/api/perf/write-sources", s.handlePerfWriteSources).Methods("GET") r.HandleFunc("/api/mqtt/status", s.handleMqttStatus).Methods("GET") r.Handle("/api/perf/reset", s.requireAdmin(http.HandlerFunc(s.handlePerfReset))).Methods("POST") @@ -1066,22 +1068,28 @@ func (s *Server) handlePerf(w http.ResponseWriter, r *http.Request) { breakdownNote = memBreakdownNote } - // SQLite stats - var sqliteStats *SqliteStats + // PostgreSQL diagnostics reuse the bounded database sample. + var postgresStats, sqliteStats, databaseStats *PostgresStats if s.db != nil { ss := s.db.GetDBSizeStatsTyped() - sqliteStats = &ss + databaseStats = &ss + if ss.Engine == "sqlite" { + sqliteStats = &ss + } else { + postgresStats = &ss + } } writeJSON(w, PerfResponse{ - Uptime: uptimeSec, - TotalRequests: totalRequests, - AvgMs: safeAvg(totalMs, float64(totalRequests)), - Endpoints: summary, - SlowQueries: slowQueries, - Cache: perfCS, - PacketStore: pktStoreStats, - Sqlite: sqliteStats, + Uptime: uptimeSec, + TotalRequests: totalRequests, + AvgMs: safeAvg(totalMs, float64(totalRequests)), + Endpoints: summary, + SlowQueries: slowQueries, + Cache: perfCS, + PacketStore: pktStoreStats, + Postgres: postgresStats, + Sqlite: sqliteStats, Database: databaseStats, MemoryBreakdown: memBreakdown, MemoryBreakdownNote: breakdownNote, GoRuntime: func() *GoRuntimeStats { @@ -1129,7 +1137,7 @@ func (s *Server) handlePackets(w http.ResponseWriter, r *http.Request) { cleaned = append(cleaned, pk) } } - // Each entry costs one SQLite lookup (resolveNodePubkey) while the + // Each entry costs one database lookup (resolveNodePubkey) while the // packet store's read lock is held. A 1 MB URL fits ~15k pubkeys, // which is seconds of work per request, so cap the list. if len(cleaned) > maxMultiNodePubkeys { @@ -1319,7 +1327,7 @@ func (s *Server) handlePacketDetail(w http.ResponseWriter, r *http.Request) { } } } - // DB fallback: in-memory PacketStore prunes old entries, but the SQLite + // DB fallback: in-memory PacketStore prunes old entries, but the database // DB retains them and is the source for /api/nodes recentAdverts. Without // this fallback, links from node-detail pages 404 once the packet ages out. if packet == nil && s.db != nil { @@ -3987,7 +3995,7 @@ func (s *Server) handleScopeStats(w http.ResponseWriter, r *http.Request) { // geo_filter. Nodes with no GPS fix are always kept. Requires geo_filter to be // configured. // -// Since #1283/#1289 the server opens SQLite read-only, so the actual DELETE is +// Since #1283/#1289 the server opens telemetry read-only, so the actual DELETE is // performed by the ingestor. The server writes a request marker file (see // internal/prunequeue); the ingestor's maintenance loop consumes it and writes a // result marker. The confirm response is 202 Accepted with a request id; @@ -4071,13 +4079,13 @@ func (s *Server) handlePruneGeoFilter(w http.ResponseWriter, r *http.Request) { Reason: "geo-prune", Pubkeys: pubkeys, } - if err := prunequeue.WriteRequest(s.db.path, req); err != nil { + if err := prunequeue.WriteRequest(s.db.statePath(), req); err != nil { log.Printf("[geo-prune] failed to enqueue request %s: %v", id, err) writeError(w, http.StatusInternalServerError, "failed to enqueue prune request") return } log.Printf("[geo-prune] enqueued request %s for %d node(s) (queue dir=%s)", - id, len(pubkeys), prunequeue.QueueDir(s.db.path)) + id, len(pubkeys), prunequeue.QueueDir(s.db.statePath())) w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusAccepted) @@ -4103,7 +4111,7 @@ func (s *Server) handlePruneGeoFilterStatus(w http.ResponseWriter, r *http.Reque return } - res, err := prunequeue.ReadResult(s.db.path, id) + res, err := prunequeue.ReadResult(s.db.statePath(), id) if err != nil { if strings.Contains(err.Error(), "invalid prune request id") { writeError(w, http.StatusBadRequest, "invalid id") @@ -4128,7 +4136,7 @@ func (s *Server) handlePruneGeoFilterStatus(w http.ResponseWriter, r *http.Reque return } - pending, err := prunequeue.RequestExists(s.db.path, id) + pending, err := prunequeue.RequestExists(s.db.statePath(), id) if err != nil { if strings.Contains(err.Error(), "invalid prune request id") { writeError(w, http.StatusBadRequest, "invalid id") diff --git a/cmd/server/routes_test.go b/cmd/server/routes_test.go index bac94ed0a..73598a122 100644 --- a/cmd/server/routes_test.go +++ b/cmd/server/routes_test.go @@ -552,9 +552,9 @@ func TestPerfEndpoint(t *testing.T) { } // Verify sqlite stats - sqliteStats, ok := body["sqlite"].(map[string]interface{}) + sqliteStats, ok := body[testNativeSQL("sqlite", "postgres")].(map[string]interface{}) if !ok { - t.Fatal("expected sqlite object in perf response") + t.Fatal("expected postgres object in perf response") } if _, ok := sqliteStats["dbSizeMB"]; !ok { t.Error("expected dbSizeMB in sqlite") @@ -1174,10 +1174,10 @@ func TestChannelMessagesWithRegion(t *testing.T) { db := setupTestDB(t) seedTestData(t, db) db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('EEFF', 'chanextra000001', ?, 1, 5, '{"type":"CHAN","channel":"#test","text":"OtherUser: Cross region","sender":"OtherUser"}')`, + VALUES ('EEFF', 'chanextra000001', $1, 1, 5, '{"type":"CHAN","channel":"#test","text":"OtherUser: Cross region","sender":"OtherUser"}')`, time.Now().UTC().Add(-30*time.Minute).Format(time.RFC3339)) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (4, 2, 11.0, -89, '[]', ?)`, time.Now().UTC().Add(-30*time.Minute).Unix()) + VALUES (4, 2, 11.0, -89, '[]', $1)`, time.Now().UTC().Add(-30*time.Minute).Unix()) cfg := &Config{Port: 3000} hub := NewHub() @@ -2059,7 +2059,9 @@ func TestHandlerErrorPaths(t *testing.T) { srv.RegisterRoutes(router) t.Run("stats error", func(t *testing.T) { - db.conn.Exec("DROP TABLE IF EXISTS transmissions") + if _, err := db.conn.Exec(testNativeSQL("DROP TABLE IF EXISTS transmissions", "DROP TABLE IF EXISTS transmissions CASCADE")); err != nil { + t.Fatal(err) + } req := httptest.NewRequest("GET", "/api/stats", nil) w := httptest.NewRecorder() router.ServeHTTP(w, req) @@ -2078,7 +2080,9 @@ func TestHandlerErrorChannels(t *testing.T) { router := mux.NewRouter() srv.RegisterRoutes(router) - db.conn.Exec("DROP TABLE IF EXISTS transmissions") + if _, err := db.conn.Exec(testNativeSQL("DROP TABLE IF EXISTS transmissions", "DROP TABLE IF EXISTS transmissions CASCADE")); err != nil { + t.Fatal(err) + } req := httptest.NewRequest("GET", "/api/channels", nil) w := httptest.NewRecorder() @@ -2174,7 +2178,9 @@ func TestHandlerErrorPackets(t *testing.T) { srv.RegisterRoutes(router) // Drop transmissions table to trigger error in transmission-centric query - db.conn.Exec("DROP TABLE IF EXISTS transmissions") + if _, err := db.conn.Exec(testNativeSQL("DROP TABLE IF EXISTS transmissions", "DROP TABLE IF EXISTS transmissions CASCADE")); err != nil { + t.Fatal(err) + } req := httptest.NewRequest("GET", "/api/packets?limit=10", nil) w := httptest.NewRecorder() @@ -2414,7 +2420,7 @@ func TestGetNodeHashSizeInfoFlipFlop(t *testing.T) { store.WaitIndexesReady(5 * time.Second) pk := "abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890" - db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role) VALUES (?, 'TestNode', 'repeater')", pk) + db.conn.Exec("INSERT INTO nodes (public_key, name, role) VALUES ($1, 'TestNode', 'repeater') ON CONFLICT DO NOTHING", pk) decoded := `{"name":"TestNode","pubKey":"` + pk + `"}` raw1 := "11" + "01" + "aabb" @@ -2463,7 +2469,7 @@ func TestGetNodeHashSizeInfoDominant(t *testing.T) { store.WaitIndexesReady(5 * time.Second) pk := "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" - db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role) VALUES (?, 'Repeater2B', 'repeater')", pk) + db.conn.Exec("INSERT INTO nodes (public_key, name, role) VALUES ($1, 'Repeater2B', 'repeater') ON CONFLICT DO NOTHING", pk) decoded := `{"name":"Repeater2B","pubKey":"` + pk + `"}` raw1byte := "11" + "01" + "aabb" // FLOOD, pathByte=0x01 → hashSize=1 @@ -2507,7 +2513,7 @@ func TestGetNodeHashSizeInfoLatestWins(t *testing.T) { store.WaitIndexesReady(5 * time.Second) pk := "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" - db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role) VALUES (?, 'LatestWins', 'repeater')", pk) + db.conn.Exec("INSERT INTO nodes (public_key, name, role) VALUES ($1, 'LatestWins', 'repeater') ON CONFLICT DO NOTHING", pk) decoded := `{"name":"LatestWins","pubKey":"` + pk + `"}` raw1byte := "11" + "01" + "aabb" // FLOOD, pathByte=0x01 → hashSize=1 @@ -2557,7 +2563,7 @@ func TestGetNodeHashSizeInfoIgnoreDirectZeroHop(t *testing.T) { store.WaitIndexesReady(5 * time.Second) pk := "dddd111122223333444455556666777788889999aaaabbbbccccddddeeee3333" - db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role) VALUES (?, 'DirIgnore', 'repeater')", pk) + db.conn.Exec("INSERT INTO nodes (public_key, name, role) VALUES ($1, 'DirIgnore', 'repeater') ON CONFLICT DO NOTHING", pk) decoded := `{"name":"DirIgnore","pubKey":"` + pk + `"}` rawFlood2B := "11" + "40" + "aabb" // FLOOD advert, hashSize=2 @@ -2605,7 +2611,7 @@ func TestGetNodeHashSizeInfoOnlyDirectZeroHopIgnored(t *testing.T) { store.WaitIndexesReady(5 * time.Second) pk := "eeee111122223333444455556666777788889999aaaabbbbccccddddeeee4444" - db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role) VALUES (?, 'OnlyDirect', 'repeater')", pk) + db.conn.Exec("INSERT INTO nodes (public_key, name, role) VALUES ($1, 'OnlyDirect', 'repeater') ON CONFLICT DO NOTHING", pk) decoded := `{"name":"OnlyDirect","pubKey":"` + pk + `"}` rawDirect0 := "12" + "00" + "aabb" @@ -2640,7 +2646,7 @@ func TestGetNodeHashSizeInfoDirectNonZeroHopCounted(t *testing.T) { store.WaitIndexesReady(5 * time.Second) pk := "ffff111122223333444455556666777788889999aaaabbbbccccddddeeee5555" - db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role) VALUES (?, 'DirNonZero', 'repeater')", pk) + db.conn.Exec("INSERT INTO nodes (public_key, name, role) VALUES ($1, 'DirNonZero', 'repeater') ON CONFLICT DO NOTHING", pk) decoded := `{"name":"DirNonZero","pubKey":"` + pk + `"}` // DIRECT advert (route type 2 = 0x02 in bits 0-1), path byte 0x41: @@ -2680,7 +2686,7 @@ func TestGetNodeHashSizeInfoNoAdverts(t *testing.T) { store.WaitIndexesReady(5 * time.Second) pk := "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" - db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role) VALUES (?, 'NoAdverts', 'repeater')", pk) + db.conn.Exec("INSERT INTO nodes (public_key, name, role) VALUES ($1, 'NoAdverts', 'repeater') ON CONFLICT DO NOTHING", pk) // Add a non-advert packet (payload_type=2 = TXT_MSG) payloadType := 2 @@ -2720,7 +2726,7 @@ func TestHashAnalyticsZeroHopAdvert(t *testing.T) { baseDist1 := baseDist["1"] pk := "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" - db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role) VALUES (?, 'ZeroHop', 'repeater')", pk) + db.conn.Exec("INSERT INTO nodes (public_key, name, role) VALUES ($1, 'ZeroHop', 'repeater') ON CONFLICT DO NOTHING", pk) store.InvalidateNodeCache() decoded := `{"name":"ZeroHop","pubKey":"` + pk + `"}` @@ -2777,8 +2783,8 @@ func TestAnalyticsHashSizeSameNameDifferentPubkey(t *testing.T) { pk2 := "aaaa111122223333444455556666777788889999aaaabbbbccccddddeeee2222" // Insert both nodes as repeaters so they appear in distributionByRepeaters. - db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role) VALUES (?, 'SameName', 'repeater')", pk1) - db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role) VALUES (?, 'SameName', 'repeater')", pk2) + db.conn.Exec("INSERT INTO nodes (public_key, name, role) VALUES ($1, 'SameName', 'repeater') ON CONFLICT DO NOTHING", pk1) + db.conn.Exec("INSERT INTO nodes (public_key, name, role) VALUES ($1, 'SameName', 'repeater') ON CONFLICT DO NOTHING", pk2) store.InvalidateNodeCache() decoded1 := `{"name":"SameName","pubKey":"` + pk1 + `"}` @@ -2856,7 +2862,7 @@ func TestInconsistentNodesExcludesCompanions(t *testing.T) { } for ni, n := range nodes { - db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role) VALUES (?, ?, ?)", n.pk, "Node-"+n.role, n.role) + db.conn.Exec("INSERT INTO nodes (public_key, name, role) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING", n.pk, "Node-"+n.role, n.role) decoded := `{"name":"Node-` + n.role + `","pubKey":"` + n.pk + `"}` // Create flip-flop pattern: 1-byte, 2-byte, 1-byte (transitions=2 → inconsistent) // Use header 0x11 (routeType=FLOOD, payloadType=4) and pathByte 0x41/0x81 @@ -2927,7 +2933,7 @@ func TestHashSizeInfoTimeWindow(t *testing.T) { store.WaitIndexesReady(5 * time.Second) pk := "dd44444444444444444444444444444444444444444444444444444444444444" - db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role) VALUES (?, 'OldNode', 'repeater')", pk) + db.conn.Exec("INSERT INTO nodes (public_key, name, role) VALUES ($1, 'OldNode', 'repeater') ON CONFLICT DO NOTHING", pk) decoded := `{"name":"OldNode","pubKey":"` + pk + `"}` payloadType := 4 @@ -2958,7 +2964,7 @@ func TestHashSizeInfoTimeWindow(t *testing.T) { // Now add recent adverts with consistent hash size — should appear in info pk2 := "ee55555555555555555555555555555555555555555555555555555555555555" - db.conn.Exec("INSERT OR IGNORE INTO nodes (public_key, name, role) VALUES (?, 'NewNode', 'repeater')", pk2) + db.conn.Exec("INSERT INTO nodes (public_key, name, role) VALUES ($1, 'NewNode', 'repeater') ON CONFLICT DO NOTHING", pk2) decoded2 := `{"name":"NewNode","pubKey":"` + pk2 + `"}` recentTime := time.Now().UTC().Format("2006-01-02T15:04:05.000Z") for i := 0; i < 3; i++ { @@ -3139,23 +3145,23 @@ func TestQueryGroupedPacketsSortedByLatest(t *testing.T) { oldEpoch := now.Add(-72 * time.Hour).Unix() db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('sortobs', 'Sort Observer', 'TST', ?, '2026-01-01T00:00:00Z', 1)`, now.Format(time.RFC3339)) + VALUES ('sortobs', 'Sort Observer', 'TST', $1, '2026-01-01T00:00:00Z', 1)`, now.Format(time.RFC3339)) // Packet A: old first_seen, but a very recent observation — should sort first. db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('AA01', 'sort_old_first_recent_obs', ?, 1, 2, '{"type":"TXT_MSG","text":"old first"}')`, oldFirst) + VALUES ('AA01', 'sort_old_first_recent_obs', $1, 1, 2, '{"type":"TXT_MSG","text":"old first"}')`, oldFirst) var idA int64 db.conn.QueryRow(`SELECT id FROM transmissions WHERE hash='sort_old_first_recent_obs'`).Scan(&idA) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (?, 1, 10.0, -90, '[]', ?)`, idA, recentEpoch) + VALUES ($1, 1, 10.0, -90, '[]', $2)`, idA, recentEpoch) // Packet B: newer first_seen, but an old observation — should sort second. db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('BB02', 'sort_new_first_old_obs', ?, 1, 2, '{"type":"TXT_MSG","text":"new first"}')`, newFirst) + VALUES ('BB02', 'sort_new_first_old_obs', $1, 1, 2, '{"type":"TXT_MSG","text":"new first"}')`, newFirst) var idB int64 db.conn.QueryRow(`SELECT id FROM transmissions WHERE hash='sort_new_first_old_obs'`).Scan(&idB) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (?, 1, 10.0, -90, '[]', ?)`, idB, oldEpoch) + VALUES ($1, 1, 10.0, -90, '[]', $2)`, idB, oldEpoch) store := NewPacketStore(db, nil) if err := store.Load(); err != nil { @@ -3592,9 +3598,9 @@ func TestHashCollisionsWithCollision(t *testing.T) { // Two repeater nodes with same first byte 'CC' and hash_size=1 db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES ('CC11223344556677', 'Node1', 'repeater', 37.5, -122.0, ?, '2026-01-01T00:00:00Z', 5)`, recent) + VALUES ('CC11223344556677', 'Node1', 'repeater', 37.5, -122.0, $1, '2026-01-01T00:00:00Z', 5)`, recent) db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES ('CC99887766554433', 'Node2', 'repeater', 37.51, -122.01, ?, '2026-01-01T00:00:00Z', 5)`, recent) + VALUES ('CC99887766554433', 'Node2', 'repeater', 37.51, -122.01, $1, '2026-01-01T00:00:00Z', 5)`, recent) cfg := &Config{Port: 3000} hub := NewHub() @@ -3661,7 +3667,7 @@ func TestHashCollisionsShortPublicKey(t *testing.T) { recent := now.Add(-1 * time.Hour).Format(time.RFC3339) db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES ('A', 'ShortKey', 'repeater', 0, 0, ?, '2026-01-01T00:00:00Z', 1)`, recent) + VALUES ('A', 'ShortKey', 'repeater', 0, 0, $1, '2026-01-01T00:00:00Z', 1)`, recent) cfg := &Config{Port: 3000} hub := NewHub() @@ -3692,9 +3698,9 @@ func TestHashCollisionsMissingCoordinates(t *testing.T) { // Two nodes same prefix, no coordinates db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES ('BB11223344556677', 'NoCoords1', 'repeater', 0, 0, ?, '2026-01-01T00:00:00Z', 1)`, recent) + VALUES ('BB11223344556677', 'NoCoords1', 'repeater', 0, 0, $1, '2026-01-01T00:00:00Z', 1)`, recent) db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES ('BB99887766554433', 'NoCoords2', 'repeater', 0, 0, ?, '2026-01-01T00:00:00Z', 1)`, recent) + VALUES ('BB99887766554433', 'NoCoords2', 'repeater', 0, 0, $1, '2026-01-01T00:00:00Z', 1)`, recent) cfg := &Config{Port: 3000} hub := NewHub() @@ -3744,16 +3750,16 @@ func TestHashCollisionsOnlyRepeaters(t *testing.T) { // 5. sensor with hash_size=1 → should be excluded now := time.Now().Format("2006-01-02 15:04:05") db.conn.Exec(`INSERT INTO nodes (public_key, name, role, last_seen) VALUES - ('aa11223344556677', 'Repeater1', 'repeater', ?), - ('aa99887766554433', 'UnknownNode', 'repeater', ?), - ('aadeadbeefcafe01', 'Companion1', 'companion', ?), - ('aabbcc1122334455', 'Room1', 'room', ?), - ('aabbcc9988776655', 'Sensor1', 'sensor', ?)`, now, now, now, now, now) + ('aa11223344556677', 'Repeater1', 'repeater', $1), + ('aa99887766554433', 'UnknownNode', 'repeater', $2), + ('aadeadbeefcafe01', 'Companion1', 'companion', $3), + ('aabbcc1122334455', 'Room1', 'room', $4), + ('aabbcc9988776655', 'Sensor1', 'sensor', $5)`, now, now, now, now, now) // We also need a second repeater with hash_size=1 and same prefix to // confirm that genuine collisions ARE still detected. db.conn.Exec(`INSERT INTO nodes (public_key, name, role, last_seen) VALUES - ('aa00112233445566', 'Repeater2', 'repeater', ?)`, now) + ('aa00112233445566', 'Repeater2', 'repeater', $1)`, now) cfg := &Config{Port: 3000} hub := NewHub() @@ -3828,10 +3834,10 @@ func TestHashCollisionsOneByteIncludesMultiBytePrefixRepeaters(t *testing.T) { // One unrelated repeater with first byte "DD" must NOT appear in CC cell. now := time.Now().Format("2006-01-02 15:04:05") db.conn.Exec(`INSERT INTO nodes (public_key, name, role, last_seen) VALUES - ('cc11223344556677', 'Rep1B', 'repeater', ?), - ('cc22aabbccddeeff', 'Rep2B', 'repeater', ?), - ('cc33bbddeeff0011', 'Rep3B', 'repeater', ?), - ('dd44556677889900', 'RepDD', 'repeater', ?)`, now, now, now, now) + ('cc11223344556677', 'Rep1B', 'repeater', $1), + ('cc22aabbccddeeff', 'Rep2B', 'repeater', $2), + ('cc33bbddeeff0011', 'Rep3B', 'repeater', $3), + ('dd44556677889900', 'RepDD', 'repeater', $4)`, now, now, now, now) cfg := &Config{Port: 3000} hub := NewHub() @@ -3937,18 +3943,18 @@ func TestNodePathsPrefixCollisionFilter(t *testing.T) { recentEpoch := now.Add(-30 * time.Minute).Unix() // Insert a second node with the same 2-char prefix - srv.db.conn.Exec(`INSERT OR IGNORE INTO nodes (public_key, name, role, last_seen, first_seen, advert_count) - VALUES ('aacafe0000000000', 'CollisionNode', 'repeater', ?, '2026-01-01T00:00:00Z', 5)`, recent) + srv.db.conn.Exec(`INSERT INTO nodes (public_key, name, role, last_seen, first_seen, advert_count) + VALUES ('aacafe0000000000', 'CollisionNode', 'repeater', $1, '2026-01-01T00:00:00Z', 5) ON CONFLICT DO NOTHING`, recent) // Insert a transmission with path hop "aa" that resolves to the OTHER node srv.db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('FF01', 'collision_test_hash', ?, 1, 4, '{}')`, recent) + VALUES ('FF01', 'collision_test_hash', $1, 1, 4, '{}')`, recent) // Get its ID var collisionTxID int srv.db.conn.QueryRow(`SELECT id FROM transmissions WHERE hash='collision_test_hash'`).Scan(&collisionTxID) srv.db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp, resolved_path) - VALUES (?, 1, 10.0, -90, '["aa","bb"]', ?, '["aacafe0000000000","eeff00112233aabb"]')`, + VALUES ($1, 1, 10.0, -90, '["aa","bb"]', $2, '["aacafe0000000000","eeff00112233aabb"]')`, collisionTxID, recentEpoch) // Reload store to pick up new data @@ -4088,7 +4094,7 @@ func TestMetricsAPIEndpoints(t *testing.T) { now := time.Now().UTC() t1 := now.Add(-1 * time.Hour).Format(time.RFC3339) - srv.db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor) VALUES (?, ?, ?)", + srv.db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor) VALUES ($1, $2, $3)", "obs1", t1, -112.0) // Test /api/observers/obs1/metrics @@ -4217,16 +4223,16 @@ func TestPacketDetailFallsBackToDBWhenStoreMisses(t *testing.T) { now := time.Now().UTC().Format(time.RFC3339) if _, err := srv.db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('FFEE', ?, ?, 1, 4, '{"type":"ADVERT"}')`, dbOnlyHash, now); err != nil { + VALUES ('FFEE', $1, $2, 1, 4, '{"type":"ADVERT"}')`, dbOnlyHash, now); err != nil { t.Fatalf("insert: %v", err) } var txID int - if err := srv.db.conn.QueryRow("SELECT id FROM transmissions WHERE hash = ?", dbOnlyHash).Scan(&txID); err != nil { + if err := srv.db.conn.QueryRow("SELECT id FROM transmissions WHERE hash = $1", dbOnlyHash).Scan(&txID); err != nil { t.Fatalf("lookup tx id: %v", err) } if _, err := srv.db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (?, 1, 7.5, -99, '[]', ?)`, txID, time.Now().Unix()); err != nil { + VALUES ($1, 1, 7.5, -99, '[]', $2)`, txID, time.Now().Unix()); err != nil { t.Fatalf("insert obs: %v", err) } @@ -4307,7 +4313,7 @@ func TestHandleScopeStats(t *testing.T) { srv.db.hasScopeName = true // Clear seed transmissions so this test isolates scope-stats math. - if _, err := srv.db.conn.Exec(`DELETE FROM transmissions`); err != nil { + if _, err := srv.db.conn.Exec(`DELETE FROM observations; DELETE FROM transmissions`); err != nil { t.Fatalf("clear transmissions: %v", err) } @@ -4331,7 +4337,7 @@ func TestHandleScopeStats(t *testing.T) { scopeArg = nil // unscoped (NULL) } if _, err := srv.db.conn.Exec( - `INSERT INTO transmissions (raw_hex,hash,first_seen,route_type,payload_type,scope_name) VALUES (?,?,?,?,5,?)`, + `INSERT INTO transmissions (raw_hex,hash,first_seen,route_type,payload_type,scope_name) VALUES ($1,$2,$3,$4,5,$5)`, "aa", r.hash, now, r.route, scopeArg, ); err != nil { t.Fatalf("seed row %d: %v", i, err) @@ -4697,7 +4703,7 @@ func TestPruneGeoFilterEndpoint(t *testing.T) { } // And the marker file must exist on disk. - pending, err := prunequeue.RequestExists(srv.db.path, id) + pending, err := prunequeue.RequestExists(srv.db.statePath(), id) if err != nil { t.Fatalf("RequestExists: %v", err) } @@ -4735,7 +4741,7 @@ func TestPruneGeoFilterEndpoint(t *testing.T) { } // Simulate the ingestor completing the request. - if err := prunequeue.WriteResult(srv.db.path, prunequeue.Result{ + if err := prunequeue.WriteResult(srv.db.statePath(), prunequeue.Result{ ID: id, RequestedAt: time.Now().Add(-1 * time.Second).UTC(), CompletedAt: time.Now().UTC(), diff --git a/cmd/server/row_scan_allocs_test.go b/cmd/server/row_scan_allocs_test.go index 2e9b32dc8..de8610d26 100644 --- a/cmd/server/row_scan_allocs_test.go +++ b/cmd/server/row_scan_allocs_test.go @@ -1,8 +1,6 @@ package main import ( - "os" - "path/filepath" "runtime" "testing" "time" @@ -19,11 +17,9 @@ import ( func rowScanStore(t *testing.T, n int) *PacketStore { t.Helper() - dir := t.TempDir() - dbPath := filepath.Join(dir, "rows.db") + dbPath := postgresTestDSN(t) createTestDBAt(t, dbPath, n) - t.Cleanup(func() { os.RemoveAll(dir) }) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatalf("OpenDB: %v", err) } diff --git a/cmd/server/runstartup_load_test.go b/cmd/server/runstartup_load_test.go index fa7f22f7c..123411ced 100644 --- a/cmd/server/runstartup_load_test.go +++ b/cmd/server/runstartup_load_test.go @@ -19,7 +19,6 @@ package main import ( "fmt" - "path/filepath" "strings" "sync" "sync/atomic" @@ -31,13 +30,12 @@ import ( // when hotStartupHours == 0 the bg loader has no work to do; healthz // must NOT be stuck on backgroundLoadComplete=false. func TestRunStartupLoad_HotStartupHoursZero_SetsDoneImmediately(t *testing.T) { - dir := t.TempDir() - dbPath := filepath.Join(dir, "test.db") + dbPath := postgresTestDSN(t) nowSec := time.Now().UTC().Unix() createTestDBWithLastSeen(t, dbPath, 10, 1, nowSec, 30*time.Minute, 30*time.Minute) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatalf("OpenDB: %v", err) } @@ -67,13 +65,12 @@ func TestRunStartupLoad_HotStartupHoursZero_SetsDoneImmediately(t *testing.T) { // implementation left done=false, leaving healthz wedged on the // failure path (dij #1 / adv #7). func TestRunStartupLoad_LoadChunkedError_SetsFailedTerminal(t *testing.T) { - dir := t.TempDir() - dbPath := filepath.Join(dir, "test.db") + dbPath := postgresTestDSN(t) nowSec := time.Now().UTC().Unix() createTestDBWithLastSeen(t, dbPath, 5, 1, nowSec, 30*time.Minute, 30*time.Minute) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatalf("OpenDB: %v", err) } @@ -107,12 +104,11 @@ func TestRunStartupLoad_LoadChunkedError_SetsFailedTerminal(t *testing.T) { // loadBackgroundChunks must reach its coverage block (totalInDB==0 → // ratio=1.0) and set done=true rather than leaving the store stuck. func TestRunStartupLoad_EmptyDB_SetsDoneTerminal(t *testing.T) { - dir := t.TempDir() - dbPath := filepath.Join(dir, "test.db") + dbPath := postgresTestDSN(t) createTestDBWithLastSeen(t, dbPath, 0, 0, time.Now().UTC().Unix(), 30*time.Minute, 30*time.Minute) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatalf("OpenDB: %v", err) } @@ -152,13 +148,12 @@ func TestRunStartupLoad_EmptyDB_SetsDoneTerminal(t *testing.T) { // tBgEntry >= tLastChunk monotonically. The runtime invariant in // loadBackgroundChunks remains as the deterministic backstop. func TestRunStartupLoad_BgLoaderRunsAfterLoadChunkedSets_OldestLoaded(t *testing.T) { - dir := t.TempDir() - dbPath := filepath.Join(dir, "test.db") + dbPath := postgresTestDSN(t) nowSec := time.Now().UTC().Unix() createTestDBWithLastSeen(t, dbPath, 50, 1, nowSec, 30*time.Minute, 30*time.Minute) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatalf("OpenDB: %v", err) } @@ -223,14 +218,13 @@ func TestRunStartupLoad_BgLoaderRunsAfterLoadChunkedSets_OldestLoaded(t *testing // fails and the terminal state is still failed=true,done=true with a // non-empty error. func TestRunStartupLoad_LoadChunkedError_MidLoad(t *testing.T) { - dir := t.TempDir() - dbPath := filepath.Join(dir, "test.db") + dbPath := postgresTestDSN(t) nowSec := time.Now().UTC().Unix() // Seed enough rows so a small chunk size yields >1 chunk. createTestDBWithLastSeen(t, dbPath, 200, 1, nowSec, 30*time.Minute, 30*time.Minute) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatalf("OpenDB: %v", err) } @@ -270,8 +264,7 @@ func TestRunStartupLoad_LoadChunkedError_MidLoad(t *testing.T) { // must fire so future refactors cannot silently re-introduce the // #1809 race. func TestLoadBackgroundChunks_PanicsOnOldestLoadedEmpty_Invariant(t *testing.T) { - dir := t.TempDir() - dbPath := filepath.Join(dir, "test.db") + dbPath := postgresTestDSN(t) // Reuse the existing schema-only fixture helper (0 rows) so this // test does not introduce a new inline CREATE TABLE block (pr-preflight // async-migration gate). The fixture provides exactly the bare schema @@ -279,7 +272,7 @@ func TestLoadBackgroundChunks_PanicsOnOldestLoadedEmpty_Invariant(t *testing.T) createTestDBWithLastSeen(t, dbPath, 0, 0, time.Now().UTC().Unix(), 30*time.Minute, 30*time.Minute) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatalf("OpenDB: %v", err) } diff --git a/cmd/server/rx_coverage.go b/cmd/server/rx_coverage.go index c5b98bf04..02af92dc6 100644 --- a/cmd/server/rx_coverage.go +++ b/cmd/server/rx_coverage.go @@ -2,6 +2,7 @@ package main import ( "encoding/json" + "fmt" "net/http" "sort" "strconv" @@ -261,12 +262,18 @@ func prefixOrEmpty(s string, n int) string { return "" } -// sqlPlaceholders returns "?,?,…" with n placeholders (n >= 1). -func sqlPlaceholders(n int) string { - if n <= 1 { - return "?" +// sqlPlaceholders builds native PostgreSQL positional parameters for an IN-list. +// The optional first index supports lists following other bound predicates. +func (db *DB) sqlPlaceholders(n int, first ...int) string { + start := 1 + if len(first) > 0 { + start = first[0] } - return strings.Repeat("?,", n-1) + "?" + parts := make([]string, n) + for i := range parts { + parts[i] = fmt.Sprintf("%s", db.parameter(start+i)) + } + return strings.Join(parts, ",") } // queryCoverageRows returns raw coverage rows where the directly-heard node @@ -279,11 +286,14 @@ func (s *Server) queryCoverageRows(pubkey string, b bbox) ([]coverageRow, error) args = append(args, c) } args = append(args, b.MinLat, b.MaxLat, b.MinLon, b.MaxLon) + // A cached generic plan can treat a wide bbox as selective and scan the + // entire geo index before filtering the heard key. Plan this SELECT with + // its actual bounds; Exec still uses server-bound extended-protocol values. rows, err := s.db.conn.Query(` SELECT lat, lon, snr, rssi, heard_key, rx_at FROM client_receptions - WHERE heard_key IN (`+sqlPlaceholders(len(cands))+`) - AND lat BETWEEN ? AND ? AND lon BETWEEN ? AND ?`, args...) + WHERE heard_key IN (`+s.db.sqlPlaceholders(len(cands))+`) + AND lat BETWEEN `+fmt.Sprintf("%s AND %s AND lon BETWEEN %s AND %s", s.db.parameter(len(cands)+1), s.db.parameter(len(cands)+2), s.db.parameter(len(cands)+3), s.db.parameter(len(cands)+4)), s.db.planWithValues(args)...) if err != nil { return nil, err } @@ -304,7 +314,7 @@ func (s *Server) mobileRxStats(pubkey string) (count, clients int) { } s.db.conn.QueryRow(` SELECT COUNT(*), COUNT(DISTINCT rx_pubkey) FROM client_receptions - WHERE heard_key IN (`+sqlPlaceholders(len(cands))+`)`, args...).Scan(&count, &clients) + WHERE heard_key IN (`+s.db.sqlPlaceholders(len(cands))+`)`, args...).Scan(&count, &clients) return count, clients } diff --git a/cmd/server/rx_coverage_endpoint_test.go b/cmd/server/rx_coverage_endpoint_test.go index f5fa92a83..a2a65d2af 100644 --- a/cmd/server/rx_coverage_endpoint_test.go +++ b/cmd/server/rx_coverage_endpoint_test.go @@ -10,9 +10,11 @@ import ( func seedCoverageDB(t *testing.T) *DB { db := setupTestDBv2(t) - mustExecDB(t, db, `CREATE TABLE client_receptions ( + mustExecDB(t, db, testNativeSQL(`CREATE TABLE client_receptions ( id INTEGER PRIMARY KEY AUTOINCREMENT, rx_pubkey TEXT, heard_key TEXT, heard_keylen INTEGER, - rssi INTEGER, snr REAL, lat REAL, lon REAL, pos_acc_m REAL, rx_at TEXT, ingested_at TEXT, src TEXT)`) + rssi INTEGER, snr DOUBLE PRECISION, lat DOUBLE PRECISION, lon DOUBLE PRECISION, pos_acc_m DOUBLE PRECISION, rx_at TEXT, ingested_at TEXT, src TEXT)`, `CREATE TABLE client_receptions ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, rx_pubkey TEXT, heard_key TEXT, heard_keylen INTEGER, + rssi INTEGER, snr DOUBLE PRECISION, lat DOUBLE PRECISION, lon DOUBLE PRECISION, pos_acc_m DOUBLE PRECISION, rx_at TEXT, ingested_at TEXT, src TEXT)`)) mustExecDB(t, db, `CREATE TABLE client_observers (pubkey TEXT PRIMARY KEY, name TEXT, last_seen TEXT)`) return db } diff --git a/cmd/server/rx_coverage_gaps.go b/cmd/server/rx_coverage_gaps.go index 1d99cd2cd..2d7ed83fe 100644 --- a/cmd/server/rx_coverage_gaps.go +++ b/cmd/server/rx_coverage_gaps.go @@ -87,14 +87,14 @@ func aggregateGaps(track []trackPoint, receptions []coverageRow, res int) (gaps // layer it keeps rows without a noise_floor reading: any sample proves the // companion was connected there. Read-only. func (s *Server) queryTrackPoints(rx string, days int, b bbox) ([]trackPoint, error) { - where := []string{"lat BETWEEN ? AND ?", "lon BETWEEN ? AND ?", "stationary = 0"} + where := []string{"lat BETWEEN " + s.db.parameter(1) + " AND " + s.db.parameter(2), "lon BETWEEN " + s.db.parameter(3) + " AND " + s.db.parameter(4), "stationary = 0"} args := []interface{}{b.MinLat, b.MaxLat, b.MinLon, b.MaxLon} if rx != "" { - where = append(where, "rx_pubkey = ?") + where = append(where, "rx_pubkey = "+s.db.parameter(len(args)+1)) args = append(args, strings.ToLower(rx)) } if days > 0 { - where = append(where, "sampled_at >= ?") + where = append(where, "sampled_at >= "+s.db.parameter(len(args)+1)) args = append(args, time.Now().UTC().AddDate(0, 0, -days).Format(time.RFC3339)) } rows, err := s.db.conn.Query("SELECT lat, lon FROM client_rf_samples WHERE "+strings.Join(where, " AND "), args...) diff --git a/cmd/server/rx_coverage_gaps_test.go b/cmd/server/rx_coverage_gaps_test.go index 0da5dd52e..d4eaef339 100644 --- a/cmd/server/rx_coverage_gaps_test.go +++ b/cmd/server/rx_coverage_gaps_test.go @@ -92,7 +92,7 @@ func TestAggregateGapsCapsKeepsMostSampled(t *testing.T) { func seedGapsDB(t *testing.T) *DB { db := seedCoverageDB(t) mustExecDB(t, db, `CREATE TABLE client_rf_samples ( - id INTEGER PRIMARY KEY AUTOINCREMENT, rx_pubkey TEXT, sampled_at TEXT, ingested_at TEXT, + id `+testNativeSQL("INTEGER PRIMARY KEY AUTOINCREMENT", "BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY")+`, rx_pubkey TEXT, sampled_at TEXT, ingested_at TEXT, lat REAL, lon REAL, pos_acc_m REAL, stationary INTEGER NOT NULL DEFAULT 0, uptime_secs INTEGER, battery_mv INTEGER, queue_len INTEGER, errors INTEGER, noise_floor INTEGER, last_rssi INTEGER, last_snr REAL, tx_air_secs INTEGER, diff --git a/cmd/server/rx_coverage_postgres_plan_test.go b/cmd/server/rx_coverage_postgres_plan_test.go new file mode 100644 index 000000000..cee157b84 --- /dev/null +++ b/cmd/server/rx_coverage_postgres_plan_test.go @@ -0,0 +1,116 @@ +package main + +import ( + "crypto/ed25519" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "reflect" + "sort" + "testing" + "time" +) + +// A generic plan estimates an unknown bbox as tiny. With a wide viewport it +// instead visits all 128K receptions through the geo index, rejecting 127936 +// rows after fetching them. The first five custom plans hide this regression. +func TestCoverageQueriesKeepValueSpecificPlans(t *testing.T) { + postgresOnly(t) + dsn := postgresTestDSN(t) + writer, err := openFixtureSQL(dsn) + if err != nil { + t.Fatal(err) + } + defer writer.Close() + if err := applyTestSchema(t, writer); err != nil { + t.Fatal(err) + } + keys := make([]string, 2000) + for i := range keys { + seed := sha256.Sum256([]byte(fmt.Sprintf("corescope-bench/%d/node/%d", 20261008, i))) + key := ed25519.NewKeyFromSeed(seed[:]) + keys[i] = hex.EncodeToString(key.Public().(ed25519.PublicKey)) + } + copyTestRows(t, writer, "nodes", []string{"public_key", "name"}, len(keys), func(i int) []any { + return []any{keys[i], fmt.Sprintf("Synthetic-%04d", i)} + }) + stamp := time.Now().UTC().Format("2006-01-02T15:04:05") + copyTestRows(t, writer, "client_receptions", []string{"rx_pubkey", "heard_key", "heard_keylen", "rssi", "snr", "lat", "lon", "rx_at", "ingested_at", "src"}, 128000, func(i int) []any { + size := []int{32, 8, 3, 2}[i%4] + var snr, rssi any = 6.5, -95 + if i == 0 { + snr, rssi = nil, nil + } + lat, lon := 20+float64(i%2000)/10000, 30+float64(i%2000)/10000 + if i == 2000 { + lat, lon = 20.0123456789, 30.07654321 + } + return []any{keys[(i+100)%len(keys)], keys[i%len(keys)][:size*2], size, rssi, snr, lat, lon, fmt.Sprintf("%s.%09dZ", stamp, i), stamp + "Z", "synthetic"} + }) + if _, err := writer.Exec("ANALYZE nodes; ANALYZE client_receptions"); err != nil { + t.Fatal(err) + } + db, err := openFixtureReader(t, dsn) + if err != nil { + t.Fatal(err) + } + defer db.Close() + // Keep this test on one session so pg_prepared_statements observes every + // execution. Production pools retain the same per-connection plan behavior. + db.conn.SetMaxOpenConns(1) + srv := &Server{db: db, cfg: &Config{ClientRxCoverage: &ClientRxCoverageConfig{Enabled: true}}} + for _, tc := range []struct { + name, pattern string + query func(bbox) ([]coverageRow, error) + }{ + {"node", "%WHERE heard_key IN%", func(b bbox) ([]coverageRow, error) { return srv.queryCoverageRows(keys[0], b) }}, + {"dashboard", "%WHERE lat BETWEEN%", func(b bbox) ([]coverageRow, error) { return srv.queryCoverageFiltered(keys[0], "", nil, 0, b) }}, + } { + t.Run(tc.name, func(t *testing.T) { + var first []coverageRow + for n := 0; n < 10; n++ { + rows, err := tc.query(bbox{19, 29, 22, 32}) + if err != nil || len(rows) != 64 { + t.Fatalf("pass %d: rows=%d, err=%v", n, len(rows), err) + } + sort.Slice(rows, func(i, j int) bool { return rows[i].RxAt < rows[j].RxAt }) + if n == 0 { + first = rows + } else if !reflect.DeepEqual(rows, first) { + t.Fatal("repeated coverage query changed its rows") + } + } + if first[0].SNR != nil || first[0].RSSI != nil || first[1].SNR == nil || *first[1].SNR != 6.5 || first[1].RSSI == nil || *first[1].RSSI != -95 || first[1].Lat != 20.0123456789 || first[1].Lon != 30.07654321 { + t.Fatalf("coverage null/numeric values changed: %+v", first[:2]) + } + if rows, err := tc.query(bbox{20, 30, 20, 30}); err != nil || len(rows) != 63 { + t.Fatalf("inclusive narrow bbox: rows=%d, err=%v", len(rows), err) + } + if rows, err := tc.query(bbox{0, 0, 1, 1}); err != nil || len(rows) != 0 { + t.Fatalf("empty bbox: rows=%d, err=%v", len(rows), err) + } + var generic int64 + if err := db.conn.QueryRow(`SELECT COALESCE(SUM(generic_plans),0)::bigint FROM pg_prepared_statements + WHERE statement LIKE $1 AND statement LIKE $2 AND statement LIKE $3`, + "%SELECT lat, lon, snr, rssi, heard_key, rx_at%", "%FROM client_receptions%", tc.pattern).Scan(&generic); err != nil { + t.Fatal(err) + } + if generic != 0 { + t.Fatalf("bbox-dependent coverage SELECT reused %d generic plans; broad views must retain value-specific planning", generic) + } + }) + } + rr := serveRxCoverage(srv, "/api/nodes/"+keys[0]+"/rx-coverage?bbox=19,29,22,32&z=14") + var fc CoverageFeatureCollection + if rr.Code != 200 || json.Unmarshal(rr.Body.Bytes(), &fc) != nil { + t.Fatalf("coverage API status=%d", rr.Code) + } + count := 0 + for _, feature := range fc.Features { + count += feature.Properties.Count + } + if count != 64 || fc.MobileReceptions != 64 || fc.MobileClients != 1 { + t.Fatalf("coverage API totals changed: cells=%d node=%d clients=%d", count, fc.MobileReceptions, fc.MobileClients) + } +} diff --git a/cmd/server/rx_dashboard.go b/cmd/server/rx_dashboard.go index b5c04a942..6428cb2de 100644 --- a/cmd/server/rx_dashboard.go +++ b/cmd/server/rx_dashboard.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "encoding/json" + "fmt" "log" "net/http" "sort" @@ -99,7 +100,7 @@ func (s *Server) batchResolveHeardKeys(keys []string) map[string][2]string { // so this stays injection-safe regardless of how hexPrefixRe later // evolves. The per-prefix LIMIT 2 lives in a subquery because a bare // LIMIT on a UNION ALL term is a SQLite syntax error. - parts[j] = "SELECT * FROM (SELECT ? AS pfx, public_key, COALESCE(name,'') AS nm FROM nodes WHERE public_key LIKE ? LIMIT 2)" + parts[j] = fmt.Sprintf(s.db.nativeSQL("SELECT * FROM (SELECT CAST(%s AS TEXT) AS pfx, public_key, COALESCE(name,'') AS nm FROM nodes WHERE public_key LIKE %s LIMIT 2) AS matched", "SELECT * FROM (SELECT %s::text AS pfx, public_key, COALESCE(name,'') AS nm FROM nodes WHERE public_key LIKE %s LIMIT 2) AS matched"), s.db.parameter(len(args)+1), s.db.parameter(len(args)+2)) args = append(args, k, k+"%") } rows, err := s.db.conn.Query(strings.Join(parts, " UNION ALL "), args...) @@ -163,33 +164,34 @@ func (s *Server) queryCoverageFiltered(node, rx string, rxIn []string, days int, if rxIn != nil && len(rxIn) == 0 { return nil, nil } - where := []string{"lat BETWEEN ? AND ?", "lon BETWEEN ? AND ?"} + where := []string{"lat BETWEEN " + s.db.parameter(1) + " AND " + s.db.parameter(2), "lon BETWEEN " + s.db.parameter(3) + " AND " + s.db.parameter(4)} args := []interface{}{b.MinLat, b.MaxLat, b.MinLon, b.MaxLon} if node != "" { // Sargable heard_key IN-list (see coverageHeardKeyCandidates) so the // (heard_key, …) composite index is used instead of a substr() scan (#5). cands := coverageHeardKeyCandidates(node) - where = append(where, "heard_key IN ("+sqlPlaceholders(len(cands))+")") + where = append(where, "heard_key IN ("+s.db.sqlPlaceholders(len(cands), len(args)+1)+")") for _, c := range cands { args = append(args, c) } } if rx != "" { - where = append(where, "rx_pubkey = ?") + where = append(where, fmt.Sprintf("rx_pubkey = %s", s.db.parameter(len(args)+1))) args = append(args, strings.ToLower(rx)) } if len(rxIn) > 0 { - where = append(where, "rx_pubkey IN ("+sqlPlaceholders(len(rxIn))+")") + where = append(where, "rx_pubkey IN ("+s.db.sqlPlaceholders(len(rxIn), len(args)+1)+")") for _, pk := range rxIn { args = append(args, pk) } } if days > 0 { since := time.Now().UTC().AddDate(0, 0, -days).Format(time.RFC3339) - where = append(where, "rx_at >= ?") + where = append(where, fmt.Sprintf("rx_at >= %s", s.db.parameter(len(args)+1))) args = append(args, since) } - rows, err := s.db.conn.Query("SELECT lat, lon, snr, rssi, heard_key, rx_at FROM client_receptions WHERE "+strings.Join(where, " AND "), args...) + // As in queryCoverageRows, choose the plan using the actual bbox and filters. + rows, err := s.db.conn.Query("SELECT lat, lon, snr, rssi, heard_key, rx_at FROM client_receptions WHERE "+strings.Join(where, " AND "), s.db.planWithValues(args)...) if err != nil { return nil, err } @@ -337,9 +339,9 @@ func (s *Server) rxLeaderboard(ctx context.Context, days, limit int) ([]LeaderOb FROM client_receptions cr LEFT JOIN nodes n ON n.public_key = cr.rx_pubkey LEFT JOIN client_observers co ON co.pubkey = cr.rx_pubkey - WHERE cr.rx_at >= ? - ORDER BY cr.rx_at DESC - LIMIT ?`, since, leaderboardScanCap) + WHERE cr.rx_at >= `+s.db.parameter(1)+` + ORDER BY cr.rx_at DESC NULLS LAST + LIMIT `+s.db.parameter(2), since, leaderboardScanCap) if err != nil { return nil, err } diff --git a/cmd/server/scope_audit.go b/cmd/server/scope_audit.go index cf6cb6160..a4ebe98d8 100644 --- a/cmd/server/scope_audit.go +++ b/cmd/server/scope_audit.go @@ -20,6 +20,7 @@ package main // #1975 come from a 1179-repeater instance. import ( + "context" "database/sql" "fmt" "log" @@ -104,14 +105,14 @@ func splitRegionsCSV(csv string) []string { // supervisord starts both processes together, so a server that probed first // would ignore every mobile-client region answer until its next restart. // Once seen the answer latches, so the steady state costs nothing; until -// then it is one sqlite_master lookup per caller, and both callers sit +// then it is one schema catalog lookup per caller, and both callers sit // behind 30s caches. func (db *DB) declaredRegionsTablePresent() bool { if db.hasDeclaredRegionsTable || db.declaredRegionsTableLate.Load() { return true } var n int - if err := db.conn.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'node_declared_regions'`).Scan(&n); err != nil || n == 0 { + if err := db.conn.QueryRow(db.nativeSQL("SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='node_declared_regions'", `SELECT COUNT(*) FROM information_schema.tables WHERE table_schema=current_schema() AND table_name='node_declared_regions'`)).Scan(&n); err != nil || n == 0 { return false } db.declaredRegionsTableLate.Store(true) @@ -190,7 +191,7 @@ func (db *DB) AllCurrentDeclaredRegions() ([]DeclaredRegionsRow, error) { rows, err := db.conn.Query(` WITH ranked AS ( SELECT target, observed_at, regions_csv, truncated, - ROW_NUMBER() OVER (PARTITION BY target ORDER BY observed_at DESC) AS rn + ROW_NUMBER() OVER (PARTITION BY target ORDER BY observed_at DESC NULLS LAST) AS rn FROM node_declared_regions ) SELECT target, observed_at, regions_csv, truncated @@ -251,7 +252,7 @@ func (db *DB) scopeAuditNodeIdentities(pubkeys []string) map[string]scopeAuditNo placeholders := make([]string, len(pubkeys)) args := make([]interface{}, len(pubkeys)) for i, k := range pubkeys { - placeholders[i] = "?" + placeholders[i] = fmt.Sprintf("%s", db.parameter(i+1)) args[i] = strings.ToLower(k) } rows, err := db.conn.Query( @@ -374,7 +375,7 @@ func scopeAuditPrefixIndex(targets []string) map[int]map[string][]string { // belt-and-braces: one path can carry the same target on several hops. // It applies the SAME three conditions scopeConformanceQuery does — // minForwarderHopHexLen, scopeConformanceForwarderRouteTypesSQL, and the -// explicit json_valid guard against a single malformed path_json row +// explicit JSON-array guard against a single malformed path_json row // erroring the whole query — but does not join against any target list: // attribution to a specific declared target happens in Go // (ScopeAuditForwarding), against the small in-memory prefix index built by @@ -384,12 +385,20 @@ var scopeAuditForwarderScanQuery = ` SELECT t.id, je.value FROM transmissions t JOIN observations o ON o.transmission_id = t.id - JOIN json_each(o.path_json) je - WHERE t.first_seen >= ? + CROSS JOIN LATERAL json_array_elements_text(CASE WHEN o.path_json IS JSON ARRAY THEN o.path_json::json ELSE '[]'::json END) je + WHERE t.first_seen >= $1 + AND ` + scopeConformanceForwarderRouteTypesSQL + ` + AND o.path_json IS NOT NULL + AND LENGTH(je.value) >= ` + fmt.Sprint(minForwarderHopHexLen) + ` +` +var sqliteScopeAuditForwarderScanQuery = ` + SELECT t.id, je.value + FROM transmissions t + JOIN observations o ON o.transmission_id = t.id + JOIN json_each(CASE WHEN json_valid(o.path_json) THEN CASE WHEN json_type(o.path_json)='array' THEN o.path_json ELSE '[]' END ELSE '[]' END) je + WHERE t.first_seen >= $1 AND ` + scopeConformanceForwarderRouteTypesSQL + ` AND o.path_json IS NOT NULL - AND json_valid(o.path_json) - AND json_array_length(o.path_json) > 0 AND LENGTH(je.value) >= ` + fmt.Sprint(minForwarderHopHexLen) + ` ` @@ -407,7 +416,7 @@ func (db *DB) observerRegionClause(q interface { if db.isV3 { col, obsCol = "o.observer_idx", "rowid" } - placeholders := strings.TrimSuffix(strings.Repeat("?,", len(codes)), ",") + placeholders := db.sqlPlaceholders(len(codes)) args := make([]interface{}, len(codes)) for i, c := range codes { args[i] = c @@ -429,13 +438,13 @@ func (db *DB) observerRegionClause(q interface { return "", nil, err } if len(ids) == 0 { - return " AND 0", nil, nil + return " AND 1=0", nil, nil } // The unary + keeps SQLite off the observer index. Driven from it, the // scan reads every observation the region's observers ever made and checks // the window per row (13.4s for one region over 24h on a live-shaped database); // driven from the first_seen window like the unfiltered scan, 0.25s. - return " AND +" + col + " IN (" + strings.TrimSuffix(strings.Repeat("?,", len(ids)), ",") + ")", ids, nil + return " AND " + db.nativeSQL("+", "") + col + " IN (" + db.sqlPlaceholders(len(ids), 2) + ")", ids, nil } // scopeAuditWindowMetaQuery reads the two per-TRANSMISSION facts the hop scan @@ -501,7 +510,7 @@ func (db *DB) observerRegionClause(q interface { var scopeAuditWindowMetaQuery = ` SELECT t.id, t.scope_name, t.first_seen FROM transmissions t - WHERE t.first_seen >= ? + WHERE t.first_seen >= $1 AND ` + scopeConformanceForwarderRouteTypesSQL + ` ` @@ -611,6 +620,10 @@ type scopeAuditSeenKey struct { // the only thing keeping one transmission from counting several times for the // same target; TestScopeAuditForwardingCountsOneTransmissionOncePerTarget pins // it. +func (db *DB) beginScopeAuditSnapshot(ctx context.Context) (*sql.Tx, error) { + return db.conn.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelRepeatableRead, ReadOnly: true}) +} + func (s *PacketStore) ScopeAuditForwarding(sinceISO string, targets []string) (map[string]*scopeAuditTargetAgg, error) { return s.ScopeAuditForwardingHeardIn(sinceISO, targets, nil) } @@ -629,7 +642,7 @@ func (s *PacketStore) ScopeAuditForwardingHeardIn(sinceISO string, targets []str // transmission arriving between them would otherwise appear in the hop scan // with no metadata to attribute it by — rare, but the fix is a shared // snapshot rather than a rule about what to do with the leftovers. - tx, err := s.db.conn.Begin() + tx, err := s.db.beginScopeAuditSnapshot(context.Background()) if err != nil { return nil, fmt.Errorf("scope audit forwarder scan begin: %w", err) } @@ -644,7 +657,7 @@ func (s *PacketStore) ScopeAuditForwardingHeardIn(sinceISO string, targets []str if err != nil { return nil, fmt.Errorf("scope audit region observers: %w", err) } - rows, err := tx.Query(scopeAuditForwarderScanQuery+observerClause, append([]interface{}{sinceISO}, observerArgs...)...) + rows, err := tx.Query(s.db.nativeSQL(sqliteScopeAuditForwarderScanQuery, scopeAuditForwarderScanQuery)+observerClause, append([]interface{}{sinceISO}, observerArgs...)...) if err != nil { return nil, fmt.Errorf("scope audit forwarder scan: %w", err) } diff --git a/cmd/server/scope_audit_region_test.go b/cmd/server/scope_audit_region_test.go index 80bc09d92..8bf132a42 100644 --- a/cmd/server/scope_audit_region_test.go +++ b/cmd/server/scope_audit_region_test.go @@ -22,7 +22,7 @@ func setupScopeRegionServer(t *testing.T) (*Server, *mux.Router) { t.Helper() srv, router := setupScopeAuditServer(t) for _, ddl := range []string{ - `CREATE TABLE observers (id TEXT, name TEXT, iata TEXT)`, + testNativeSQL(`CREATE TABLE observers (id TEXT, name TEXT, iata TEXT)`, `CREATE TABLE observers (rowid BIGSERIAL PRIMARY KEY, id TEXT, name TEXT, iata TEXT)`), `ALTER TABLE observations ADD COLUMN observer_idx INTEGER`, `INSERT INTO observers (rowid, id, name, iata) VALUES (1, 'obs-sfo', 'San Francisco', 'SFO'), (2, 'obs-bru', 'Brussels', 'BRU')`, } { @@ -46,18 +46,18 @@ var heardCounter int func seedHeard(t *testing.T, srv *Server, hop string, seed scopeSeed, observerRowid int) { t.Helper() heardCounter++ - res, err := srv.db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, code1, code2, scope_name) - VALUES ('AA', ?, ?, ?, 1, ?, '00', ?)`, fmt.Sprintf("heard%d", heardCounter), - time.Now().UTC().Add(-time.Minute).Format(time.RFC3339), RouteFlood, seed.code1, seed.scopeName) + var id int64 + err := srv.db.conn.QueryRow(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, code1, code2, scope_name) + VALUES ('AA', $1, $2, $3, 1, $4, '00', $5) RETURNING id`, fmt.Sprintf("heard%d", heardCounter), + time.Now().UTC().Add(-time.Minute).Format(time.RFC3339), RouteFlood, seed.code1, seed.scopeName).Scan(&id) if err != nil { t.Fatal(err) } - id, _ := res.LastInsertId() path := fmt.Sprintf(`["%s"]`, strings.ToUpper(hop)) if observerRowid == 0 { // v2 fixture: the caller attributes observer_id itself - _, err = srv.db.conn.Exec(`INSERT INTO observations (transmission_id, path_json, timestamp) VALUES (?, ?, ?)`, id, path, time.Now().Unix()) + _, err = srv.db.conn.Exec(`INSERT INTO observations (transmission_id, path_json, timestamp) VALUES ($1, $2, $3)`, id, path, time.Now().Unix()) } else { - _, err = srv.db.conn.Exec(`INSERT INTO observations (transmission_id, path_json, timestamp, observer_idx) VALUES (?, ?, ?, ?)`, id, path, time.Now().Unix(), observerRowid) + _, err = srv.db.conn.Exec(`INSERT INTO observations (transmission_id, path_json, timestamp, observer_idx) VALUES ($1, $2, $3, $4)`, id, path, time.Now().Unix(), observerRowid) } if err != nil { t.Fatal(err) @@ -154,7 +154,7 @@ func TestScopeAudit_RegionWithNoObserversMatchesNothing(t *testing.T) { func TestScopeTransport_RegionOnV2Schema(t *testing.T) { srv, router := setupScopeAuditServer(t) for _, ddl := range []string{ - `CREATE TABLE observers (id TEXT, name TEXT, iata TEXT)`, + testNativeSQL(`CREATE TABLE observers (id TEXT, name TEXT, iata TEXT)`, `CREATE TABLE observers (rowid BIGSERIAL PRIMARY KEY, id TEXT, name TEXT, iata TEXT)`), `ALTER TABLE observations ADD COLUMN observer_id TEXT`, `INSERT INTO observers (id, name, iata) VALUES ('obs-sfo', 'San Francisco', 'SFO'), ('obs-bru', 'Brussels', 'BRU')`, } { @@ -172,7 +172,7 @@ func TestScopeTransport_RegionOnV2Schema(t *testing.T) { addRepeater(t, srv, testFullPubkeyB, "Bravo") seedHeard(t, srv, testFullPubkeyA[:4], scopeMatched("#be"), 0) seedHeard(t, srv, testFullPubkeyB[:4], scopeMatched("#fr"), 0) - if _, err := srv.db.conn.Exec(`UPDATE observations SET observer_id = CASE WHEN path_json LIKE ? THEN 'obs-sfo' ELSE 'obs-bru' END`, + if _, err := srv.db.conn.Exec(`UPDATE observations SET observer_id = CASE WHEN path_json LIKE $1 THEN 'obs-sfo' ELSE 'obs-bru' END`, `%`+strings.ToUpper(testFullPubkeyA[:4])+`%`); err != nil { t.Fatal(err) } @@ -188,6 +188,9 @@ func TestScopeTransport_RegionOnV2Schema(t *testing.T) { // live-shaped database that was 13.4s for one region over 24h against 0.25s when // the scan starts from the first_seen window as the unfiltered scan does. func TestScopeAudit_RegionScanKeepsTheTimeIndex(t *testing.T) { + if testBackendValue() != "sqlite" { + t.Skip("SQLite query-plan assertion") + } srv, _ := setupScopeRegionServer(t) for _, ddl := range []string{ `CREATE INDEX IF NOT EXISTS idx_transmissions_first_seen ON transmissions(first_seen)`, @@ -202,7 +205,7 @@ func TestScopeAudit_RegionScanKeepsTheTimeIndex(t *testing.T) { if err != nil || clause == "" { t.Fatalf("clause=%q err=%v", clause, err) } - rows, err := srv.db.conn.Query("EXPLAIN QUERY PLAN "+scopeAuditForwarderScanQuery+clause, + rows, err := srv.db.conn.Query("EXPLAIN QUERY PLAN "+sqliteScopeAuditForwarderScanQuery+clause, append([]interface{}{"2000-01-01T00:00:00Z"}, args...)...) if err != nil { t.Fatal(err) diff --git a/cmd/server/scope_audit_test.go b/cmd/server/scope_audit_test.go index 444c34e60..2156fd0f4 100644 --- a/cmd/server/scope_audit_test.go +++ b/cmd/server/scope_audit_test.go @@ -2,7 +2,6 @@ package main import ( "context" - "database/sql" "encoding/hex" "encoding/json" "fmt" @@ -31,12 +30,12 @@ var ( // fiction, so the join behaves the way it does against a real database. func setupScopeConformanceDB(t *testing.T) *DB { t.Helper() - conn, err := sql.Open("sqlite3", ":memory:") + conn, err := openPostgresTestDB(t) if err != nil { t.Fatal(err) } conn.SetMaxOpenConns(1) - schema := ` + schema := testNativeSQL(` CREATE TABLE transmissions ( id INTEGER PRIMARY KEY AUTOINCREMENT, raw_hex TEXT NOT NULL, @@ -54,11 +53,29 @@ func setupScopeConformanceDB(t *testing.T) *DB { path_json TEXT, timestamp INTEGER NOT NULL ); - ` + `, ` + CREATE TABLE transmissions ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + raw_hex TEXT NOT NULL, + hash TEXT NOT NULL UNIQUE, + first_seen TEXT NOT NULL, + route_type INTEGER, + payload_type INTEGER, + code1 TEXT, + code2 TEXT, + scope_name TEXT + ); + CREATE TABLE observations ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + transmission_id INTEGER NOT NULL REFERENCES transmissions(id), + path_json TEXT, + timestamp INTEGER NOT NULL + ); + `) if _, err := conn.Exec(schema); err != nil { t.Fatal(err) } - return &DB{conn: conn} + return &DB{backend: testBackendValue(), conn: conn} } // newScopeTestStore wires a *PacketStore to a freshly seeded scope-conformance @@ -115,22 +132,23 @@ func seedTransmissionRouteAt(t *testing.T, s *PacketStore, forwarder string, see scopeSeedCounter++ hash := fmt.Sprintf("scopehash%d", scopeSeedCounter) - res, err := s.db.conn.Exec( + var txID int64 + err := s.db.conn.QueryRow( `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, code1, code2, scope_name) - VALUES ('AA', ?, ?, ?, 1, ?, '00', ?)`, + VALUES ('AA', $1, $2, $3, 1, $4, '00', $5) RETURNING id`, hash, firstSeen, routeType, seed.code1, seed.scopeName, - ) + ).Scan(&txID) if err != nil { t.Fatalf("seed transmission: %v", err) } - txID, err := res.LastInsertId() + if err != nil { t.Fatalf("seed transmission id: %v", err) } pathJSON := fmt.Sprintf(`["%s"]`, strings.ToUpper(forwarder)) if _, err := s.db.conn.Exec( - `INSERT INTO observations (transmission_id, path_json, timestamp) VALUES (?, ?, ?)`, + `INSERT INTO observations (transmission_id, path_json, timestamp) VALUES ($1, $2, $3)`, txID, pathJSON, time.Now().Unix(), ); err != nil { t.Fatalf("seed observation: %v", err) @@ -257,7 +275,7 @@ func insertDeclared(t *testing.T, srv *Server, target, observedAt, regionsCSV st t.Helper() _ = truncated if _, err := srv.db.conn.Exec( - `INSERT INTO nodes (public_key, configured_scope, configured_scope_at) VALUES (?, ?, ?) + `INSERT INTO nodes (public_key, configured_scope, configured_scope_at) VALUES ($1, $2, $3) ON CONFLICT(public_key) DO UPDATE SET configured_scope = excluded.configured_scope, configured_scope_at = excluded.configured_scope_at`, target, regionsCSV, observedAt); err != nil { @@ -383,7 +401,7 @@ func TestHandleScopeAuditOmitsRepeaterWithNoDeclaredRow(t *testing.T) { declaredPk := testFullPubkeyA neverAskedPk := testFullPubkeyB insertDeclared(t, srv, declaredPk, time.Now().UTC().Format(time.RFC3339), "be", 0) - if _, err := srv.db.conn.Exec(`INSERT INTO nodes (public_key, name, role) VALUES (?, 'never-asked', 'repeater')`, neverAskedPk); err != nil { + if _, err := srv.db.conn.Exec(`INSERT INTO nodes (public_key, name, role) VALUES ($1, 'never-asked', 'repeater')`, neverAskedPk); err != nil { t.Fatal(err) } @@ -596,7 +614,7 @@ func TestHandleScopeAuditFiltersHiddenNamePrefix(t *testing.T) { // (configured_scope), so insertDeclared has already created it. The fork // kept them in separate tables and a plain INSERT was safe there. if _, err := srv.db.conn.Exec( - `INSERT INTO nodes (public_key, name, role) VALUES (?, ?, 'repeater') + `INSERT INTO nodes (public_key, name, role) VALUES ($1, $2, 'repeater') ON CONFLICT(public_key) DO UPDATE SET name = excluded.name, role = excluded.role`, hiddenPk, "🚫 ban me"); err != nil { t.Fatal(err) @@ -727,7 +745,7 @@ func TestHandleScopeAuditConfigStateAllFourShapes(t *testing.T) { // would skip the table and every assertion below would pass vacuously. func createDeclaredRegionsTable(t *testing.T, srv *Server) { t.Helper() - if _, err := srv.db.conn.Exec(` + if _, err := srv.db.conn.Exec(testNativeSQL(` CREATE TABLE node_declared_regions ( id INTEGER PRIMARY KEY AUTOINCREMENT, target TEXT NOT NULL, @@ -737,7 +755,17 @@ func createDeclaredRegionsTable(t *testing.T, srv *Server) { regions_csv TEXT NOT NULL, truncated INTEGER NOT NULL DEFAULT 0, UNIQUE(target, rx_pubkey, observed_at) - )`); err != nil { + )`, ` + CREATE TABLE node_declared_regions ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + target TEXT NOT NULL, + rx_pubkey TEXT NOT NULL, + observed_at TEXT NOT NULL, + ingested_at TEXT NOT NULL, + regions_csv TEXT NOT NULL, + truncated INTEGER NOT NULL DEFAULT 0, + UNIQUE(target, rx_pubkey, observed_at) + )`)); err != nil { t.Fatal(err) } if err := srv.db.detectSchema(context.Background(), srv.db.conn); err != nil { @@ -759,7 +787,7 @@ func TestDeclaredRegionsTableCreatedAfterStartupIsRead(t *testing.T) { } // Created behind the server's back, with no detectSchema re-probe: exactly // what happens when the ingestor creates it after the server started. - if _, err := srv.db.conn.Exec(` + if _, err := srv.db.conn.Exec(testNativeSQL(` CREATE TABLE node_declared_regions ( id INTEGER PRIMARY KEY AUTOINCREMENT, target TEXT NOT NULL, @@ -769,7 +797,17 @@ func TestDeclaredRegionsTableCreatedAfterStartupIsRead(t *testing.T) { regions_csv TEXT NOT NULL, truncated INTEGER NOT NULL DEFAULT 0, UNIQUE(target, rx_pubkey, observed_at) - )`); err != nil { + )`, ` + CREATE TABLE node_declared_regions ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + target TEXT NOT NULL, + rx_pubkey TEXT NOT NULL, + observed_at TEXT NOT NULL, + ingested_at TEXT NOT NULL, + regions_csv TEXT NOT NULL, + truncated INTEGER NOT NULL DEFAULT 0, + UNIQUE(target, rx_pubkey, observed_at) + )`)); err != nil { t.Fatal(err) } if srv.db.hasDeclaredRegionsTable { @@ -789,7 +827,7 @@ func seedSecondSource(t *testing.T, srv *Server, target, observedAt, regionsCSV t.Helper() if _, err := srv.db.conn.Exec( `INSERT INTO node_declared_regions (target, rx_pubkey, observed_at, ingested_at, regions_csv, truncated) - VALUES (?, 'rxpubkeyhex', ?, ?, ?, ?)`, + VALUES ($1, 'rxpubkeyhex', $2, $3, $4, $5)`, target, observedAt, observedAt, regionsCSV, truncated); err != nil { t.Fatal(err) } @@ -911,15 +949,16 @@ func seedTransmissionPathAt(t *testing.T, s *PacketStore, hops []string, seed sc scopeSeedCounter++ hash := fmt.Sprintf("scopehash%d", scopeSeedCounter) - res, err := s.db.conn.Exec( + var txID int64 + err := s.db.conn.QueryRow( `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, code1, code2, scope_name) - VALUES ('AA', ?, ?, ?, 1, ?, '00', ?)`, + VALUES ('AA', $1, $2, $3, 1, $4, '00', $5) RETURNING id`, hash, firstSeen, routeType, seed.code1, seed.scopeName, - ) + ).Scan(&txID) if err != nil { t.Fatalf("seed transmission: %v", err) } - txID, err := res.LastInsertId() + if err != nil { t.Fatalf("seed transmission id: %v", err) } @@ -930,7 +969,7 @@ func seedTransmissionPathAt(t *testing.T, s *PacketStore, hops []string, seed sc } pathJSON := "[" + strings.Join(quoted, ",") + "]" if _, err := s.db.conn.Exec( - `INSERT INTO observations (transmission_id, path_json, timestamp) VALUES (?, ?, ?)`, + `INSERT INTO observations (transmission_id, path_json, timestamp) VALUES ($1, $2, $3)`, txID, pathJSON, time.Now().Unix(), ); err != nil { t.Fatalf("seed observation: %v", err) @@ -1328,19 +1367,20 @@ func seedUnmatchedRawAt(t *testing.T, s *PacketStore, forwarder, rawHex string, t.Helper() scopeSeedCounter++ hash := fmt.Sprintf("scoperaw%d", scopeSeedCounter) - res, err := s.db.conn.Exec( + var txID int64 + err := s.db.conn.QueryRow( `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, code1, code2, scope_name) - VALUES (?, ?, ?, ?, 5, '9209', '0000', '')`, - rawHex, hash, firstSeen, routeType) + VALUES ($1, $2, $3, $4, 5, '9209', '0000', '') RETURNING id`, + rawHex, hash, firstSeen, routeType).Scan(&txID) if err != nil { t.Fatal(err) } - txID, err := res.LastInsertId() + if err != nil { t.Fatal(err) } if _, err := s.db.conn.Exec( - `INSERT INTO observations (transmission_id, path_json, timestamp) VALUES (?, ?, 0)`, + `INSERT INTO observations (transmission_id, path_json, timestamp) VALUES ($1, $2, 0)`, txID, `["`+strings.ToUpper(forwarder)+`"]`); err != nil { t.Fatal(err) } diff --git a/cmd/server/scope_config_state_api_test.go b/cmd/server/scope_config_state_api_test.go index c1b5b0ca4..30b9d8ac3 100644 --- a/cmd/server/scope_config_state_api_test.go +++ b/cmd/server/scope_config_state_api_test.go @@ -271,19 +271,20 @@ func TestHandleNodesReportsObservedForUndeclaredForwarder(t *testing.T) { const pk = "a1b2c3d4e5f60718293a4b5c6d7e8f90a1b2c3d4e5f60718293a4b5c6d7e8f90" if _, err := db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, 'silent-forwarder', 'repeater', 51.0, 4.0, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z', 1)`, pk, + VALUES ($1, 'silent-forwarder', 'repeater', 51.0, 4.0, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z', 1)`, pk, ); err != nil { t.Fatal(err) } - res, err := db.conn.Exec( + var txID int64 + err := db.conn.QueryRow( `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json, scope_name) - VALUES ('AA', 'observed-scope-hash', ?, 0, 1, 1, '{}', '#be')`, + VALUES ('AA', 'observed-scope-hash', $1, 0, 1, 1, '{}', '#be') RETURNING id`, time.Now().UTC().Format(time.RFC3339), - ) + ).Scan(&txID) if err != nil { t.Fatal(err) } - txID, err := res.LastInsertId() + if err != nil { t.Fatal(err) } @@ -296,7 +297,7 @@ func TestHandleNodesReportsObservedForUndeclaredForwarder(t *testing.T) { // too, which also pins the lower-casing in addTxToPathHopIndex. if _, err := db.conn.Exec( `INSERT INTO observations (transmission_id, observer_idx, direction, snr, rssi, score, path_json, timestamp) - VALUES (?, 0, 'rx', 1.0, -100, 0, ?, ?)`, + VALUES ($1, 0, 'rx', 1.0, -100, 0, $2, $3)`, txID, `["`+strings.ToUpper(pk)+`"]`, time.Now().Unix(), ); err != nil { t.Fatal(err) @@ -358,7 +359,7 @@ func TestScopeAuditAndNodesAgreeEndToEnd(t *testing.T) { for pk, csv := range cases { if _, err := srv.db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count, configured_scope, configured_scope_at) - VALUES (?, ?, 'repeater', 51.0, 4.0, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z', 1, ?, '2026-01-01T00:00:00Z')`, + VALUES ($1, $2, 'repeater', 51.0, 4.0, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z', 1, $3, '2026-01-01T00:00:00Z')`, pk, "rp-"+pk, csv, ); err != nil { t.Fatal(err) diff --git a/cmd/server/scope_stats_region_index_test.go b/cmd/server/scope_stats_region_index_test.go index 57a96e080..5d9729dcf 100644 --- a/cmd/server/scope_stats_region_index_test.go +++ b/cmd/server/scope_stats_region_index_test.go @@ -60,7 +60,7 @@ func scopeStatsFixtureDB(tb testing.TB, numTx int) *DB { tb.Fatal(err) } stmt, err := rw.Prepare(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, scope_name) - VALUES ('aa', ?, ?, ?, 5, ?)`) + VALUES ('aa', $1, $2, $3, 5, $4)`) if err != nil { tb.Fatal(err) } @@ -126,7 +126,7 @@ func TestScopeStatsByRegion_CountsMatchWindow(t *testing.T) { since := time.Now().Add(-24 * time.Hour).UTC().Format(time.RFC3339) var want int if err := db.conn.QueryRow(`SELECT COUNT(*) FROM transmissions WHERE route_type IN (0,3) - AND scope_name IS NOT NULL AND scope_name != '' AND first_seen >= ?`, since).Scan(&want); err != nil { + AND scope_name IS NOT NULL AND scope_name != '' AND first_seen >= $1`, since).Scan(&want); err != nil { t.Fatal(err) } got := 0 diff --git a/cmd/server/scope_transport_2142_test.go b/cmd/server/scope_transport_2142_test.go index b37617954..e2d6d3d49 100644 --- a/cmd/server/scope_transport_2142_test.go +++ b/cmd/server/scope_transport_2142_test.go @@ -53,7 +53,7 @@ func transportRow(t *testing.T, rows []ScopeTransportRow, pk string) ScopeTransp func addRepeater(t *testing.T, srv *Server, pk, name string) { t.Helper() - if _, err := srv.db.conn.Exec(`INSERT INTO nodes (public_key, name, role) VALUES (?, ?, 'repeater') + if _, err := srv.db.conn.Exec(`INSERT INTO nodes (public_key, name, role) VALUES ($1, $2, 'repeater') ON CONFLICT(public_key) DO UPDATE SET name = excluded.name, role = 'repeater'`, pk, name); err != nil { t.Fatal(err) } @@ -207,7 +207,7 @@ func BenchmarkComputeScopeTransport(b *testing.B) { if i%7 == 0 { scope = sql.NullString{String: "be,be-van,*", Valid: true} } - conn.Exec(`INSERT INTO nodes (public_key, name, role, configured_scope, configured_scope_at) VALUES (?, ?, 'repeater', ?, ?)`, + conn.Exec(`INSERT INTO nodes (public_key, name, role, configured_scope, configured_scope_at) VALUES ($1, $2, 'repeater', $3, $4)`, pks[i], fmt.Sprintf("Repeater %d", i), scope, time.Now().UTC().Format(time.RFC3339)) } regions := []string{"#be", "#be-van", "#be-ant", "#be-gnt", "#be-lge", "#nl", "#nl-ams", "#de"} @@ -219,11 +219,11 @@ func BenchmarkComputeScopeTransport(b *testing.B) { route, scope = 0, regions[i%len(regions)] } res, _ := conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, code1, code2, scope_name) - VALUES ('AA', ?, ?, ?, 1, '1234', '00', ?)`, fmt.Sprintf("bench%d", i), now.Add(-time.Duration(i)*2*time.Second).Format(time.RFC3339), route, scope) + VALUES ('AA', $1, $2, $3, 1, '1234', '00', $4)`, fmt.Sprintf("bench%d", i), now.Add(-time.Duration(i)*2*time.Second).Format(time.RFC3339), route, scope) id, _ := res.LastInsertId() path := fmt.Sprintf(`["%s","%s","%s","%s"]`, strings.ToUpper(pks[i%fleet][:6]), strings.ToUpper(pks[(i*7)%fleet][:6]), strings.ToUpper(pks[(i*11)%fleet][:6]), strings.ToUpper(pks[(i*17)%fleet][:6])) - conn.Exec(`INSERT INTO observations (transmission_id, path_json, timestamp) VALUES (?, ?, ?)`, id, path, now.Unix()) + conn.Exec(`INSERT INTO observations (transmission_id, path_json, timestamp) VALUES ($1, $2, $3)`, id, path, now.Unix()) } conn.Exec("COMMIT") cfg := &Config{Port: 3000} diff --git a/cmd/server/scope_verify.go b/cmd/server/scope_verify.go index 3988df440..5c7893353 100644 --- a/cmd/server/scope_verify.go +++ b/cmd/server/scope_verify.go @@ -184,11 +184,11 @@ func (s *PacketStore) unmatchedTransmissionsInWindow(sinceISO string) ([]unmatch rows, err := s.db.conn.Query(` SELECT t.id, t.raw_hex FROM transmissions t - WHERE t.first_seen >= ? + WHERE t.first_seen >= `+s.db.parameter(1)+` AND t.scope_name = '' AND `+scopeConformanceForwarderRouteTypesSQL+` ORDER BY t.first_seen DESC - LIMIT ?`, sinceISO, scopeVerifyMaxWindowPackets) + LIMIT `+s.db.parameter(2), sinceISO, scopeVerifyMaxWindowPackets) if err != nil { return nil, false, fmt.Errorf("unmatched transmissions scan: %w", err) } diff --git a/cmd/server/startup_admission_summary_test.go b/cmd/server/startup_admission_summary_test.go new file mode 100644 index 000000000..a24b361f6 --- /dev/null +++ b/cmd/server/startup_admission_summary_test.go @@ -0,0 +1,143 @@ +package main + +import ( + "fmt" + "strings" + "testing" + "time" +) + +func TestStartupAdmissionSummaryReheardOldHashKeepsCoverageFloor(t *testing.T) { + dsn := postgresTestDSN(t) + writer, err := openFixtureSQL(dsn) + if err != nil { + t.Fatal(err) + } + defer writer.Close() + if err := applyTestSchema(t, writer); err != nil { + t.Fatal(err) + } + now := time.Now().UTC().Truncate(time.Second) + reheardFirst := now.Add(-30 * 24 * time.Hour).Format(time.RFC3339) + inactive := now.Add(-48 * time.Hour) + if _, err := writer.Exec(`INSERT INTO observers(rowid,id,name) VALUES(1,'summary-observer','Summary fixture')`); err != nil { + t.Fatal(err) + } + if _, err := writer.Exec(`INSERT INTO transmissions(id,raw_hex,hash,first_seen,last_seen,route_type,payload_type,decoded_json) + VALUES(1,'AA','summary-reheard',$1,$2,0,4,'{}'), + (2,'BB','summary-inactive',$3,$4,0,4,'{}')`, + reheardFirst, now.Unix(), inactive.Format(time.RFC3339), inactive.Unix()); err != nil { + t.Fatal(err) + } + if _, err := writer.Exec(`INSERT INTO observations(id,transmission_id,observer_idx,path_json,timestamp) + VALUES(1,1,1,'[]',$1),(2,2,1,'[]',$2)`, now.Unix(), inactive.Unix()); err != nil { + t.Fatal(err) + } + db, err := openFixtureReader(t, dsn) + if err != nil { + t.Fatal(err) + } + defer db.conn.Close() + s := NewPacketStore(db, &PacketStoreConfig{RetentionHours: 24, HotStartupHours: 0}) + started := time.Now().UTC() + logs := captureLog(func() { + if err := s.RunStartupLoad(1); err != nil { + t.Fatal(err) + } + }) + finished := time.Now().UTC() + if len(s.packets) != 1 || s.byTxID[1] == nil || s.byTxID[2] != nil { + t.Fatal("fixture must retain the re-heard hash and omit the inactive hash") + } + if !strings.Contains(logs, fmt.Sprintf("firstSeenRange=[%q,%q]", reheardFirst, reheardFirst)) { + t.Fatal("summary lost the actual retained first-seen range", logs) + } + floor := loggedStartupCoverage(t, logs) + if floor.Before(started.Add(-24*time.Hour).Truncate(time.Second)) || floor.After(finished.Add(-24*time.Hour)) { + t.Fatalf("summary claims coverageFrom=%q outside the complete 24-hour window; retained first_seen=%q", floor.Format(time.RFC3339), reheardFirst) + } +} + +func loggedStartupCoverage(t *testing.T, logs string) time.Time { + t.Helper() + _, rest, found := strings.Cut(logs, `coverageFrom="`) + coverage, _, closed := strings.Cut(rest, `"`) + if !found || !closed { + t.Fatal("summary has no quoted coverage floor", logs) + } + floor, err := time.Parse(time.RFC3339, coverage) + if err != nil { + t.Fatal("summary has an invalid coverage floor", coverage, err) + } + return floor +} + +func TestStartupAdmissionSummary(t *testing.T) { + for _, tc := range []struct { + name string + transmissions int + observations int + hotHours float64 + budgetMB int + closeDB bool + reason string + }{ + {"empty", 0, 0, 0, 0, false, "window_complete"}, + {"full", 4, 2, 0, 0, false, "window_complete"}, + {"background", 4, 2, 1, 0, false, "window_complete"}, + {"budget", 40, 300, 0, 1, false, "memory_budget"}, + {"background_budget", 40, 300, 1, 1, false, "memory_budget"}, + {"error", 4, 2, 1, 1, true, "error"}, + } { + t.Run(tc.name, func(t *testing.T) { + db := startupMemoryFixture(t, tc.transmissions, tc.observations, tc.hotHours == 0) + s := NewPacketStore(db, &PacketStoreConfig{ + MaxMemoryMB: tc.budgetMB, RetentionHours: 24, HotStartupHours: tc.hotHours, + }) + if tc.closeDB { + db.conn.Close() + } + started := time.Now().UTC() + logs := captureLog(func() { + err := s.RunStartupLoad(2) + if (err != nil) != tc.closeDB { + t.Fatalf("startup error=%v, want error=%v", err, tc.closeDB) + } + }) + finished := time.Now().UTC() + var summaries []string + for _, line := range strings.Split(logs, "\n") { + if strings.Contains(line, "[store] startup admission:") { + summaries = append(summaries, line) + } + } + if len(summaries) != 1 { + t.Fatalf("want one terminal admission summary, got %d:\n%s", len(summaries), logs) + } + first, last := "", "" + if len(s.packets) > 0 { + first, last = s.packets[0].FirstSeen, s.packets[len(s.packets)-1].FirstSeen + } + for _, want := range []string{ + fmt.Sprintf("budgetMB=%d", tc.budgetMB), + fmt.Sprintf("transmissions=%d observations=%d", len(s.packets), s.totalObs), + fmt.Sprintf("firstSeenRange=[%q,%q]", first, last), + "stop=" + tc.reason, + "formerPolicyTransmissionEstimate=", + } { + if !strings.Contains(summaries[0], want) { + t.Errorf("summary missing %q: %s", want, summaries[0]) + } + } + if tc.name == "background" && len(s.packets) != tc.transmissions { + t.Fatalf("summary preceded background fill: %d/%d packets", len(s.packets), tc.transmissions) + } + floor := loggedStartupCoverage(t, summaries[0]) + earliest, _ := s.coverageStart(started) + latest, _ := s.coverageStart(finished) + if floor.Before(earliest.Truncate(time.Second)) || floor.After(latest) { + t.Fatalf("summary coverage floor=%s, want complete coverage between %s and %s", floor, earliest, latest) + } + }) + } +} diff --git a/cmd/server/startup_memory_budget_test.go b/cmd/server/startup_memory_budget_test.go new file mode 100644 index 000000000..5e3ba3b25 --- /dev/null +++ b/cmd/server/startup_memory_budget_test.go @@ -0,0 +1,276 @@ +package main + +import ( + "fmt" + "runtime" + "testing" + "time" +) + +// Each transmission has many distinct observer/path pairs. A transmission-count +// estimate cannot bound this fixture: every row is retained by the real deduper. +func startupMemoryFixture(t testing.TB, transmissions, observations int, recent bool) *DB { + t.Helper() + dsn := postgresTestDSN(t) + createTestDBAt(t, dsn, 0) + w, err := openFixtureSQL(dsn) + if err != nil { + t.Fatal(err) + } + defer w.Close() + if _, err := w.Exec("ALTER TABLE transmissions ADD COLUMN last_seen BIGINT"); err != nil { + t.Fatal(err) + } + // Use the production v3 epoch timestamp/observer-index representation, so + // SQL fallback and PostgreSQL parameter types are exercised realistically. + if _, err := w.Exec("DROP TABLE observations"); err != nil { + t.Fatal(err) + } + if _, err := w.Exec(`CREATE TABLE observations ( + id BIGINT PRIMARY KEY, transmission_id BIGINT, observer_idx BIGINT, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, + path_json TEXT, timestamp BIGINT, raw_hex TEXT + )`); err != nil { + t.Fatal(err) + } + now := time.Now().UTC().Truncate(time.Second) + stamp := func(n int) time.Time { + if recent || n == 0 { + return now.Add(-30 * time.Minute) + } + return now.Add(-2 * time.Hour).Add(-time.Duration(n/3) * time.Second) + } + copyTestRows(t, w, "transmissions", []string{"id", "raw_hex", "hash", "first_seen", "route_type", "payload_type", "decoded_json", "last_seen"}, transmissions, func(n int) []any { + return []any{n, "aabb", fmt.Sprintf("budget-%04d", n), stamp(n).Format(time.RFC3339), 0, 4, "{}", stamp(n).Unix()} + }) + copyTestRows(t, w, "observers", []string{"rowid", "id", "name"}, observations, func(n int) []any { + return []any{n, fmt.Sprintf("observer-%04d", n), "Synthetic observer"} + }) + copyTestRows(t, w, "observations", []string{"id", "transmission_id", "observer_idx", "direction", "snr", "rssi", "score", "path_json", "timestamp"}, transmissions*observations, func(n int) []any { + return []any{n + 1, n / observations, n % observations, "rx", 5.0, -80.0, 5, `["aa","bb","cc","dd","ee","ff","ab","bc","cd","de","ef","fa"]`, stamp(n / observations).Unix()} + }) + for _, query := range []string{ + "CREATE INDEX budget_observations ON observations(transmission_id, timestamp)", + } { + if _, err := w.Exec(query); err != nil { + t.Fatal(err) + } + } + r, err := openFixtureReader(t, dsn) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { r.conn.Close() }) + return r +} + +func assertStartupBudget(t *testing.T, s *PacketStore, observations int) { + t.Helper() + if s.trackedBytes > int64(s.maxMemoryMB)*1048576 { + t.Fatalf("cache exceeds %d MiB budget: %.2f MiB, %d transmissions/%d observations", s.maxMemoryMB, s.trackedMemoryMB(), len(s.packets), s.totalObs) + } + for i, tx := range s.packets { + if len(tx.Observations) != observations { + t.Fatalf("partial transmission %d cached: %d/%d observations", tx.ID, len(tx.Observations), observations) + } + if s.byHash[tx.Hash] != tx || s.byTxID[tx.ID] != tx { + t.Fatalf("transmission %d missing from indexes", tx.ID) + } + if i > 0 && s.packets[i-1].FirstSeen > tx.FirstSeen { + t.Fatal("packet slice is not sorted oldest-first") + } + } + if len(s.byHash) != len(s.packets) || len(s.byTxID) != len(s.packets) || len(s.byObsID) != s.totalObs { + t.Fatal("partial transmission leaked into an index") + } + if s.startupMemoryLimited { + edge, reason := s.coverageStart(time.Now().UTC()) + if edge.Format(time.RFC3339) != s.oldestLoaded || reason != coverageMemory { + t.Fatalf("memory-limited coverage edge=%s (%s), want conservative boundary %s", edge, reason, s.oldestLoaded) + } + } +} + +func TestStartupMemoryBudget_HighFanout(t *testing.T) { + for _, hotHours := range []float64{0, 1} { + t.Run(fmt.Sprintf("hot-%g", hotHours), func(t *testing.T) { + db := startupMemoryFixture(t, 120, 300, hotHours == 0) + s := NewPacketStore(db, &PacketStoreConfig{MaxMemoryMB: 1, RetentionHours: 24, HotStartupHours: hotHours}) + if err := s.RunStartupLoad(10000); err != nil { + t.Fatal(err) + } + assertStartupBudget(t, s, 300) + if len(s.packets) == 0 || len(s.packets) >= 120 { + t.Fatalf("expected a useful bounded cache, got %d/120", len(s.packets)) + } + if !s.LoadComplete() || !s.backgroundLoadDone.Load() || s.backgroundLoadFailed.Load() { + t.Fatalf("configured memory limit must finish successfully: %q", s.BackgroundLoadError()) + } + if s.LoadCoverageRatio() >= 1 { + t.Fatal("bounded cache must not claim complete database coverage") + } + if s.maxTxID != 119 || s.maxObsID != 120*300 { + t.Fatalf("poll cursors replay excluded history: tx=%d, obs=%d", s.maxTxID, s.maxObsID) + } + if replayed, _ := s.IngestNewFromDB(s.MaxTransmissionID(), 500); len(replayed) != 0 { + t.Fatalf("poll replayed %d omitted historical transmissions", len(replayed)) + } + if replayed := s.IngestNewObservations(s.MaxObservationID(), 500); len(replayed) != 0 { + t.Fatalf("poll replayed %d omitted historical observations", len(replayed)) + } + if hotHours > 0 && s.byTxID[0] == nil { + t.Fatal("old transmission ID with newest traffic was lost") + } + result := s.QueryPackets(PacketQuery{Since: time.Now().UTC().Add(-24 * time.Hour).Format(time.RFC3339), Limit: 200}) + if result.Total != 120 { + t.Fatalf("SQL history fallback omitted transmissions: %d/120", result.Total) + } + }) + } +} + +func TestStartupMemoryBudget_OversizedTransmission(t *testing.T) { + db := startupMemoryFixture(t, 1, 5000, true) + s := NewPacketStore(db, &PacketStoreConfig{MaxMemoryMB: 1, RetentionHours: 24, HotStartupHours: 1}) + if err := s.RunStartupLoad(10000); err != nil { + t.Fatal(err) + } + assertStartupBudget(t, s, 5000) + if len(s.packets) != 0 || s.totalObs != 0 { + t.Fatal("oversized transmission must stay in SQL, not partially in cache") + } + if s.QueryPackets(PacketQuery{Hash: "budget-0000", Limit: 1}).Total != 1 { + t.Fatal("omitted transmission is unavailable without a date filter") + } + if s.QueryGroupedPackets(PacketQuery{Hash: "budget-0000", Limit: 1}).Total != 1 { + t.Fatal("omitted transmission is unavailable in grouped packet queries") + } + if !s.backgroundLoadDone.Load() || s.backgroundLoadFailed.Load() { + t.Fatalf("oversized row must not prevent readiness: %q", s.BackgroundLoadError()) + } +} + +func TestStartupMemoryBudget_TimeAndIDCursor(t *testing.T) { + db := startupMemoryFixture(t, 100, 30, true) + s := NewPacketStore(db, &PacketStoreConfig{MaxMemoryMB: 1, RetentionHours: 24}) + if err := s.RunStartupLoad(2); err != nil { + t.Fatal(err) + } + assertStartupBudget(t, s, 30) + if s.byTxID[99] == nil || s.byTxID[0] != nil { + t.Fatal("equal-time rows must load highest IDs first") + } + for id := 100 - len(s.packets); id < 100; id++ { + if s.byTxID[id] == nil { + t.Fatalf("time/ID cursor skipped transmission %d", id) + } + } + // All timestamps tie; the floor must be conservative across the omitted + // part of that second, rather than claiming that every tied row was loaded. + if s.oldestLoaded <= s.packets[0].FirstSeen { + t.Fatalf("cache coverage floor includes omitted tied rows: %s", s.oldestLoaded) + } +} + +func TestStartupMemoryBudget_MergesLiveObservations(t *testing.T) { + db, writer := openPostgresValueFixture(t) + now := time.Now().UTC().Truncate(time.Second) + for i := 1; i <= 4; i++ { + if _, err := writer.Exec(`INSERT INTO observers(rowid,id,name) VALUES($1,$2,$2)`, i, fmt.Sprintf("merge-%d", i)); err != nil { + t.Fatal(err) + } + } + if _, err := writer.Exec(`INSERT INTO transmissions(id,raw_hex,hash,first_seen,last_seen,payload_type) VALUES(1,'AA','merge-history',$1,$2,4)`, now.Add(-3*time.Hour).Format(time.RFC3339), now.Unix()); err != nil { + t.Fatal(err) + } + addObservation := func(id int) { + t.Helper() + if _, err := writer.Exec(`INSERT INTO observations(id,transmission_id,observer_idx,path_json,timestamp) VALUES($1,1,$1,'[]',$2)`, id, now.Add(time.Duration(id)*time.Minute).Unix()); err != nil { + t.Fatal(err) + } + } + addObservation(3) + s := NewPacketStore(db, &PacketStoreConfig{MaxMemoryMB: 1, RetentionHours: 24}) + s.IngestNewFromDB(0, 10) + original := s.byTxID[1] + if original == nil || len(original.Observations) != 1 { + t.Fatal("live observation setup failed") + } + addObservation(1) + addObservation(2) + // The loader has finished its SQL scan but has not published it yet. + // A live poll is queued behind the advert-evidence gate with an observation + // not present in the loader's snapshot. It must extend the same pointer. + hookCalls := 0 + pollDone := make(chan struct{}) + db.advertEvidenceReadHook = func() { + hookCalls++ + if hookCalls == 1 { + addObservation(4) + go func() { + s.IngestNewObservations(3, 10) + close(pollDone) + }() + } + } + if err := s.loadChunk(now.Add(-24*time.Hour), now.Add(time.Hour)); err != nil { + t.Fatal(err) + } + if hookCalls == 0 { + t.Fatal("interleaving hook did not run") + } + select { + case <-pollDone: + case <-time.After(5 * time.Second): + t.Fatal("live poll remained blocked after loading") + } + assertStartupBudget(t, s, 4) + if s.byTxID[1] != original || len(s.packets) != 1 { + t.Fatal("load replaced the live pointer or duplicated its packet") + } + latest, err := time.Parse(time.RFC3339Nano, original.LatestSeen) + if err != nil || !latest.Equal(now.Add(4*time.Minute)) { + t.Fatalf("newer live observation lost: %q", original.LatestSeen) + } + bytes := s.trackedBytes + if err := s.loadChunk(now.Add(-24*time.Hour), now.Add(time.Hour)); err != nil { + t.Fatal(err) + } + assertStartupBudget(t, s, 4) + if s.trackedBytes != bytes || len(s.byPayloadType[4]) != 1 { + t.Fatal("overlapping load duplicated accounting or payload index") + } +} + +func BenchmarkStartupMemoryBudget(b *testing.B) { + benchmarkStartupMemoryBudget(b, 6000, 65) +} + +func BenchmarkStartupMemoryBudgetTypicalFanout(b *testing.B) { + benchmarkStartupMemoryBudget(b, 30000, 6) +} + +func benchmarkStartupMemoryBudget(b *testing.B, transmissions, observations int) { + b.StopTimer() + db := startupMemoryFixture(b, transmissions, observations, false) + var retained uint64 + for i := 0; i < b.N; i++ { + runtime.GC() + var before, after runtime.MemStats + runtime.ReadMemStats(&before) + s := NewPacketStore(db, &PacketStoreConfig{MaxMemoryMB: 16, RetentionHours: 24, HotStartupHours: 1}) + b.StartTimer() + if err := s.RunStartupLoad(10000); err != nil { + b.Fatal(err) + } + b.StopTimer() + runtime.GC() + runtime.ReadMemStats(&after) + if after.HeapAlloc > before.HeapAlloc { + retained += after.HeapAlloc - before.HeapAlloc + } + b.ReportMetric(s.trackedMemoryMB(), "cache-MiB") + runtime.KeepAlive(s) + } + b.ReportMetric(float64(retained)/float64(b.N)/1048576, "heap-MiB") +} diff --git a/cmd/server/startup_query_plan_test.go b/cmd/server/startup_query_plan_test.go new file mode 100644 index 000000000..78dc0eb20 --- /dev/null +++ b/cmd/server/startup_query_plan_test.go @@ -0,0 +1,58 @@ +package main + +import ( + "encoding/json" + "fmt" + "strconv" + "testing" + "time" +) + +func TestPostgresStartupUnlimitedCursorUsesPrimaryKeySeek(t *testing.T) { + postgresOnly(t) + db, writer := openPostgresValueFixture(t) + stamp := time.Now().UTC().Add(-2 * time.Hour).Truncate(time.Second) + copyTestRows(t, writer, "transmissions", []string{"id", "raw_hex", "hash", "first_seen", "last_seen"}, 20000, func(n int) []any { + // A large tie group proves the ID half of the cursor also seeks. + return []any{n, "AA", fmt.Sprintf("seek-%d", n), stamp.Format(time.RFC3339), stamp.Unix()} + }) + if _, err := writer.Exec("ANALYZE transmissions"); err != nil { + t.Fatal(err) + } + s := NewPacketStore(db, &PacketStoreConfig{}) + query, args := s.packetLoadQuery(stamp.Add(-time.Hour), time.Time{}, packetLoadCursor{id: 10000, recency: strconv.FormatInt(stamp.Unix(), 10)}, 100) + var raw []byte + if err := db.conn.QueryRow("EXPLAIN (ANALYZE, FORMAT JSON) "+query, args...).Scan(&raw); err != nil { + t.Fatal(err) + } + type planNode struct { + Type string `json:"Node Type"` + Relation string `json:"Relation Name"` + Index string `json:"Index Name"` + Rows float64 `json:"Actual Rows"` + Filtered float64 `json:"Rows Removed by Filter"` + Plans []planNode `json:"Plans"` + } + var plans []struct{ Plan planNode } + if err := json.Unmarshal(raw, &plans); err != nil || len(plans) != 1 { + t.Fatalf("invalid plan: %v", err) + } + pending := []planNode{plans[0].Plan} + indexed := false + for len(pending) > 0 { + node := pending[0] + pending = append(pending[1:], node.Plans...) + if node.Relation != "transmissions" { + continue + } + if node.Index == "transmissions_pkey" { + indexed = true + } + if node.Rows+node.Filtered > 100 { + t.Fatalf("100-row page rescanned history: %s (%s), rows=%g filtered=%g", node.Type, node.Index, node.Rows, node.Filtered) + } + } + if !indexed { + t.Fatal("unlimited startup cursor did not use the primary-key index") + } +} diff --git a/cmd/server/state_dir.go b/cmd/server/state_dir.go new file mode 100644 index 000000000..9e9f2c735 --- /dev/null +++ b/cmd/server/state_dir.go @@ -0,0 +1,21 @@ +package main + +import ( + "github.com/meshcore-analyzer/dbconfig" + "path/filepath" +) + +// runtimeStateDir is set once before the server's background workers start. +var runtimeStateDir = "data" + +// Queue helpers use an anchor filename; it is unrelated to the database URL. +func (db *DB) statePath() string { + dir := db.stateDir + if dir == "" { + dir = runtimeStateDir + } + if db.Backend() == dbconfig.SQLite { + return filepath.Join(dir, filepath.Base(db.path)) + } + return filepath.Join(dir, "meshcore") +} diff --git a/cmd/server/storage_backend.go b/cmd/server/storage_backend.go new file mode 100644 index 000000000..20b3ec6a6 --- /dev/null +++ b/cmd/server/storage_backend.go @@ -0,0 +1,117 @@ +package main + +import ( + "context" + "database/sql" + "fmt" + "net/url" + "path/filepath" + "strings" + "time" + + "github.com/jackc/pgx/v5" + _ "github.com/mattn/go-sqlite3" + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/dbschema/legacy" + "github.com/meshcore-analyzer/pgutil" +) + +// Backend is selected before opening a connection, never after an open fails. +func (db *DB) Backend() dbconfig.Backend { + if db == nil || db.backend == "" { + return dbconfig.SQLite + } + return db.backend +} +func (db *DB) parameter(n int) string { return db.Backend().Parameter(n) } +func (db *DB) nativeSQL(sqlite, postgres string) string { + if db.Backend() == dbconfig.SQLite { + return sqlite + } + return postgres +} + +// OpenStorage opens only the selected reader. The application performs the +// selected schema/import readiness check before starting its HTTP listener. +func OpenStorage(storage dbconfig.Storage) (*DB, error) { + var conn *sql.DB + var err error + path := storage.DBPath + switch storage.Backend { + case dbconfig.SQLite: + if strings.Contains(path, "://") || strings.HasPrefix(path, "postgres:") || strings.HasPrefix(path, "postgresql:") { + return nil, fmt.Errorf("SQLite telemetry requires a filesystem path") + } + var dsn string + dsn, err = dbconfig.SQLiteURI(path, url.Values{"mode": {"ro"}, "_query_only": {"on"}, "_busy_timeout": {"5000"}, "_cache_size": {"-2000"}}) + if err == nil { + conn, err = sql.Open("sqlite3", dsn) + } + case dbconfig.Postgres: + path = storage.ReaderDatabaseURL + conn, err = pgutil.Open(path, true) + default: + return nil, fmt.Errorf("telemetry requires an explicitly selected storage backend") + } + if err != nil { + return nil, err + } + conn.SetMaxOpenConns(4) + conn.SetMaxIdleConns(4) + if storage.Backend == dbconfig.Postgres { + if err = pgutil.AssertReadOnly(conn); err != nil { + conn.Close() + return nil, err + } + } + state := storage.StateDir + if state == "" { + if storage.Backend == dbconfig.SQLite { + state = filepath.Dir(path) + } else { + state = "data" + } + } + d := &DB{conn: conn, path: path, stateDir: state, backend: storage.Backend} + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + sc, err := conn.Conn(ctx) + if err != nil { + conn.Close() + return nil, fmt.Errorf("schema detection: acquire connection: %w", err) + } + err = d.detectSchema(ctx, sc) + _ = sc.Close() + if err != nil { + conn.Close() + return nil, fmt.Errorf("schema detection failed: %w", err) + } + if err = d.prepareStatements(); err != nil { + conn.Close() + return nil, fmt.Errorf("prepare statements: %w", err) + } + return d, nil +} + +func (db *DB) tableHasColumn(table, column string) (bool, error) { + if db.Backend() == dbconfig.SQLite { + return legacy.TableHasColumn(db.conn, table, column) + } + return dbschema.TableHasColumn(db.conn, table, column) +} + +// Only the PostgreSQL driver understands this extended-protocol query option. +func (db *DB) planWithValues(args []any) []any { + if db.Backend() == dbconfig.Postgres { + return append([]any{pgx.QueryExecModeExec}, args...) + } + return args +} + +func (db *DB) AssertReady() error { + if db.Backend() == dbconfig.SQLite { + return dbschema.AssertSQLiteReady(db.conn) + } + return dbschema.AssertPostgresReady(db.conn) +} diff --git a/cmd/server/storage_backend_test.go b/cmd/server/storage_backend_test.go new file mode 100644 index 000000000..645ea18a3 --- /dev/null +++ b/cmd/server/storage_backend_test.go @@ -0,0 +1,73 @@ +package main + +import ( + "database/sql" + "errors" + "github.com/meshcore-analyzer/dbconfig" + "os" + "path/filepath" + "testing" + + _ "github.com/mattn/go-sqlite3" + "github.com/meshcore-analyzer/dbschema" + "github.com/meshcore-analyzer/dbschema/legacy" +) + +func TestOpenDBNativeSQLiteWithoutPostgres(t *testing.T) { + t.Setenv("CORESCOPE_TEST_POSTGRES_URL", "") + path := filepath.Join(t.TempDir(), "telemetry.db") + dsn, err := legacy.WriterDSN(path) + if err != nil { + t.Fatal(err) + } + owner, err := sql.Open("sqlite3", dsn) + if err != nil { + t.Fatal(err) + } + defer owner.Close() + if err := dbschema.ApplySQLite(owner, nil); err != nil { + t.Fatal(err) + } + reader, err := OpenDB(path) + if err != nil { + t.Fatalf("native SQLite reader must work without PostgreSQL: %v", err) + } + defer reader.Close() + if _, err := reader.conn.Exec("CREATE TABLE forbidden_write(id INTEGER)"); err == nil { + t.Fatal("reader acquired write access") + } + var count int + if err := reader.conn.QueryRow("SELECT count(*) FROM transmissions").Scan(&count); err != nil || count != 0 { + t.Fatalf("reader count=%d err=%v", count, err) + } + if _, err := owner.Exec(`CREATE TABLE ` + dbconfig.SQLiteImportPendingTable + `(phase TEXT)`); err != nil { + t.Fatal(err) + } + if err := reader.AssertReady(); !errors.Is(err, dbconfig.ErrSQLiteImportIncomplete) { + t.Fatal("reader accepted incomplete converted target", err) + } +} + +func testBackend(t testing.TB) dbconfig.Backend { + t.Helper() + v := os.Getenv("CORESCOPE_TEST_BACKEND") + if v != "" && v != "sqlite" && v != "postgres" { + t.Fatalf("invalid CORESCOPE_TEST_BACKEND") + } + if v == "postgres" { + return dbconfig.Postgres + } + return dbconfig.SQLite +} +func testBackendValue() dbconfig.Backend { + if os.Getenv("CORESCOPE_TEST_BACKEND") == "postgres" { + return dbconfig.Postgres + } + return dbconfig.SQLite +} +func testNativeSQL(sqlite, postgres string) string { + if testBackendValue() == dbconfig.Postgres { + return postgres + } + return sqlite +} diff --git a/cmd/server/storage_config.go b/cmd/server/storage_config.go new file mode 100644 index 000000000..b554f062a --- /dev/null +++ b/cmd/server/storage_config.go @@ -0,0 +1,112 @@ +package main + +import ( + "errors" + "io" + "path/filepath" + "strings" + + "github.com/meshcore-analyzer/dbconfig" +) + +type storageFlags struct { + Backend dbconfig.Backend + DBPath, DatabaseURL, StateDir string +} + +// storageInputs gathers bootstrap values without guessing an installation's +// state. The shared persisted selector applies before ResolveStorage runs. +func (c *Config) storageInputs(baseDir string, flags storageFlags, getenv func(string) string) (dbconfig.StorageInputs, error) { + base, err := filepath.Abs(baseDir) + if err != nil { + return dbconfig.StorageInputs{}, errors.New("resolve storage base directory") + } + in := dbconfig.StorageInputs{BaseDir: base, DBPath: c.DBPath, StateDir: envOrValue(getenv, "CORESCOPE_STATE_DIR", c.StateDir), + EnvBackend: dbconfig.Backend(getenv("CORESCOPE_DB_BACKEND")), + DatabaseURL: envOrValue(getenv, "CORESCOPE_DATABASE_URL", c.DatabaseURL), + ReaderDatabaseURL: getenv("CORESCOPE_READER_DATABASE_URL"), WriterDatabaseURL: getenv("CORESCOPE_WRITER_DATABASE_URL"), + UsersDatabaseURL: getenv("CORESCOPE_USERS_DATABASE_URL"), ApprovedChannelsDatabaseURL: getenv("CORESCOPE_APPROVED_CHANNELS_DATABASE_URL")} + if in.DBPath == "" { + in.DBPath = getenv("DB_PATH") + } + if c.DB != nil { + in.Backend = c.DB.Backend + } + if c.UserManagement != nil { + in.UsersDBPath = strings.TrimSpace(c.UserManagement.DBPath) + in.UsersDatabaseURL = envOrValue(getenv, "CORESCOPE_USERS_DATABASE_URL", c.UserManagement.DatabaseURL) + } + if flags.Backend != "" { + in.Backend = flags.Backend + } + if flags.DBPath != "" { + in.DBPath = flags.DBPath + } + if flags.DatabaseURL != "" { + in.DatabaseURL = flags.DatabaseURL + in.ReaderDatabaseURL = flags.DatabaseURL + } + if flags.StateDir != "" { + in.StateDir = flags.StateDir + } + // Legacy explicit paths are relative to the working directory, whereas the + // server's omitted telemetry path defaults relative to config-dir. + for _, p := range []*string{&in.DBPath, &in.UsersDBPath} { + if *p == "" || strings.Contains(*p, "://") { + continue + } + if filepath.VolumeName(*p) != "" && !filepath.IsAbs(*p) { + return in, errors.New("drive-relative storage paths are ambiguous") + } + *p, err = filepath.Abs(*p) + if err != nil { + return in, errors.New("resolve SQLite storage path") + } + } + return in, nil +} + +// Discovery uses the stable legacy anchor, including after a backend switch. +func storageSelectionPath(raw dbconfig.StorageInputs) (string, error) { + state := raw.StateDir + if state == "" { + if strings.Contains(raw.DBPath, "://") || strings.HasPrefix(raw.DBPath, "postgres:") || strings.HasPrefix(raw.DBPath, "postgresql:") { + return "", errors.New("a database URL cannot be used as the legacy state directory anchor") + } + path := raw.DBPath + if path == "" { + path = filepath.Join(raw.BaseDir, "data", "meshcore.db") + } + state = filepath.Dir(path) + } + if strings.Contains(state, "://") || strings.ContainsRune(state, 0) { + return "", errors.New("invalid storage state directory") + } + if !filepath.IsAbs(state) { + state = filepath.Join(raw.BaseDir, state) + } + return dbconfig.SelectionPath(filepath.Clean(state)), nil +} +func resolveRuntimeStorage(raw dbconfig.StorageInputs) (dbconfig.Storage, io.Closer, error) { + path, err := storageSelectionPath(raw) + if err != nil { + return dbconfig.Storage{}, nil, err + } + selected, lease, err := dbconfig.OpenSelection(path) + if err != nil { + return dbconfig.Storage{}, nil, err + } + fail := func(err error) (dbconfig.Storage, io.Closer, error) { + lease.Close() + return dbconfig.Storage{}, nil, err + } + raw, err = selected.ApplyTo(raw) + if err != nil { + return fail(err) + } + storage, err := dbconfig.ResolveStorage(raw) + if err != nil { + return fail(err) + } + return storage, lease, nil +} diff --git a/cmd/server/storage_config_test.go b/cmd/server/storage_config_test.go new file mode 100644 index 000000000..75c2074c7 --- /dev/null +++ b/cmd/server/storage_config_test.go @@ -0,0 +1,155 @@ +package main + +import ( + "errors" + "path/filepath" + "testing" + + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/pgutil" +) + +func TestStorageInputsKeepBootstrapRawAndLegacyPrecedence(t *testing.T) { + base := t.TempDir() + cfg := &Config{DBPath: "configured.db", UserManagement: &UserManagementConfig{DBPath: "accounts.db"}} + env := func(k string) string { + if k == "DB_PATH" { + return "environment.db" + } + return "" + } + in, err := cfg.storageInputs(base, storageFlags{}, env) + if err != nil { + t.Fatal(err) + } + want, _ := filepath.Abs("configured.db") + if in.DBPath != want || in.StateDir != "" || in.Backend != "" || in.FreshInstall { + t.Fatalf("raw values/defaults changed: %+v", in) + } + in, err = cfg.storageInputs(base, storageFlags{DBPath: "cli.db"}, env) + if err != nil { + t.Fatal(err) + } + want, _ = filepath.Abs("cli.db") + if in.DBPath != want { + t.Fatal("CLI path did not win") + } + in, err = (&Config{}).storageInputs(base, storageFlags{}, func(string) string { return "" }) + if err != nil { + t.Fatal(err) + } + if _, err = dbconfig.ResolveStorage(in); err == nil { + t.Fatal("unknown existing install was guessed fresh") + } + in.FreshInstall = true + got, err := dbconfig.ResolveStorage(in) + if err != nil { + t.Fatal(err) + } + if got.Backend != dbconfig.SQLite || got.DBPath != filepath.Join(base, "data", "meshcore.db") { + t.Fatalf("fresh default: %+v", got) + } +} + +func TestRecordedPostgresPortCannotDriftThroughEnvironment(t *testing.T) { + t.Setenv("PGPORT", "") + const target = "postgresql://reader@localhost/telemetry?sslmode=disable" + dir := t.TempDir() + selected, err := dbconfig.NewSelection(dbconfig.Storage{Backend: dbconfig.Postgres, StateDir: dir, ReaderDatabaseURL: target, WriterDatabaseURL: target}) + if err != nil { + t.Fatal(err) + } + t.Setenv("PGPORT", "6543") + raw, err := selected.ApplyTo(dbconfig.StorageInputs{BaseDir: dir, DatabaseURL: target}) + if err != nil { + return + } // Rejecting a changed effective endpoint is safe too. + storage, err := dbconfig.ResolveStorage(raw) + if err != nil { + t.Fatal(err) + } + config, err := pgutil.ParseConfig(storage.ReaderDatabaseURL) + if err != nil { + t.Fatal(err) + } + if config.Port != selected.Telemetry.Postgres.Port { + t.Fatalf("recorded port %d became effective port %d", selected.Telemetry.Postgres.Port, config.Port) + } +} + +func TestStorageInputsReaderURLAndStatePrecedence(t *testing.T) { + base := t.TempDir() + cfg := &Config{DB: &DBConfig{Backend: dbconfig.Postgres}, DatabaseURL: "postgresql://reader@localhost/config", StateDir: "configured"} + env := func(k string) string { + switch k { + case "CORESCOPE_DATABASE_URL": + return "postgresql://reader@localhost/generic" + case "CORESCOPE_READER_DATABASE_URL": + return "postgresql://reader@localhost/reader" + case "CORESCOPE_STATE_DIR": + return "state" + } + return "" + } + in, err := cfg.storageInputs(base, storageFlags{}, env) + if err != nil { + t.Fatal(err) + } + got, err := dbconfig.ResolveStorage(in) + if err != nil { + t.Fatal(err) + } + if got.ReaderDatabaseURL != env("CORESCOPE_READER_DATABASE_URL") || got.StateDir != filepath.Join(base, "state") { + t.Fatal("reader environment precedence changed") + } + in, err = cfg.storageInputs(base, storageFlags{DatabaseURL: "postgresql://reader@localhost/cli", StateDir: "cli-state"}, env) + if err != nil { + t.Fatal(err) + } + got, err = dbconfig.ResolveStorage(in) + if err != nil { + t.Fatal(err) + } + if got.ReaderDatabaseURL != "postgresql://reader@localhost/cli" || got.WriterDatabaseURL != "postgresql://reader@localhost/cli" || got.StateDir != filepath.Join(base, "cli-state") { + t.Fatal("CLI overrides lost") + } +} + +func TestRuntimeUsesInstalledSelectionAndHoldsLease(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "selected.db") + selected, err := dbconfig.NewSelection(dbconfig.Storage{Backend: dbconfig.SQLite, StateDir: dir, DBPath: path, UsersDBPath: filepath.Join(dir, "users.db")}) + if err != nil { + t.Fatal(err) + } + if _, err = dbconfig.AdoptSelection(dbconfig.SelectionPath(dir), selected); err != nil { + t.Fatal(err) + } + raw := dbconfig.StorageInputs{BaseDir: dir, StateDir: dir, Backend: dbconfig.Postgres, EnvBackend: dbconfig.Postgres, DBPath: filepath.Join(dir, "old.db"), DatabaseURL: "postgresql://unused@localhost/old"} + storage, lease, err := resolveRuntimeStorage(raw) + if err != nil { + t.Fatal(err) + } + defer lease.Close() + if storage.Backend != dbconfig.SQLite || storage.DBPath != path { + t.Fatal("recorded SQLite choice not honored") + } + if _, err := dbconfig.BeginSelectionUpdate(dbconfig.SelectionPath(dir), selected.Generation); !errors.Is(err, dbconfig.ErrSelectionBusy) { + t.Fatal("reader did not retain selection lease", err) + } +} +func TestReaderMissingSelectionDoesNotAdoptOrHoldLease(t *testing.T) { + dir := t.TempDir() + raw := dbconfig.StorageInputs{BaseDir: dir, StateDir: dir, DBPath: filepath.Join(dir, "legacy.db")} + if _, lease, err := resolveRuntimeStorage(raw); !errors.Is(err, dbconfig.ErrSelectionMissing) || lease != nil { + t.Fatal("reader guessed unrecorded storage", err) + } + update, err := dbconfig.BeginSelectionUpdate(dbconfig.SelectionPath(dir), "") + if err != nil { + t.Fatal("reader blocked setup adoption", err) + } + defer update.Close() + if err = update.Abort(); err != nil { + t.Fatal(err) + } +} diff --git a/cmd/server/storage_diagnostics_test.go b/cmd/server/storage_diagnostics_test.go new file mode 100644 index 000000000..d48eb0d2f --- /dev/null +++ b/cmd/server/storage_diagnostics_test.go @@ -0,0 +1,44 @@ +package main + +import ( + "encoding/json" + "net/http/httptest" + "testing" + + "github.com/meshcore-analyzer/dbconfig" +) + +func TestDatabaseDiagnosticsMatchSelectedBackend(t *testing.T) { + fixture := setupTestDB(t) + reader, err := openFixtureReader(t, fixture.path) + if err != nil { + t.Fatal(err) + } + defer reader.Close() + s := &Server{db: reader} + w := httptest.NewRecorder() + s.handlePerfPostgres(w, httptest.NewRequest("GET", "/api/perf/database", nil)) + var response struct { + Engine string `json:"engine"` + PageCount *int64 `json:"pageCount"` + WalSize *int64 `json:"walSize"` + CacheHitRate *float64 `json:"cacheHitRate"` + } + if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil { + t.Fatal(err) + } + if reader.Backend() == dbconfig.SQLite { + if response.Engine != "sqlite" || response.PageCount == nil || *response.PageCount <= 0 || response.WalSize == nil { + t.Fatalf("missing native SQLite diagnostics: %s", w.Body.String()) + } + if response.CacheHitRate != nil { + t.Fatal("SQLite must not label PacketStore hits as database cache hits") + } + var stats int + if err := reader.conn.QueryRow(`SELECT count(*) FROM sqlite_master WHERE name='sqlite_stat1'`).Scan(&stats); err != nil || stats != 0 { + t.Fatal("reader diagnostics attempted planner maintenance", stats, err) + } + } else if response.Engine != "postgresql" || response.PageCount != nil || response.WalSize != nil { + t.Fatalf("PostgreSQL must not publish SQLite counters: %s", w.Body.String()) + } +} diff --git a/cmd/server/store.go b/cmd/server/store.go index a21758220..a8f76fb1f 100644 --- a/cmd/server/store.go +++ b/cmd/server/store.go @@ -61,11 +61,11 @@ type StoreTx struct { LatestSeen string // max observation timestamp (or FirstSeen if no observations) UniqueObserverCount int // cached count of distinct observer IDs // Cached parsed fields (set once, read many) - parsedPath []string // cached parsePathJSON result - pathParsed bool // whether parsedPath has been set - AdvertRouteEvidence uint8 // union of known flood/direct-empty-path evidence - decodedOnce sync.Once // guards parsedDecoded - parsedDecoded map[string]interface{} // cached json.Unmarshal of DecodedJSON + parsedPath []string // cached parsePathJSON result + pathParsed bool // whether parsedPath has been set + AdvertRouteEvidence uint8 // union of known flood/direct-empty-path evidence + decodedMu sync.Mutex + decodedCache atomic.Pointer[txDecodedCache] // Dedup maps, built only once a transmission passes obsIndexThreshold // observations (see obs_dedup.go); nil below it. obsKeys map[obsDedupKey]struct{} @@ -85,7 +85,7 @@ type StoreObs struct { Direction string SNR *float64 RSSI *float64 - Score *int + Score *float64 PathJSON string RawHex string Timestamp string @@ -171,18 +171,51 @@ func broadcastPayload(tx *StoreTx) json.RawMessage { return json.RawMessage(tx.DecodedJSON) } -// ParsedDecoded returns the parsed DecodedJSON map, caching the result. -// Thread-safe via sync.Once — the first call parses, subsequent calls return cached. +// Each decoded source generation owns its once and immutable parsed map. +// A rewrite retires the whole generation; readers can finish using the old one. +type txDecodedCache struct { + source string + once sync.Once + parsed map[string]interface{} +} + +// ParsedDecoded returns a cached, read-only map. Callers must not mutate it or +// its nested values. The publication mutex never covers JSON parsing: a rewrite +// holding store.mu can publish a new generation without waiting for old readers. func (tx *StoreTx) ParsedDecoded() map[string]interface{} { - tx.decodedOnce.Do(func() { - if tx.DecodedJSON != "" { - json.Unmarshal([]byte(tx.DecodedJSON), &tx.parsedDecoded) + cache := tx.decodedCache.Load() + if cache == nil { + tx.decodedMu.Lock() + cache = tx.decodedCache.Load() + if cache == nil { + cache = &txDecodedCache{source: tx.DecodedJSON} + tx.decodedCache.Store(cache) + } + tx.decodedMu.Unlock() + } + cache.once.Do(func() { + if cache.source != "" { + json.Unmarshal([]byte(cache.source), &cache.parsed) for _, k := range decodedCacheDroppedKeys { - delete(tx.parsedDecoded, k) + delete(cache.parsed, k) } } }) - return tx.parsedDecoded + return cache.parsed +} + +// setDecodedJSON is the only mutation of a published transmission's decoded +// source. The caller holds store.mu to protect ordinary field readers; decodedMu +// also protects cache initialization by analytics that already released store.mu. +func (tx *StoreTx) setDecodedJSON(decoded string) bool { + tx.decodedMu.Lock() + defer tx.decodedMu.Unlock() + if tx.DecodedJSON == decoded { + return false + } + tx.DecodedJSON = decoded + tx.decodedCache.Store(nil) + return true } // PacketStore holds all transmissions in memory with indexes for fast queries. @@ -515,36 +548,38 @@ type PacketStore struct { // Clock skew detection engine. clockSkew *ClockSkewEngine - // Bounded cold load: oldest packet timestamp loaded into memory. - // Empty string means all data is in memory (no limit applied). - oldestLoaded string + // Conservative timestamp boundary for complete cache coverage. A partial + // timestamp group stays behind this boundary and is queried from SQL. + // Empty string means no fallback boundary has been established. + oldestLoaded string + startupMemoryLimited bool // bounded cache is complete; omitted history remains in SQL // evictedThrough is the newest FirstSeen eviction has removed (#748). // Retention eviction stays behind now-retentionHours, but memory-pressure // eviction cuts further in; coverageStart needs that edge. evictedThrough string - // loadCapEdge is when a startup load cut short by the memory cap - // finished (#748). The rows the cap left out are never loaded later - // (the poller starts from the DB's max id), so the store is only - // complete from here on. + // loadCapEdge is the conservative recency boundary of the first group + // omitted by memory-limited startup (#748). Newest-first admission keeps + // every newer group; the poller starts from the DB's high-water marks. loadCapEdge string // Hot startup atomic gates — see contract below. // // Contract / ordering invariant (PR #1187): - // * hashMigrationComplete (set by migrateContentHashesAsync) gates - // content-hash–dependent code paths (e.g. dedup correctness on the - // write side). Set true ONLY after the migration loop finishes. + // * hashMigrationComplete (set by verifyContentHashesAsync) gates + // the diagnostic that all loaded content hashes match the current + // formula. Only the ingestor or offline migration can repair hashes. // * backgroundLoadDone is set true exactly once, after // loadBackgroundChunks finishes its loop AND its post-load index - // rebuild. It gates "hot startup has finished filling - // retentionHours of data into memory" (used by /api/perf and the + // rebuild. It gates "hot startup has filled the configured retention + // window or reached its memory budget" (used by /api/perf and the // UI's hot-load banner). It says nothing about success — see // backgroundLoadFailed for the success/failure signal. // * backgroundLoadFailed is set true ONLY if at least one chunk - // errored during background load. /api/perf surfaces it so - // operators can distinguish "done & full" from "done & partial". + // errored or coverage unexpectedly fell short during background load. + // Reaching the configured byte budget is not an error; the separate + // loadCoverageRatio continues to report the actual cached fraction. // Read order MUST be: load backgroundLoadDone first; only if true // is backgroundLoadFailed meaningful. // 0 = disabled (current behavior). Background loader fills the rest. @@ -584,7 +619,7 @@ type PacketStore struct { // perf payload for prod observability. loadCoverageRatio float64 - // Async hash migration state: set after migrateContentHashesAsync completes. + // Loaded content-hash verification state; retained API field name. hashMigrationComplete atomic.Bool // Chunked startup load state (#1009). LoadChunked closes @@ -859,10 +894,13 @@ func NewPacketStore(db *DB, cfg *PacketStoreConfig, cacheTTLs ...map[string]inte return ps } -// Load reads transmissions + observations from SQLite into memory. +// Load reads transmissions + observations from the selected backend into memory. // When maxMemoryMB > 0, loads only the newest N transmissions that fit // within the memory budget, avoiding OOM on large databases. func (s *PacketStore) Load() error { + if s.maxMemoryMB > 0 { + return s.LoadChunked(0) + } s.advertEvidenceMu.Lock() defer s.advertEvidenceMu.Unlock() s.mu.Lock() @@ -882,27 +920,6 @@ func (s *PacketStore) Load() error { t0 := time.Now() - // Count total transmissions for logging. - var totalInDB int - if err := s.db.conn.QueryRow("SELECT COUNT(*) FROM transmissions").Scan(&totalInDB); err != nil { - totalInDB = -1 // non-fatal - } - - // Calculate max packets to load based on memory budget. - var maxPackets int64 - if s.maxMemoryMB > 0 { - // Use a typical packet with ~10 observations as the estimate. - avgBytes := int64(1000) // conservative floor - if sample := estimateStoreTxBytesTypical(10); sample > avgBytes { - avgBytes = sample - } - maxPackets = (int64(s.maxMemoryMB) * 1048576) / avgBytes - if maxPackets < 1000 { - maxPackets = 1000 // minimum 1000 packets - } - } - // maxPackets == 0 means unlimited - var loadSQL string rpCol := "" if s.db.hasResolvedPath { @@ -944,25 +961,19 @@ func (s *PacketStore) Load() error { loadConditions = append(loadConditions, fmt.Sprintf("t.first_seen >= '%s'", hotCutoffStr)) } } - if maxPackets > 0 { - if s.db.hasLastSeen { - loadConditions = append(loadConditions, fmt.Sprintf( - "t.id IN (SELECT id FROM transmissions ORDER BY last_seen DESC LIMIT %d)", maxPackets)) - } else { - loadConditions = append(loadConditions, fmt.Sprintf( - "t.id IN (SELECT id FROM transmissions ORDER BY first_seen DESC LIMIT %d)", maxPackets)) - } - } filterClause := "" if len(loadConditions) > 0 { filterClause = "\n\t\t\tWHERE " + strings.Join(loadConditions, "\n\t\t\t AND ") } if s.db.isV3 { - loadSQL = `SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type, + loadSQL = s.db.nativeSQL(`SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type, t.payload_type, t.payload_version, t.decoded_json, o.id, obs.id, obs.name, COALESCE(obs.iata, ''), o.direction, - o.snr, o.rssi, o.score, o.path_json, strftime('%Y-%m-%dT%H:%M:%fZ', o.timestamp, 'unixepoch')` + obsRawHexCol + rpCol + scopeNameCol + ` + o.snr, o.rssi, o.score, o.path_json, strftime('%Y-%m-%dT%H:%M:%fZ', o.timestamp, 'unixepoch')`, `SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type, + t.payload_type, t.payload_version, t.decoded_json, + o.id, obs.id, obs.name, COALESCE(obs.iata, ''), o.direction, + o.snr, o.rssi, o.score, o.path_json, to_char(to_timestamp(o.timestamp) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')`) + obsRawHexCol + rpCol + scopeNameCol + ` FROM transmissions t LEFT JOIN observations o ON o.transmission_id = t.id LEFT JOIN observers obs ON obs.rowid = o.observer_idx` + filterClause + ` @@ -986,7 +997,7 @@ func (s *PacketStore) Load() error { // which would silently mis-attribute hops). Fetched BEFORE opening // the rows cursor below: getCachedNodesAndPM issues its own DB query, // which would deadlock against the still-open cursor on a single- - // connection SQLite pool. Load holds s.mu.Lock(), so calling + // connection database pool. Load holds s.mu.Lock(), so calling // getCachedNodesAndPM directly is safe. _, relayPM := s.getCachedNodesAndPM() var coldLoadAmbiguousHopsSkipped int @@ -1007,7 +1018,7 @@ func (s *PacketStore) Load() error { var obsID sql.NullInt64 var observerID, observerName, observerIATA, direction, pathJSON, obsTimestamp sql.NullString var snr, rssi sql.NullFloat64 - var score sql.NullInt64 + var score sql.NullFloat64 var obsRawHex sql.NullString var resolvedPathStr sql.NullString var scopeName sql.NullString @@ -1080,7 +1091,7 @@ func (s *PacketStore) Load() error { Direction: internStr(nullStrVal(direction)), SNR: nullFloatPtr(snr), RSSI: nullFloatPtr(rssi), - Score: nullIntPtr(score), + Score: nullFloatPtr(score), PathJSON: obsPJ, Timestamp: normalizeTimestamp(nullStrVal(obsTimestamp)), } @@ -1174,19 +1185,10 @@ func (s *PacketStore) Load() error { s.oldestLoaded = s.packets[0].FirstSeen } - if maxPackets > 0 && totalInDB > len(s.packets) { - s.loadCapEdge = time.Now().UTC().Format(time.RFC3339) - } - s.loaded = true elapsed := time.Since(t0) - if maxPackets > 0 && totalInDB > len(s.packets) { - log.Printf("[store] Loaded %d/%d transmissions (%d observations) in %v — bounded by %dMB budget (tracked ~%.0fMB, heap ~%.0fMB)", - len(s.packets), totalInDB, s.totalObs, elapsed, s.maxMemoryMB, s.trackedMemoryMB(), s.estimatedMemoryMB()) - } else { - log.Printf("[store] Loaded %d transmissions (%d observations) in %v (tracked ~%.0fMB, heap ~%.0fMB)", - len(s.packets), s.totalObs, elapsed, s.trackedMemoryMB(), s.estimatedMemoryMB()) - } + log.Printf("[store] Loaded %d transmissions (%d observations) in %v (tracked ~%.0fMB, heap ~%.0fMB)", + len(s.packets), s.totalObs, elapsed, s.trackedMemoryMB(), s.estimatedMemoryMB()) s.loadMultibyteCapFromDB() if coldLoadAmbiguousHopsSkipped > 0 { log.Printf("[store] cold load: skipped %d ambiguous-prefix relay hops (unique_prefix gate, PR #1643 R1)", @@ -1201,7 +1203,7 @@ func (s *PacketStore) Load() error { return nil } -// loadChunk queries a [from, to) time window from SQLite without holding the +// loadChunk queries a [from, to) time window without holding the // write lock, builds local data structures, then merges them into the store // under s.mu.Lock(). It is the building block for the background loader. // @@ -1212,412 +1214,12 @@ func (s *PacketStore) Load() error { // distHops are NOT updated here — the caller (loadBackgroundChunks) rebuilds // those once after all chunks are merged. -// accumulateDedup appends pks to byTx[txID], deduplicating per-tx via the -// parallel seenByTx set. Used by loadChunk to build the per-tx relay-hop -// pubkey unions outside the merge critical section. -func accumulateDedup(byTx map[int][]string, seenByTx map[int]map[string]bool, txID int, pks []string) { - if len(pks) == 0 { - return - } - seen := seenByTx[txID] - if seen == nil { - seen = make(map[string]bool, len(pks)) - seenByTx[txID] = seen - } - for _, pk := range pks { - if seen[pk] { - continue - } - seen[pk] = true - byTx[txID] = append(byTx[txID], pk) - } -} - +// loadChunk uses the same observation-aware budget and bounded SQL pages as +// the initial load. A daily background window never becomes an unbounded +// intermediate slice before the cache can enforce its limit. func (s *PacketStore) loadChunk(from, to time.Time) error { - fromStr := from.UTC().Format(time.RFC3339) - toStr := to.UTC().Format(time.RFC3339) - fromUnix := from.UTC().Unix() - toUnix := to.UTC().Unix() - _ = fromStr - _ = toStr - - // Build the same SQL as Load() but with a [from, to) window. - rpCol := "" - if s.db.hasResolvedPath { - rpCol = ",\n\t\t\t\to.resolved_path" - } - obsRawHexCol := "" - if s.db.hasObsRawHex { - obsRawHexCol = ", o.raw_hex" - } - // #1751: scope_name is on the transmission row; append as the last column. - scopeNameCol := "" - if s.db.hasScopeName { - scopeNameCol = ", t.scope_name" - } - - // #1690: window on the denormalized last_seen (effective recency) - // rather than first_seen. See chunked_load.go for the full rationale. - // Test/legacy DBs without the column fall back to first_seen. - var filterClause string - var qArgs []interface{} - if s.db.hasLastSeen { - filterClause = "\n\t\t\tWHERE t.last_seen >= ? AND t.last_seen < ?" - qArgs = []interface{}{fromUnix, toUnix} - } else { - filterClause = "\n\t\t\tWHERE t.first_seen >= ? AND t.first_seen < ?" - qArgs = []interface{}{fromStr, toStr} - } - - var chunkSQL string - if s.db.isV3 { - chunkSQL = `SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type, - t.payload_type, t.payload_version, t.decoded_json, - o.id, obs.id, obs.name, COALESCE(obs.iata, ''), o.direction, - o.snr, o.rssi, o.score, o.path_json, strftime('%Y-%m-%dT%H:%M:%fZ', o.timestamp, 'unixepoch')` + obsRawHexCol + rpCol + scopeNameCol + ` - FROM transmissions t - LEFT JOIN observations o ON o.transmission_id = t.id - LEFT JOIN observers obs ON obs.rowid = o.observer_idx` + filterClause + ` - ORDER BY t.first_seen ASC, o.timestamp DESC` - } else { - chunkSQL = `SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type, - t.payload_type, t.payload_version, t.decoded_json, - o.id, o.observer_id, o.observer_name, COALESCE(obs.iata, ''), o.direction, - o.snr, o.rssi, o.score, o.path_json, o.timestamp` + obsRawHexCol + rpCol + scopeNameCol + ` - FROM transmissions t - LEFT JOIN observations o ON o.transmission_id = t.id - LEFT JOIN observers obs ON obs.id = o.observer_id` + filterClause + ` - ORDER BY t.first_seen ASC, o.timestamp DESC` - } - - // Relay-hop fallback inputs. observations.resolved_path is NULL on - // every live deployment (since #1287 the ingestor persists relay data - // as aggregate neighbor_edges, not per-observation resolved_path), so - // for this background-loaded older window we re-resolve relay hops - // from the persisted path_json using the prefix map. PR #1643 R1 - // munger #1: cold load resolves ONLY when the prefix is unique - // (affinity-tier resolution against ≤168h-old observations would - // silently mis-attribute hops). Fetched BEFORE opening the rows - // cursor below: getCachedNodesAndPM issues its own DB query, which - // would deadlock against the open cursor on a single-connection SQLite - // pool. - s.mu.RLock() - _, relayPM := s.getCachedNodesAndPM() - s.mu.RUnlock() - var coldLoadAmbiguousHopsSkipped int - - rows, err := s.db.conn.Query(chunkSQL, qArgs...) - if err != nil { - return err - } - defer rows.Close() - - // Local data structures — built without holding the lock. - localByHash := make(map[string]*StoreTx) - localPackets := make([]*StoreTx, 0) - localByTxID := make(map[int]*StoreTx) - localByObsID := make(map[int]*StoreObs) - localByObserver := make(map[string][]*StoreObs) - // Issue #1558: accumulate the union of resolved_path relay-hop - // pubkeys per tx outside the lock. We unmarshal + dedupe here so the - // merge critical section below only does map-append work, mirroring - // the rest of loadChunk's "build local, merge under lock" shape. - localResolvedPKsByTx := make(map[int][]string) - localResolvedSeenByTx := make(map[int]map[string]bool) - // Path_json fallback pubkeys (resolved_path NULL): indexed into byNode - // ONLY at merge, kept separate from localResolvedPKsByTx because they must - // NOT enter the resolved_path/path-hop indexes (see the per-row comment). - localByNodePKsByTx := make(map[int][]string) - localByNodeSeenByTx := make(map[int]map[string]bool) - var localTotalObs int - var localTrackedBytes int64 - var localMaxTxID int - var localMaxObsID int - - // Scan overwrites every target on each row, so one set (and one - // argument slice) serves the whole loop instead of escaping per row. - var txID int - var rawHex, hash, firstSeen, decodedJSON sql.NullString - var routeType, payloadType, payloadVersion sql.NullInt64 - var obsID sql.NullInt64 - var observerID, observerName, observerIATA, direction, pathJSON, obsTimestamp sql.NullString - var snr, rssi sql.NullFloat64 - var score sql.NullInt64 - var obsRawHex sql.NullString - var resolvedPathStr sql.NullString - var scopeName sql.NullString - - scanArgs := []interface{}{&txID, &rawHex, &hash, &firstSeen, &routeType, &payloadType, - &payloadVersion, &decodedJSON, - &obsID, &observerID, &observerName, &observerIATA, &direction, - &snr, &rssi, &score, &pathJSON, &obsTimestamp} - if s.db.hasObsRawHex { - scanArgs = append(scanArgs, &obsRawHex) - } - if s.db.hasResolvedPath { - scanArgs = append(scanArgs, &resolvedPathStr) - } - if s.db.hasScopeName { - scanArgs = append(scanArgs, &scopeName) - } - - for rows.Next() { - if err := rows.Scan(scanArgs...); err != nil { - log.Printf("[store] loadChunk scan error: %v", err) - continue - } - - hashStr := nullStrVal(hash) - tx := localByHash[hashStr] - if tx == nil { - tx = &StoreTx{ - ID: txID, - RawHex: nullStrVal(rawHex), - Hash: hashStr, - FirstSeen: nullStrVal(firstSeen), - LatestSeen: nullStrVal(firstSeen), - RouteType: nullIntPtr(routeType), - PayloadType: nullIntPtr(payloadType), - DecodedJSON: nullStrVal(decodedJSON), - ScopeName: nullStrPtr(scopeName), - } - localByHash[hashStr] = tx - localPackets = append(localPackets, tx) - localByTxID[txID] = tx - if txID > localMaxTxID { - localMaxTxID = txID - } - localTrackedBytes += rechargeTx(tx) - } - - if obsID.Valid { - oid := int(obsID.Int64) - obsIDStr := nullStrVal(observerID) - obsPJ := nullStrVal(pathJSON) - - if tx.hasObservation(obsIDStr, obsPJ) { - continue - } - - obs := &StoreObs{ - ID: oid, - TransmissionID: txID, - ObserverID: internStr(obsIDStr), - ObserverName: internStr(nullStrVal(observerName)), - ObserverIATA: internStr(nullStrVal(observerIATA)), - Direction: internStr(nullStrVal(direction)), - SNR: nullFloatPtr(snr), - RSSI: nullFloatPtr(rssi), - Score: nullIntPtr(score), - PathJSON: obsPJ, - Timestamp: normalizeTimestamp(nullStrVal(obsTimestamp)), - } - - tx.addObservation(obs) - if obs.Timestamp > tx.LatestSeen { - tx.LatestSeen = obs.Timestamp - } - - localByObsID[oid] = obs - if oid > localMaxObsID { - localMaxObsID = oid - } - if obsIDStr != "" { - localByObserver[obsIDStr] = append(localByObserver[obsIDStr], obs) - } - localTotalObs++ - localTrackedBytes += estimateStoreObsBytes(obs) - - // Issue #1558: collect resolved_path relay-hop pubkeys for - // this observation. We unmarshal + dedupe OUTSIDE the merge - // critical section so the lock-held work in the per-batch - // merge below stays bounded. Without this, background-loaded - // transmissions silently miss byNode entries for every relay - // they were heard via — Load() does the equivalent inline. - rpStr := nullStrVal(resolvedPathStr) - if rpStr != "" { - pks := resolvedPubkeysFromJSON(rpStr) - accumulateDedup(localResolvedPKsByTx, localResolvedSeenByTx, txID, pks) - } else if relayPM != nil && obsPJ != "" && obsPJ != "[]" { - // resolved_path not persisted — reconstruct relay hops from - // path_json so the older retention window keeps relay-node - // history across a restart (mirrors the hot-window fix in - // scanAndMergeChunk and the live ingest path). These go into a - // SEPARATE accumulator indexed into byNode ONLY at merge — they - // must not enter the resolved_path/path-hop indexes, which - // handleNodePaths cross-checks against the NULL resolved_path - // column (#1352). - // PR #1643 R1 munger #1: unique_prefix-only gate. - rp := resolvePathForObsColdLoad(obsPJ, obsIDStr, tx, relayPM, &coldLoadAmbiguousHopsSkipped) - pks := extractResolvedPubkeys(rp) - accumulateDedup(localByNodePKsByTx, localByNodeSeenByTx, txID, pks) - } - } - } - if err := rows.Err(); err != nil { - return err - } - - // Pick best observation for each local packet before merging. - for _, tx := range localPackets { - pickBestObservation(tx) - localTrackedBytes += rechargeTx(tx) - } - - if len(localPackets) == 0 { - return nil - } - rows.Close() - s.advertEvidenceMu.Lock() - defer s.advertEvidenceMu.Unlock() - ids := advertTxIDs(localPackets) - masks, err := s.db.advertEvidenceForIDs(ids) - if err != nil { - return err - } - for _, tx := range localPackets { - tx.AdvertRouteEvidence = masks[tx.ID] - } - - // PR #1187 r3 MUST-FIX 1: index↔slice consistency. - // - // The previous (r2 #6, commit 2ec762aa) merge order was: - // 1) prepend s.packets in one critical section - // 2) populate s.byHash/byTxID/byObsID/byObserver/byNode/byPayloadType - // in separate per-batch critical sections - // 3) bump counters in a third section - // - // Any RLock-holding reader between steps observed packets present in - // s.packets but missing from s.byHash → silent partial data loss in - // GetPacketByHash and the QueryPackets hash/node fast-paths. - // - // We invert the order: build the indexes FIRST in bounded per-batch - // critical sections, then under a SINGLE final critical section - // prepend s.packets AND bump counters AND advance maxIDs. The - // invariant "every tx in s.packets is in s.byHash/s.byTxID" holds at - // every RLock instant — readers either see the old state, or an - // intermediate state where indexes contain a superset of s.packets - // (which is harmless: nothing in s.packets dangles), or the fully - // merged new state. - const mergeBatchSize = 500 - - for batchStart := 0; batchStart < len(localPackets); batchStart += mergeBatchSize { - batchEnd := batchStart + mergeBatchSize - if batchEnd > len(localPackets) { - batchEnd = len(localPackets) - } - batch := localPackets[batchStart:batchEnd] - - // Build the per-batch index views outside the lock so the - // critical section below is O(batch), not O(total local). - batchHashes := make(map[string]*StoreTx, len(batch)) - batchTxIDs := make(map[int]*StoreTx, len(batch)) - batchObsIDs := make(map[int]*StoreObs) - batchByObserver := make(map[string][]*StoreObs) - for _, tx := range batch { - batchHashes[tx.Hash] = tx - batchTxIDs[tx.ID] = tx - for _, o := range tx.Observations { - batchObsIDs[o.ID] = o - if o.ObserverID != "" { - batchByObserver[o.ObserverID] = append(batchByObserver[o.ObserverID], o) - } - } - } - - s.mu.Lock() - // Issue #1558: hopsSeen scratch map for indexResolvedPathHops → - // addResolvedPubkeysToPathHopIndex. Allocated once per batch and - // reused across txs (clear()d on each call inside the helper). - hopsSeen := make(map[string]bool) - newObsIDs := make(map[int]bool, len(batchObsIDs)) - for k := range batchObsIDs { - if s.byObsID[k] == nil { - newObsIDs[k] = true - } - } - evidenceChanged := false - for k, v := range batchHashes { - if s.byHash[k] == nil { - s.byHash[k] = v - } else { - // A live-loaded copy may already carry newer bits. Union in - // both directions before mergeChunkIntoPackets chooses it. - if unionAdvertEvidence(s.byHash[k], v.AdvertRouteEvidence) { - evidenceChanged = true - } - v.AdvertRouteEvidence |= s.byHash[k].AdvertRouteEvidence - } - } - if evidenceChanged { - s.invalidateAdvertEvidence() - } - for k, v := range batchTxIDs { - if s.byTxID[k] == nil { - s.byTxID[k] = v - } - } - for k, v := range batchObsIDs { - if newObsIDs[k] { - s.byObsID[k] = v - } - } - for observerID, obsList := range batchByObserver { - for _, o := range obsList { - if newObsIDs[o.ID] { - s.byObserver[observerID] = append(s.byObserver[observerID], o) - } - } - } - for _, tx := range batch { - s.indexByNode(tx) - if tx.PayloadType != nil { - pt := *tx.PayloadType - s.byPayloadType[pt] = append(s.byPayloadType[pt], tx) - } - s.trackAdvertPubkey(tx) - // Issue #1558: mirror Load()'s 783-799 resolved-path branch. - // Without this, background-loaded transmissions never enter - // byNode under their relay-hop pubkeys → Home-page per-node - // stats collapse after restart for relay-heavy nodes. The - // pubkey union was pre-built outside the lock above. - if pks := localResolvedPKsByTx[tx.ID]; len(pks) > 0 { - s.indexResolvedPathHops(tx, pks, hopsSeen) - } - // path_json fallback (resolved_path NULL): byNode ONLY, so relay - // nodes keep their analytics history across a restart without - // polluting the resolved_path/path-hop indexes (#1352). - for _, pk := range localByNodePKsByTx[tx.ID] { - s.addToByNode(tx, pk) - } - } - s.mu.Unlock() - runtime.Gosched() - } - - // Final atomic step: now that every tx in localPackets is fully - // indexed, publish it into s.packets and bump counters in one short - // critical section. After this point the new state is fully visible; - // before it readers see the old slice (which is still fully indexed). - s.mu.Lock() - s.packets = mergeChunkIntoPackets(localPackets, s.packets) - s.invalidateAdvertEvidence() - s.totalObs += localTotalObs - s.trackedBytes += localTrackedBytes - if localMaxTxID > s.maxTxID { - s.maxTxID = localMaxTxID - } - if localMaxObsID > s.maxObsID { - s.maxObsID = localMaxObsID - } - s.mu.Unlock() - - log.Printf("[store] background chunk [%s, %s) merged: %d tx, %d obs", fromStr, toStr, len(localPackets), localTotalObs) - if coldLoadAmbiguousHopsSkipped > 0 { - log.Printf("[store] background chunk: skipped %d ambiguous-prefix relay hops (unique_prefix gate, PR #1643 R1)", - coldLoadAmbiguousHopsSkipped) - } - return nil + _, err := s.loadPacketRange(from, to, 500, nil) + return err } // loadBackgroundChunks fills the remaining retentionHours window by loading @@ -1673,10 +1275,10 @@ func (s *PacketStore) loadBackgroundChunks() { var countQuery string var countArg interface{} if s.db.hasLastSeen { - countQuery = `SELECT COUNT(*) FROM transmissions WHERE last_seen >= ?` + countQuery = `SELECT COUNT(*) FROM transmissions WHERE last_seen >= ` + s.db.parameter(1) countArg = retentionFloor } else { - countQuery = `SELECT COUNT(*) FROM transmissions WHERE first_seen >= ?` + countQuery = `SELECT COUNT(*) FROM transmissions WHERE first_seen >= ` + s.db.parameter(1) countArg = retentionFloorStr } if err := s.db.conn.QueryRow(countQuery, countArg).Scan(&totalInDB); err != nil { @@ -1697,9 +1299,10 @@ func (s *PacketStore) loadBackgroundChunks() { for { s.mu.RLock() oldest := s.oldestLoaded + limited := s.startupMemoryLimited s.mu.RUnlock() - if oldest == "" { + if limited || oldest == "" { break } chunkEnd, err := time.Parse(time.RFC3339, oldest) @@ -1725,7 +1328,7 @@ func (s *PacketStore) loadBackgroundChunks() { // Always advance oldestLoaded to chunkStart so the loop terminates, // even when the chunk was empty (loadChunk skips the update when 0 packets). s.mu.Lock() - if s.oldestLoaded > chunkStartStr || s.oldestLoaded == "" { + if !s.startupMemoryLimited && (s.oldestLoaded > chunkStartStr || s.oldestLoaded == "") { s.oldestLoaded = chunkStartStr } s.mu.Unlock() @@ -1775,6 +1378,7 @@ func (s *PacketStore) loadBackgroundChunks() { // criterion. s.mu.RLock() loadedCount := int64(len(s.packets)) + limited := s.startupMemoryLimited s.mu.RUnlock() var ratio float64 @@ -1786,7 +1390,7 @@ func (s *PacketStore) loadBackgroundChunks() { } s.bgErrMu.Lock() s.loadCoverageRatio = ratio - if totalInDB > 0 && ratio < 0.90 { + if totalInDB > 0 && ratio < 0.90 && !limited { s.backgroundLoadFailed.Store(true) s.backgroundLoadErr = fmt.Sprintf("loaded %.1f%% of %d rows (%d in memory)", ratio*100, totalInDB, loadedCount) @@ -1799,7 +1403,8 @@ func (s *PacketStore) loadBackgroundChunks() { bgErr := s.backgroundLoadErr s.bgErrMu.Unlock() - // Only flip done=true when the store truly reflects the DB. Operators + // Reaching the configured memory limit is a successful bounded cache. + // Coverage still reports the actual ratio; older queries use SQL. Operators // (and the analytics warm-up gate, #1659) read this as "no more rows // to load" — making it lie was the cascading failure. if !s.backgroundLoadFailed.Load() { @@ -2057,12 +1662,13 @@ func (s *PacketStore) untrackAdvertPubkey(tx *StoreTx) { // QueryPackets returns filtered, paginated packets from memory. func (s *PacketStore) QueryPackets(q PacketQuery) *PacketResult { // SQL fallback: if the query window predates the in-memory window, delegate - // to the DB layer which covers the full SQLite retention period. + // to the DB layer which covers the full database retention period. s.mu.RLock() oldest := s.oldestLoaded + limited := s.startupMemoryLimited s.mu.RUnlock() if oldest != "" { - needsSQL := (q.Since != "" && q.Since < oldest) || + needsSQL := (limited && q.Since == "") || (q.Since != "" && q.Since < oldest) || (q.Until != "" && q.Until < oldest) if needsSQL { if result, err := s.db.QueryPackets(q); err == nil { @@ -2147,9 +1753,10 @@ func (s *PacketStore) queryPacketsLocked(q PacketQuery, f packetFilterSets) (*Pa func (s *PacketStore) QueryGroupedPackets(q PacketQuery) *PacketResult { s.mu.RLock() oldest := s.oldestLoaded + limited := s.startupMemoryLimited s.mu.RUnlock() if oldest != "" { - needsSQL := (q.Since != "" && q.Since < oldest) || + needsSQL := (limited && q.Since == "") || (q.Since != "" && q.Since < oldest) || (q.Until != "" && q.Until < oldest) if needsSQL { if result, err := s.db.QueryGroupedPackets(q); err == nil { @@ -2410,7 +2017,7 @@ func (s *PacketStore) GetStoreStats() (*Stats, error) { defer wg.Done() nodeErr = s.db.conn.QueryRow( `SELECT - (SELECT COUNT(*) FROM nodes WHERE last_seen > ?) AS active_nodes, + (SELECT COUNT(*) FROM nodes WHERE last_seen > `+s.db.parameter(1)+`) AS active_nodes, (SELECT COUNT(*) FROM nodes) AS all_nodes, (SELECT COUNT(*) FROM observers WHERE inactive IS NULL OR inactive = 0) AS observers`, sevenDaysAgo, @@ -2454,9 +2061,9 @@ func (s *PacketStore) refreshObsCounts(oneHourAgo, oneDayAgo int64) (int, int, e var lastHour, last24h int if qErr := s.db.conn.QueryRow( `SELECT - COALESCE(SUM(CASE WHEN timestamp > ? THEN 1 ELSE 0 END), 0), - COALESCE(SUM(CASE WHEN timestamp > ? THEN 1 ELSE 0 END), 0) - FROM observations WHERE timestamp > ?`, + COALESCE(SUM(CASE WHEN timestamp > `+s.db.parameter(1)+` THEN 1 ELSE 0 END), 0), + COALESCE(SUM(CASE WHEN timestamp > `+s.db.parameter(2)+` THEN 1 ELSE 0 END), 0) + FROM observations WHERE timestamp > `+s.db.parameter(3), oneHourAgo, oneDayAgo, oneDayAgo, ).Scan(&lastHour, &last24h); qErr != nil { return nil, qErr @@ -2962,7 +2569,7 @@ func (s *PacketStore) queryMultiNodePacketsLocked(resolved []string, limit, offs return &PacketResult{Packets: packets, Total: total}, jobs } -// IngestNewFromDB loads new transmissions from SQLite into memory and returns +// IngestNewFromDB loads new transmissions from PostgreSQL into memory and returns // broadcast-ready maps plus the new max transmission ID. func (s *PacketStore) IngestNewFromDB(sinceID, limit int) ([]map[string]interface{}, int) { if limit <= 0 { @@ -2983,14 +2590,17 @@ func (s *PacketStore) IngestNewFromDB(sinceID, limit int) ([]map[string]interfac scopeNameCol = ", t.scope_name" } if s.db.isV3 { - querySQL = `SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type, + querySQL = s.db.nativeSQL(`SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type, t.payload_type, t.payload_version, t.decoded_json, o.id, obs.id, obs.name, COALESCE(obs.iata, ''), o.direction, - o.snr, o.rssi, o.score, o.path_json, strftime('%Y-%m-%dT%H:%M:%fZ', o.timestamp, 'unixepoch')` + obsRHCol + scopeNameCol + ` + o.snr, o.rssi, o.score, o.path_json, strftime('%Y-%m-%dT%H:%M:%fZ', o.timestamp, 'unixepoch')`, `SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type, + t.payload_type, t.payload_version, t.decoded_json, + o.id, obs.id, obs.name, COALESCE(obs.iata, ''), o.direction, + o.snr, o.rssi, o.score, o.path_json, to_char(to_timestamp(o.timestamp) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')`) + obsRHCol + scopeNameCol + ` FROM transmissions t LEFT JOIN observations o ON o.transmission_id = t.id LEFT JOIN observers obs ON obs.rowid = o.observer_idx - WHERE t.id > ? + WHERE t.id > ` + s.db.parameter(1) + ` ORDER BY t.id ASC, o.timestamp DESC` } else { querySQL = `SELECT t.id, t.raw_hex, t.hash, t.first_seen, t.route_type, @@ -3000,7 +2610,7 @@ func (s *PacketStore) IngestNewFromDB(sinceID, limit int) ([]map[string]interfac FROM transmissions t LEFT JOIN observations o ON o.transmission_id = t.id LEFT JOIN observers obs ON obs.id = o.observer_id - WHERE t.id > ? + WHERE t.id > ` + s.db.parameter(1) + ` ORDER BY t.id ASC, o.timestamp DESC` } @@ -3021,7 +2631,7 @@ func (s *PacketStore) IngestNewFromDB(sinceID, limit int) ([]map[string]interfac obsRawHex string scopeName *string snr, rssi *float64 - score *int + score *float64 } var tempRows []tempRow @@ -3035,7 +2645,7 @@ func (s *PacketStore) IngestNewFromDB(sinceID, limit int) ([]map[string]interfac var obsIDVal sql.NullInt64 var observerID, observerName, observerIATA, direction, pathJSON, obsTimestamp sql.NullString var snrVal, rssiVal sql.NullFloat64 - var scoreVal sql.NullInt64 + var scoreVal sql.NullFloat64 var obsRawHex sql.NullString var scopeName sql.NullString @@ -3079,7 +2689,7 @@ func (s *PacketStore) IngestNewFromDB(sinceID, limit int) ([]map[string]interfac scopeName: nullStrPtr(scopeName), snr: nullFloatPtr(snrVal), rssi: nullFloatPtr(rssiVal), - score: nullIntPtr(scoreVal), + score: nullFloatPtr(scoreVal), } if obsIDVal.Valid { oid := int(obsIDVal.Int64) @@ -3387,21 +2997,22 @@ func (s *PacketStore) IngestNewObservations(sinceObsID, limit int) []map[string] obsRHCol2 = ", o.raw_hex" } if s.db.isV3 { - querySQL = `SELECT o.id, o.transmission_id, obs.id, obs.name, COALESCE(obs.iata, ''), o.direction, - o.snr, o.rssi, o.score, o.path_json, strftime('%Y-%m-%dT%H:%M:%fZ', o.timestamp, 'unixepoch')` + obsRHCol2 + ` + querySQL = s.db.nativeSQL(`SELECT o.id, o.transmission_id, obs.id, obs.name, COALESCE(obs.iata, ''), o.direction, + o.snr, o.rssi, o.score, o.path_json, strftime('%Y-%m-%dT%H:%M:%fZ', o.timestamp, 'unixepoch')`, `SELECT o.id, o.transmission_id, obs.id, obs.name, COALESCE(obs.iata, ''), o.direction, + o.snr, o.rssi, o.score, o.path_json, to_char(to_timestamp(o.timestamp) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')`) + obsRHCol2 + ` FROM observations o LEFT JOIN observers obs ON obs.rowid = o.observer_idx - WHERE o.id > ? + WHERE o.id > ` + s.db.parameter(1) + ` ORDER BY o.id ASC - LIMIT ?` + LIMIT ` + s.db.parameter(2) } else { querySQL = `SELECT o.id, o.transmission_id, o.observer_id, o.observer_name, COALESCE(obs.iata, ''), o.direction, o.snr, o.rssi, o.score, o.path_json, o.timestamp` + obsRHCol2 + ` FROM observations o LEFT JOIN observers obs ON obs.id = o.observer_id - WHERE o.id > ? + WHERE o.id > ` + s.db.parameter(1) + ` ORDER BY o.id ASC - LIMIT ?` + LIMIT ` + s.db.parameter(2) } rows, err := s.db.conn.Query(querySQL, sinceObsID, limit) @@ -3419,7 +3030,7 @@ func (s *PacketStore) IngestNewObservations(sinceObsID, limit int) []map[string] observerIATA string direction string snr, rssi *float64 - score *int + score *float64 pathJSON string rawHex string timestamp string @@ -3430,7 +3041,7 @@ func (s *PacketStore) IngestNewObservations(sinceObsID, limit int) []map[string] var oid, txID int var observerID, observerName, observerIATA, direction, pathJSON, ts sql.NullString var snr, rssi sql.NullFloat64 - var score sql.NullInt64 + var score sql.NullFloat64 var obsRawHex sql.NullString scanArgs3 := []interface{}{&oid, &txID, &observerID, &observerName, &observerIATA, &direction, @@ -3451,7 +3062,7 @@ func (s *PacketStore) IngestNewObservations(sinceObsID, limit int) []map[string] direction: internStr(nullStrVal(direction)), snr: nullFloatPtr(snr), rssi: nullFloatPtr(rssi), - score: nullIntPtr(score), + score: nullFloatPtr(score), pathJSON: nullStrVal(pathJSON), rawHex: nullStrVal(obsRawHex), timestamp: nullStrVal(ts), @@ -4247,7 +3858,7 @@ func enrichObsFields(obs *StoreObs, tx *StoreTx) map[string]interface{} { "direction": strOrNil(obs.Direction), "snr": floatPtrOrNil(obs.SNR), "rssi": floatPtrOrNil(obs.RSSI), - "score": intPtrOrNil(obs.Score), + "score": floatPtrOrNil(obs.Score), "path_json": strOrNil(obs.PathJSON), } @@ -5037,7 +4648,7 @@ func (s *PacketStore) buildDistanceIndex() { // // Note: ResolvedPath per-obs overhead eliminated by #800 refactor. const ( - storeTxBaseBytes = 384 // StoreTx struct fields + map headers + sync.Once + string headers + storeTxBaseBytes = 384 // StoreTx struct fields + map headers + cache guards + string headers storeObsBaseBytes = 192 // StoreObs struct fields + string headers strHdr = 16 // Go string header (ptr + len) on a 64-bit build; used by GetStoreMemoryBreakdown indexEntryBytes = 48 // average cost of one index map entry (key + pointer + bucket overhead) @@ -5052,6 +4663,10 @@ const ( // production heap profile. Analytics touch every tx, so it is charged // up front rather than when the cache fills. decodedCacheFactor = 4 + // One current txDecodedCache allocation (40 bytes, rounded to the 48-byte + // size class on 64-bit builds). Charge it before the first parse, like the + // decoded map. Retired generations live only while readers retain them. + decodedCacheGenerationBytes = 48 ) // estimateStoreTxBytes returns the estimated memory cost of a StoreTx (excluding observations). @@ -5061,6 +4676,7 @@ func estimateStoreTxBytes(tx *StoreTx) int64 { base += int64(len(tx.RawHex) + len(tx.Hash) + len(tx.DecodedJSON) + len(tx.PathJSON)) base += int64(numIndexesPerTx * indexEntryBytes) base += int64(decodedCacheFactor * len(tx.DecodedJSON)) + base += decodedCacheGenerationBytes // Path-dependent costs hops := int64(len(txGetParsedPath(tx))) @@ -5069,9 +4685,9 @@ func estimateStoreTxBytes(tx *StoreTx) int64 { return base } -// estimateStoreTxBytesTypical returns the estimated memory cost of a typical -// transmission with the given number of observations. Used for budget -// calculation during bounded cold load (no actual StoreTx needed). +// estimateStoreTxBytesTypical preserves the former admission policy's typical +// transmission estimate for the startup diagnostic. Actual admission uses each +// transmission's observation-aware charge, including the decoded cache generation. func estimateStoreTxBytesTypical(numObs int) int64 { // Typical tx: ~64 byte hash, ~200 byte decoded JSON, ~40 byte path, 3 hops base := int64(storeTxBaseBytes) + 64 + 200 + 40 @@ -5561,7 +5177,7 @@ func buildHopContextPubkeys(tx *StoreTx, pm *prefixMap) []string { } // Sender / originator pubkey from decoded payload. Use the cached - // ParsedDecoded() (sync.Once-gated) instead of re-unmarshaling — the + // ParsedDecoded() (once per source generation) instead of re-unmarshaling — the // helper is hot (3 distance sites + analytics topology, all 30k+ tx // loops). See #1197 (carmack/adversarial r1). if dec := tx.ParsedDecoded(); dec != nil { @@ -6372,7 +5988,6 @@ func (s *PacketStore) computeAnalyticsChannels(region, area string, window TimeW if region != "" { regionObs = s.resolveRegionObservers(region) } - s.mu.RLock() defer s.mu.RUnlock() @@ -9663,12 +9278,12 @@ func (s *PacketStore) loadMultibyteCapFromDB() { // RunMultibyteCapPersist consumes the file and writes confirmed / // suspected entries to the DB. // -// INVARIANT (#1289/#1324): the server is the read path and opens -// SQLite mode=ro. It MUST NOT execute any UPDATE on +// INVARIANT (#1289/#1324): the server is the read path and uses a +// restricted PostgreSQL reader. It MUST NOT execute any UPDATE on // nodes.multibyte_* — see readonly_invariant_test.go. This helper is // the only side-effect path for capability data leaving the server. func (s *PacketStore) publishMultibyteCapSnapshot(entries []MultiByteCapEntry) { - if s.db == nil || s.db.path == "" { + if s.db == nil || s.db.stateDir == "" { return } out := make([]mbcapqueue.Entry, 0, len(entries)) @@ -9679,7 +9294,7 @@ func (s *PacketStore) publishMultibyteCapSnapshot(entries []MultiByteCapEntry) { Evidence: e.Evidence, }) } - if err := mbcapqueue.WriteSnapshot(s.db.path, mbcapqueue.Snapshot{Entries: out}); err != nil { + if err := mbcapqueue.WriteSnapshot(s.db.statePath(), mbcapqueue.Snapshot{Entries: out}); err != nil { log.Printf("[multibyte] publish snapshot: %v", err) } } @@ -9882,7 +9497,7 @@ func (s *PacketStore) bulkHealthNodes(limit int, regionFilter bool, areaNodes ma if filtered { queryLimit = 10000 } - rows, err := s.db.conn.Query("SELECT public_key, name, role, lat, lon FROM nodes ORDER BY last_seen DESC LIMIT ?", queryLimit) + rows, err := s.db.conn.Query("SELECT public_key, name, role, lat, lon FROM nodes ORDER BY last_seen DESC NULLS LAST LIMIT "+s.db.parameter(1), queryLimit) if err != nil { return nil, err } @@ -10221,7 +9836,9 @@ func (s *PacketStore) coverageStart(now time.Time) (time.Time, string) { } if t, ok := parseStoreTime(s.oldestLoaded); ok { why := coverageDatabase - if s.hotStartupHours > 0 && !s.backgroundLoadDone.Load() { + if s.startupMemoryLimited { + why = coverageMemory + } else if s.hotStartupHours > 0 && !s.backgroundLoadDone.Load() { why = coverageLoading } consider(t, why) diff --git a/cmd/server/store_cold_lookup_lock_test.go b/cmd/server/store_cold_lookup_lock_test.go index fdd3b927f..66c154f98 100644 --- a/cmd/server/store_cold_lookup_lock_test.go +++ b/cmd/server/store_cold_lookup_lock_test.go @@ -24,7 +24,9 @@ func newColdLookupStore(t *testing.T) *PacketStore { s.config = &Config{Areas: map[string]AreaEntry{ "BAY": {Label: "Bay", Polygon: [][2]float64{{37.0, -122.5}, {38.0, -122.5}, {38.0, -121.5}, {37.0, -121.5}}}, }} - if err := s.Load(); err != nil { + // Exercise the real startup lifecycle: health readiness requires both + // initial and background loading to finish successfully. + if err := s.RunStartupLoad(0); err != nil { t.Fatal(err) } return s diff --git a/cmd/server/store_memory_diagnostics_test.go b/cmd/server/store_memory_diagnostics_test.go index 5bdf83185..f027a48e4 100644 --- a/cmd/server/store_memory_diagnostics_test.go +++ b/cmd/server/store_memory_diagnostics_test.go @@ -1,9 +1,7 @@ package main import ( - "database/sql" "math" - "path/filepath" "strings" "testing" "time" @@ -89,30 +87,36 @@ func TestStoreMemoryBreakdown_Empty(t *testing.T) { // falling back to the parent transmission. If this regresses, obs raw_hex would // be silently lost — so this is the safety gate for dropping obs.RawHex. func TestObsRawHexNotRetainedOnLoad(t *testing.T) { - dir := t.TempDir() - dbPath := filepath.Join(dir, "obsrawhex.db") + dbPath := postgresTestDSN(t) const txHex = "deadbeefcafe" const obsHex = "c0ffee0102" // distinct from txHex: proves we DON'T keep it - conn, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL") + conn, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } stmts := []string{ - `CREATE TABLE transmissions ( - id INTEGER PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, + testNativeSQL(`CREATE TABLE transmissions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT - )`, - `CREATE TABLE observations ( - id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, - observer_name TEXT, direction TEXT, snr REAL, rssi REAL, score INTEGER, - path_json TEXT, timestamp TEXT, raw_hex TEXT - )`, - `CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, + )`, `CREATE TABLE transmissions ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, + route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT + )`), + testNativeSQL(`CREATE TABLE observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, transmission_id INTEGER, observer_id TEXT, + observer_name TEXT, direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, + path_json TEXT, timestamp BIGINT, raw_hex TEXT + )`, `CREATE TABLE observations ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, + observer_name TEXT, direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, + path_json TEXT, timestamp BIGINT, raw_hex TEXT + )`), + testNativeSQL(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY AUTOINCREMENT, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, `CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`), `CREATE TABLE nodes ( - public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, - last_seen TEXT, first_seen TEXT, frequency REAL + public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, + last_seen TEXT, first_seen TEXT, frequency DOUBLE PRECISION )`, `CREATE INDEX idx_tx_first_seen ON transmissions(first_seen)`, } @@ -123,19 +127,19 @@ func TestObsRawHexNotRetainedOnLoad(t *testing.T) { } now := time.Now().UTC().Truncate(time.Second) if _, err := conn.Exec( - `INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + `INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`, 1, txHex, "hashobs", now.Add(-time.Minute).Format(time.RFC3339), 1, 5, 0, `{"type":"CHAN"}`); err != nil { t.Fatalf("insert tx: %v", err) } // Observation carries its OWN non-empty raw_hex in the DB. if _, err := conn.Exec( - `INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp, raw_hex) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + `INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp, raw_hex) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)`, 1, 1, "obs1", "Obs1", "rx", 5.0, -95.0, 0, `["AA"]`, now.Add(-time.Minute).Unix(), obsHex); err != nil { t.Fatalf("insert obs: %v", err) } conn.Close() - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatalf("OpenDB: %v", err) } diff --git a/cmd/server/store_obs_memory_test.go b/cmd/server/store_obs_memory_test.go new file mode 100644 index 000000000..effa63523 --- /dev/null +++ b/cmd/server/store_obs_memory_test.go @@ -0,0 +1,43 @@ +package main + +import ( + "runtime" + "testing" + "unsafe" +) + +// Observations outnumber transmissions at high-fanout receivers. Keep the +// timestamp cache packed so each retained observation fits the 192-byte +// allocation class on the supported 64-bit deployment targets. +func TestStoreObsMemoryFootprint(t *testing.T) { + if unsafe.Sizeof(uintptr(0)) != 8 { + t.Skip("allocation budget applies to 64-bit targets") + } + if size := unsafe.Sizeof(StoreObs{}); size > 192 { + t.Fatalf("StoreObs is %d bytes; want <=192 to avoid the 208-byte allocation class", size) + } +} + +// Measure live objects after GC; a heap profile taken after this benchmark +// returns would otherwise lose the observation set being compared. +func BenchmarkStoreObsRetainedMemory(b *testing.B) { + const count = 100000 + for i := 0; i < b.N; i++ { + b.StopTimer() + observations := make([]*StoreObs, count) + runtime.GC() + var before, after runtime.MemStats + runtime.ReadMemStats(&before) + b.StartTimer() + for j := range observations { + observations[j] = &StoreObs{ID: j + 1, TransmissionID: j/16 + 1, Timestamp: "2026-01-01T00:00:00Z"} + observations[j].ParsedTime() + } + b.StopTimer() + runtime.GC() + runtime.ReadMemStats(&after) + b.ReportMetric(float64(int64(after.HeapAlloc)-int64(before.HeapAlloc))/count, "retained-B/obs") + runtime.KeepAlive(observations) + b.StartTimer() + } +} diff --git a/cmd/server/subpaths_on_demand_test.go b/cmd/server/subpaths_on_demand_test.go index 7f7b1d32f..fd4acabe3 100644 --- a/cmd/server/subpaths_on_demand_test.go +++ b/cmd/server/subpaths_on_demand_test.go @@ -16,7 +16,7 @@ func setupSubpathCountDB(t *testing.T) *DB { seen := now.Add(-time.Hour).Format(time.RFC3339) epoch := now.Add(-time.Hour).Unix() db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obs1', 'Observer One', 'SJC', ?, '2025-01-01T00:00:00Z', 100)`, seen) + VALUES ('obs1', 'Observer One', 'SJC', $1, '2025-01-01T00:00:00Z', 100)`, seen) paths := []string{ `["AA","bb","aa","bb"]`, // mixed case; "aa,bb" twice in one path `["aa","bb"]`, @@ -25,9 +25,9 @@ func setupSubpathCountDB(t *testing.T) *DB { } for i, pj := range paths { db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('01', ?, ?, 1, 4, '{}')`, "subpath_count_"+string(rune('a'+i)), seen) + VALUES ('01', $1, $2, 1, 4, '{}')`, "subpath_count_"+string(rune('a'+i)), seen) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (?, 1, 10.0, -90, ?, ?)`, i+1, pj, epoch) + VALUES ($1, 1, 10.0, -90, $2, $3)`, i+1, pj, epoch) } return db } diff --git a/cmd/server/subpaths_window_test.go b/cmd/server/subpaths_window_test.go index 1d7c38c6c..6df6d30fa 100644 --- a/cmd/server/subpaths_window_test.go +++ b/cmd/server/subpaths_window_test.go @@ -36,19 +36,19 @@ func setupSubpathWindowDB(t *testing.T) *DB { // Observer db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obs1', 'Observer One', 'SJC', ?, '2025-01-01T00:00:00Z', 100)`, recent) + VALUES ('obs1', 'Observer One', 'SJC', $1, '2025-01-01T00:00:00Z', 100)`, recent) // Recent transmission with path ["aa","bb"] db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('01', 'recent_hash_window_001', ?, 1, 4, '{}')`, recent) + VALUES ('01', 'recent_hash_window_001', $1, 1, 4, '{}')`, recent) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 1, 10.0, -90, '["aa","bb"]', ?)`, recentEpoch) + VALUES (1, 1, 10.0, -90, '["aa","bb"]', $1)`, recentEpoch) // Old transmission (30d ago) with disjoint path ["cc","dd"] db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('02', 'old_hash_window_002', ?, 1, 4, '{}')`, old) + VALUES ('02', 'old_hash_window_002', $1, 1, 4, '{}')`, old) db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (2, 1, 10.0, -90, '["cc","dd"]', ?)`, oldEpoch) + VALUES (2, 1, 10.0, -90, '["cc","dd"]', $1)`, oldEpoch) return db } diff --git a/cmd/server/testdata/golden/shapes.json b/cmd/server/testdata/golden/shapes.json index 4556ddd55..b6d514126 100644 --- a/cmd/server/testdata/golden/shapes.json +++ b/cmd/server/testdata/golden/shapes.json @@ -1461,32 +1461,15 @@ } } }, - "sqlite": { + "postgres": { "type": "object", "keys": { - "dbSizeMB": { - "type": "number" - }, - "walSizeMB": { - "type": "number" + "engine": { + "type": "string" }, - "freelistMB": { + "dbSizeMB": { "type": "number" }, - "walPages": { - "type": "object", - "keys": { - "total": { - "type": "number" - }, - "checkpointed": { - "type": "number" - }, - "busy": { - "type": "number" - } - } - }, "rows": { "type": "object", "keys": { @@ -1503,6 +1486,15 @@ "type": "number" } } + }, + "sampledAt": { + "type": "string" + }, + "sampleIntervalSeconds": { + "type": "number" + }, + "stale": { + "type": "boolean" } } }, diff --git a/cmd/server/topology_dedup_test.go b/cmd/server/topology_dedup_test.go index b875eb016..17bfa3b9d 100644 --- a/cmd/server/topology_dedup_test.go +++ b/cmd/server/topology_dedup_test.go @@ -1,21 +1,16 @@ package main import ( - "database/sql" "fmt" - "path/filepath" "testing" "time" - - _ "github.com/mattn/go-sqlite3" ) // TestTopologyDedup_RepeatersMergeByPubkey verifies that topRepeaters // merges entries whose hop prefixes resolve unambiguously to the same node. func TestTopologyDedup_RepeatersMergeByPubkey(t *testing.T) { - dir := t.TempDir() - dbPath := filepath.Join(dir, "test.db") - conn, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL") + dbPath := postgresTestDSN(t) + conn, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } @@ -26,18 +21,24 @@ func TestTopologyDedup_RepeatersMergeByPubkey(t *testing.T) { t.Fatalf("SQL exec failed: %v\nSQL: %s", err, s) } } - exec(`CREATE TABLE transmissions ( - id INTEGER PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, + exec(testNativeSQL(`CREATE TABLE transmissions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT - )`) - exec(`CREATE TABLE observations ( - id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, - direction TEXT, snr REAL, rssi REAL, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT - )`) - exec(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) + )`, `CREATE TABLE transmissions ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, + route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT + )`)) + exec(testNativeSQL(`CREATE TABLE observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT + )`, `CREATE TABLE observations ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT + )`)) + exec(testNativeSQL(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY AUTOINCREMENT, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, `CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`)) exec(`CREATE TABLE nodes ( - public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, - last_seen TEXT, frequency REAL + public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, + last_seen TEXT, frequency DOUBLE PRECISION )`) exec(`CREATE TABLE schema_version (version INTEGER)`) exec(`INSERT INTO schema_version (version) VALUES (1)`) @@ -61,9 +62,9 @@ func TestTopologyDedup_RepeatersMergeByPubkey(t *testing.T) { for i := 0; i < count; i++ { ts := base.Add(time.Duration(txID) * time.Minute).Format(time.RFC3339) hash := fmt.Sprintf("h%04d", txID) - conn.Exec("INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) VALUES (?, ?, ?, ?, 0, 4, 1, ?)", + conn.Exec("INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) VALUES ($1, $2, $3, $4, 0, 4, 1, $5)", txID, "aabb", hash, ts, fmt.Sprintf(`{"pubKey":"pk%04d"}`, txID)) - conn.Exec("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + conn.Exec("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)", obsID, txID, "obs1", "Obs1", "RX", -10.0, -80.0, 5, path, ts) txID++ obsID++ @@ -78,7 +79,7 @@ func TestTopologyDedup_RepeatersMergeByPubkey(t *testing.T) { // Total: BETA appears as "99aa" (10×), "99" (5×), "99aabb" (3×) = 18 total // After dedup, each should appear ONCE with count=18. - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } @@ -137,9 +138,8 @@ func TestTopologyDedup_RepeatersMergeByPubkey(t *testing.T) { // TestTopologyDedup_AmbiguousPrefixNotMerged verifies that ambiguous short // prefixes (matching multiple nodes) are NOT merged — they stay separate. func TestTopologyDedup_AmbiguousPrefixNotMerged(t *testing.T) { - dir := t.TempDir() - dbPath := filepath.Join(dir, "test.db") - conn, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL") + dbPath := postgresTestDSN(t) + conn, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } @@ -150,18 +150,24 @@ func TestTopologyDedup_AmbiguousPrefixNotMerged(t *testing.T) { t.Fatalf("SQL exec failed: %v\nSQL: %s", err, s) } } - exec(`CREATE TABLE transmissions ( - id INTEGER PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, + exec(testNativeSQL(`CREATE TABLE transmissions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT - )`) - exec(`CREATE TABLE observations ( - id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, - direction TEXT, snr REAL, rssi REAL, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT - )`) - exec(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) + )`, `CREATE TABLE transmissions ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, + route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT + )`)) + exec(testNativeSQL(`CREATE TABLE observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT + )`, `CREATE TABLE observations ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT + )`)) + exec(testNativeSQL(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY AUTOINCREMENT, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, `CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`)) exec(`CREATE TABLE nodes ( - public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, - last_seen TEXT, frequency REAL + public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, + last_seen TEXT, frequency DOUBLE PRECISION )`) exec(`CREATE TABLE schema_version (version INTEGER)`) exec(`INSERT INTO schema_version (version) VALUES (1)`) @@ -179,9 +185,9 @@ func TestTopologyDedup_AmbiguousPrefixNotMerged(t *testing.T) { for i := 0; i < 10; i++ { ts := base.Add(time.Duration(txID) * time.Minute).Format(time.RFC3339) hash := fmt.Sprintf("h%04d", txID) - conn.Exec("INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) VALUES (?, ?, ?, ?, 0, 4, 1, ?)", + conn.Exec("INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) VALUES ($1, $2, $3, $4, 0, 4, 1, $5)", txID, "aabb", hash, ts, fmt.Sprintf(`{"pubKey":"pk%04d"}`, txID)) - conn.Exec("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + conn.Exec("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)", obsID, txID, "obs1", "Obs1", "RX", -10.0, -80.0, 5, `["ab"]`, ts) txID++ obsID++ @@ -190,15 +196,15 @@ func TestTopologyDedup_AmbiguousPrefixNotMerged(t *testing.T) { for i := 0; i < 5; i++ { ts := base.Add(time.Duration(txID) * time.Minute).Format(time.RFC3339) hash := fmt.Sprintf("h%04d", txID) - conn.Exec("INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) VALUES (?, ?, ?, ?, 0, 4, 1, ?)", + conn.Exec("INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) VALUES ($1, $2, $3, $4, 0, 4, 1, $5)", txID, "aabb", hash, ts, fmt.Sprintf(`{"pubKey":"pk%04d"}`, txID)) - conn.Exec("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + conn.Exec("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)", obsID, txID, "obs1", "Obs1", "RX", -10.0, -80.0, 5, `["ab1122"]`, ts) txID++ obsID++ } - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } @@ -243,9 +249,8 @@ func TestTopologyDedup_AmbiguousPrefixNotMerged(t *testing.T) { // TestTopologyDedup_PairsMergeByPubkey verifies that topPairs merges // pair entries whose hops resolve unambiguously to the same node pair. func TestTopologyDedup_PairsMergeByPubkey(t *testing.T) { - dir := t.TempDir() - dbPath := filepath.Join(dir, "test.db") - conn, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL") + dbPath := postgresTestDSN(t) + conn, err := openFixtureSQL(dbPath) if err != nil { t.Fatal(err) } @@ -256,18 +261,24 @@ func TestTopologyDedup_PairsMergeByPubkey(t *testing.T) { t.Fatalf("SQL exec failed: %v\nSQL: %s", err, s) } } - exec(`CREATE TABLE transmissions ( - id INTEGER PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, + exec(testNativeSQL(`CREATE TABLE transmissions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT - )`) - exec(`CREATE TABLE observations ( - id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, - direction TEXT, snr REAL, rssi REAL, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT - )`) - exec(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) + )`, `CREATE TABLE transmissions ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, + route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT + )`)) + exec(testNativeSQL(`CREATE TABLE observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT + )`, `CREATE TABLE observations ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, + direction TEXT, snr DOUBLE PRECISION, rssi DOUBLE PRECISION, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT + )`)) + exec(testNativeSQL(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY AUTOINCREMENT, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, `CREATE TABLE observers (rowid BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`)) exec(`CREATE TABLE nodes ( - public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, - last_seen TEXT, frequency REAL + public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat DOUBLE PRECISION, lon DOUBLE PRECISION, + last_seen TEXT, frequency DOUBLE PRECISION )`) exec(`CREATE TABLE schema_version (version INTEGER)`) exec(`INSERT INTO schema_version (version) VALUES (1)`) @@ -283,9 +294,9 @@ func TestTopologyDedup_PairsMergeByPubkey(t *testing.T) { for i := 0; i < count; i++ { ts := base.Add(time.Duration(txID) * time.Minute).Format(time.RFC3339) hash := fmt.Sprintf("h%04d", txID) - conn.Exec("INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) VALUES (?, ?, ?, ?, 0, 4, 1, ?)", + conn.Exec("INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) VALUES ($1, $2, $3, $4, 0, 4, 1, $5)", txID, "aabb", hash, ts, fmt.Sprintf(`{"pubKey":"pk%04d"}`, txID)) - conn.Exec("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + conn.Exec("INSERT INTO observations (id, transmission_id, observer_id, observer_name, direction, snr, rssi, score, path_json, timestamp) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)", obsID, txID, "obs1", "Obs1", "RX", -10.0, -80.0, 5, path, ts) txID++ obsID++ @@ -299,7 +310,7 @@ func TestTopologyDedup_PairsMergeByPubkey(t *testing.T) { insertTx(`["07","99aa"]`, 10) insertTx(`["0735bc","99"]`, 5) - db, err := OpenDB(dbPath) + db, err := openFixtureReader(t, dbPath) if err != nil { t.Fatal(err) } diff --git a/cmd/server/traffic_share_score_test.go b/cmd/server/traffic_share_score_test.go index 7610336de..f936faa09 100644 --- a/cmd/server/traffic_share_score_test.go +++ b/cmd/server/traffic_share_score_test.go @@ -34,7 +34,7 @@ func TestTrafficShareScore_HandleNodesSurface(t *testing.T) { for _, p := range []string{pk, left, right} { if _, err := db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, 'rpt', 'repeater', 37.5, -122.0, ?, ?, 10)`, + VALUES ($1, 'rpt', 'repeater', 37.5, -122.0, $2, $3, 10)`, p, recent, recent); err != nil { t.Fatal(err) } @@ -126,7 +126,7 @@ func TestTrafficShareScore_NodeDetail(t *testing.T) { recent := time.Now().UTC().Format("2006-01-02T15:04:05.000Z") if _, err := db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, 'rpt', 'repeater', 37.5, -122.0, ?, ?, 10)`, + VALUES ($1, 'rpt', 'repeater', 37.5, -122.0, $2, $3, 10)`, pk, recent, recent); err != nil { t.Fatal(err) } diff --git a/cmd/server/tx_rewrite_feed.go b/cmd/server/tx_rewrite_feed.go index 1ac8bca12..72f6adb39 100644 --- a/cmd/server/tx_rewrite_feed.go +++ b/cmd/server/tx_rewrite_feed.go @@ -3,7 +3,6 @@ package main import ( "database/sql" "log" - "sync" ) // tx_rewrite_feed (#2107) is the ingestor's change feed for transmissions it @@ -27,7 +26,7 @@ func (db *DB) txRewriteFeedPresent() bool { return true } var exists int - if err := db.conn.QueryRow(`SELECT 1 FROM sqlite_master WHERE type='table' AND name='tx_rewrite_feed'`).Scan(&exists); err != nil { + if err := db.conn.QueryRow(db.nativeSQL(`SELECT 1 FROM sqlite_master WHERE type='table' AND name='tx_rewrite_feed'`, `SELECT 1 FROM information_schema.tables WHERE table_schema=current_schema() AND table_name='tx_rewrite_feed'`)).Scan(&exists); err != nil { return false } db.txRewriteFeedTable.Store(true) @@ -54,7 +53,7 @@ func (s *PacketStore) pollTxRewrites(limit int) error { rows, err := s.db.conn.Query(` SELECT f.id, f.tx_id, t.decoded_json, `+scopeCol+` FROM tx_rewrite_feed f LEFT JOIN transmissions t ON t.id = f.tx_id - WHERE f.id > ? ORDER BY f.id LIMIT ?`, s.txRewriteCursor, limit) + WHERE f.id > `+s.db.parameter(1)+` ORDER BY f.id LIMIT `+s.db.parameter(2), s.txRewriteCursor, limit) if err != nil { return err } @@ -87,12 +86,7 @@ func (s *PacketStore) pollTxRewrites(limit int) error { if tx == nil { continue } - if r.decoded.Valid && r.decoded.String != tx.DecodedJSON { - tx.DecodedJSON = r.decoded.String - // ParsedDecoded caches behind a sync.Once; reset both under the - // write lock so the next reader parses the new JSON. - tx.decodedOnce = sync.Once{} - tx.parsedDecoded = nil + if r.decoded.Valid && tx.setDecodedJSON(r.decoded.String) { s.trackedBytes += rechargeTx(tx) if tx.PayloadType != nil && *tx.PayloadType == PayloadGRP_TXT { channelChanged = true diff --git a/cmd/server/tx_rewrite_feed_test.go b/cmd/server/tx_rewrite_feed_test.go index 2b07a38cc..31394b3c1 100644 --- a/cmd/server/tx_rewrite_feed_test.go +++ b/cmd/server/tx_rewrite_feed_test.go @@ -12,7 +12,7 @@ import ( func txRewriteFixture(t *testing.T, withFeed bool) (*DB, *sql.DB) { t.Helper() path := createTestDBWithResolvedPath(t, 1, []string{"fixture-relay"}) - w, err := sql.Open("sqlite3", path+"?_journal_mode=WAL") + w, err := openFixtureSQL(path) if err != nil { t.Fatal(err) } @@ -22,7 +22,7 @@ func txRewriteFixture(t *testing.T, withFeed bool) (*DB, *sql.DB) { `ALTER TABLE transmissions ADD COLUMN channel_hash TEXT`, } if withFeed { - stmts = append(stmts, txRewriteFeedDDL) + stmts = append(stmts, testNativeSQL(txRewriteFeedDDL, postgresTxRewriteFeedDDL)) } for _, stmt := range stmts { if _, err := w.Exec(stmt); err != nil { @@ -30,11 +30,15 @@ func txRewriteFixture(t *testing.T, withFeed bool) (*DB, *sql.DB) { } } now := time.Now().UTC().Add(-time.Minute).Format(time.RFC3339) - if _, err := w.Exec(`UPDATE transmissions SET payload_type=5, route_type=0, first_seen=?, scope_name='', - channel_hash='enc_AB', decoded_json='{"type":"GRP_TXT","channelHashHex":"AB"}'; UPDATE observations SET timestamp=?`, now, now); err != nil { + if _, err := w.Exec(`UPDATE transmissions SET payload_type=5, route_type=0, first_seen=$1, scope_name='', + channel_hash='enc_AB', decoded_json='{"type":"GRP_TXT","channelHashHex":"AB"}'`, now); err != nil { t.Fatal(err) } - db, err := OpenDB(path) + if _, err := w.Exec(`UPDATE observations SET timestamp=$1`, now); err != nil { + t.Fatal(err) + } + + db, err := openFixtureReader(t, path) if err != nil { t.Fatal(err) } @@ -44,13 +48,24 @@ func txRewriteFixture(t *testing.T, withFeed bool) (*DB, *sql.DB) { // Mirrors internal/dbschema/key_backfill.go; the server never creates it. const txRewriteFeedDDL = `CREATE TABLE tx_rewrite_feed (id INTEGER PRIMARY KEY AUTOINCREMENT, tx_id INTEGER NOT NULL, created_at INTEGER NOT NULL DEFAULT 0)` +const postgresTxRewriteFeedDDL = `CREATE TABLE tx_rewrite_feed (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, tx_id BIGINT NOT NULL, created_at BIGINT NOT NULL DEFAULT 0)` // backfillRewrite does what the ingestor's commit does: rewrite the row and // append its feed entry in one transaction. func backfillRewrite(t *testing.T, w *sql.DB, decoded, scope string) { t.Helper() - if _, err := w.Exec(`BEGIN; UPDATE transmissions SET decoded_json=?, channel_hash='#saar', scope_name=? WHERE id=1; - INSERT INTO tx_rewrite_feed(tx_id) VALUES(1); COMMIT`, decoded, scope); err != nil { + tx, err := w.Begin() + if err != nil { + t.Fatal(err) + } + defer tx.Rollback() + if _, err := tx.Exec(`UPDATE transmissions SET decoded_json=$1, channel_hash='#saar', scope_name=$2 WHERE id=1`, decoded, scope); err != nil { + t.Fatal(err) + } + if _, err := tx.Exec(`INSERT INTO tx_rewrite_feed(tx_id) VALUES(1)`); err != nil { + t.Fatal(err) + } + if err := tx.Commit(); err != nil { t.Fatal(err) } } @@ -106,7 +121,7 @@ func TestTxRewriteFeedAbsentThenCreated(t *testing.T) { if err := s.pollTxRewrites(500); err != nil { t.Fatalf("absent feed must be a no-op, got %v", err) } - if _, err := w.Exec(txRewriteFeedDDL); err != nil { + if _, err := w.Exec(testNativeSQL(txRewriteFeedDDL, postgresTxRewriteFeedDDL)); err != nil { t.Fatal(err) } backfillRewrite(t, w, `{"type":"CHAN","channel":"#saar"}`, "#saar") diff --git a/cmd/server/types.go b/cmd/server/types.go index d5fe34aeb..f5aef0475 100644 --- a/cmd/server/types.go +++ b/cmd/server/types.go @@ -261,25 +261,23 @@ type PerfPacketStoreStats struct { LoadCoverageRatio float64 `json:"loadCoverageRatio"` } -type WalPages struct { - Total int `json:"total"` - Checkpointed int `json:"checkpointed"` - Busy int `json:"busy"` -} - -type SqliteRowCounts struct { +type DatabaseRowCounts struct { Transmissions int `json:"transmissions"` Observations int `json:"observations"` Nodes int `json:"nodes"` Observers int `json:"observers"` } -type SqliteStats struct { - DbSizeMB float64 `json:"dbSizeMB"` - WalSizeMB float64 `json:"walSizeMB"` - FreelistMB float64 `json:"freelistMB"` - WalPages *WalPages `json:"walPages"` - Rows *SqliteRowCounts `json:"rows"` +type PostgresStats struct { + WalSizeMB *float64 `json:"walSizeMB,omitempty"` + FreelistMB *float64 `json:"freelistMB,omitempty"` + Engine string `json:"engine"` + DbSizeMB *float64 `json:"dbSizeMB"` + Rows *DatabaseRowCounts `json:"rows"` + SampledAt string `json:"sampledAt,omitempty"` + SampleIntervalSeconds int `json:"sampleIntervalSeconds"` + Stale bool `json:"stale"` + Error string `json:"error,omitempty"` } type PerfResponse struct { @@ -290,7 +288,9 @@ type PerfResponse struct { SlowQueries []SlowQuery `json:"slowQueries"` Cache PerfCacheStats `json:"cache"` PacketStore *PerfPacketStoreStats `json:"packetStore"` - Sqlite *SqliteStats `json:"sqlite"` + Postgres *PostgresStats `json:"postgres"` + Sqlite *PostgresStats `json:"sqlite,omitempty"` + Database *PostgresStats `json:"database,omitempty"` GoRuntime *GoRuntimeStats `json:"goRuntime,omitempty"` // MemoryBreakdown is populated only for /api/perf?mem=1 (an O(tx+obs) // walk, opt-in so the normal hot endpoint stays cheap). It sizes the diff --git a/cmd/server/usefulness_axes_handle_nodes_test.go b/cmd/server/usefulness_axes_handle_nodes_test.go index 9628525cb..fb5f15e47 100644 --- a/cmd/server/usefulness_axes_handle_nodes_test.go +++ b/cmd/server/usefulness_axes_handle_nodes_test.go @@ -31,7 +31,7 @@ func TestUsefulnessAxes_HandleNodesSurface(t *testing.T) { for _, pk := range pks { if _, err := db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, ?, 'repeater', 37.5, -122.0, ?, ?, 10)`, + VALUES ($1, $2, 'repeater', 37.5, -122.0, $3, $4, 10)`, pk, "node-"+pk[:4], recent, recent); err != nil { t.Fatal(err) } @@ -43,7 +43,7 @@ func TestUsefulnessAxes_HandleNodesSurface(t *testing.T) { clientPK := "eeee000000000000000000000000000000000000000000000000000000000000" if _, err := db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) - VALUES (?, 'client-eeee', 'client', 37.5, -122.0, ?, ?, 10)`, + VALUES ($1, 'client-eeee', 'client', 37.5, -122.0, $2, $3, 10)`, clientPK, recent, recent); err != nil { t.Fatal(err) } diff --git a/cmd/server/user_mgmt_config.go b/cmd/server/user_mgmt_config.go index aba0a0fe6..2407df137 100644 --- a/cmd/server/user_mgmt_config.go +++ b/cmd/server/user_mgmt_config.go @@ -16,6 +16,7 @@ import ( // UserManagementConfig is the "userManagement" block of config.json. type UserManagementConfig struct { Enabled bool `json:"enabled"` + DatabaseURL string `json:"databaseURL,omitempty"` DBPath string `json:"dbPath,omitempty"` AdminEmails []string `json:"adminEmails,omitempty"` PublicBaseURL string `json:"publicBaseUrl,omitempty"` @@ -180,19 +181,19 @@ type UsersBackupConfig struct { // backupSettings is the resolved form; the zero value means off. type backupSettings struct { enabled bool - dir string // relative paths resolve against the working directory, like dbPath + dir string // relative paths resolve against the working directory keep int // snapshots kept after each run } const defaultBackupKeep = 7 -// resolveBackup fills the defaults: dir "backups" next to users.db, keep 7 +// resolveBackup defaults to "backups" under the state directory, keep 7 // for an absent, zero or negative value. Only "enabled": false turns it off. -func resolveBackup(c *UsersBackupConfig, dbPath string) backupSettings { +func resolveBackup(c *UsersBackupConfig, stateDir string) backupSettings { if c != nil && c.Enabled != nil && !*c.Enabled { return backupSettings{} } - b := backupSettings{enabled: true, dir: filepath.Join(filepath.Dir(dbPath), "backups"), keep: defaultBackupKeep} + b := backupSettings{enabled: true, dir: filepath.Join(stateDir, "backups"), keep: defaultBackupKeep} if c == nil { return b } @@ -211,7 +212,7 @@ func (c *Config) UserManagementEnabled() bool { // userMgmtSettings is the validated, resolved form the auth service runs on. type userMgmtSettings struct { - dbPath string + databaseURL string adminEmails map[string]bool baseURL *url.URL // no trailing slash, no query or fragment secureCookie bool @@ -237,13 +238,17 @@ var fakeMailerAllowed bool // resolveUserManagement validates the block and fills defaults. It refuses // configurations where nobody could activate an account, so the server // fails at startup instead of running half-working. -func resolveUserManagement(u *UserManagementConfig, measurementDBPath string, getenv func(string) string) (*userMgmtSettings, error) { +func resolveUserManagement(u *UserManagementConfig, stateDir string, getenv func(string) string) (*userMgmtSettings, error) { set := &userMgmtSettings{adminEmails: map[string]bool{}} - set.dbPath = strings.TrimSpace(u.DBPath) - if set.dbPath == "" { - set.dbPath = filepath.Join(filepath.Dir(measurementDBPath), "users.db") + set.databaseURL = envOrValue(getenv, "CORESCOPE_USERS_DATABASE_URL", u.DatabaseURL) + if set.databaseURL == "" { + set.databaseURL = strings.TrimSpace(u.DBPath) } + if set.databaseURL == "" { + set.databaseURL = filepath.Join(stateDir, "users.db") + } + for _, raw := range u.AdminEmails { e, err := users.NormalizeEmail(raw) if err != nil { @@ -326,7 +331,7 @@ func resolveUserManagement(u *UserManagementConfig, measurementDBPath string, ge return nil, err } } - set.backup = resolveBackup(u.Backup, set.dbPath) + set.backup = resolveBackup(u.Backup, stateDir) return set, nil } diff --git a/cmd/server/user_mgmt_config_test.go b/cmd/server/user_mgmt_config_test.go index 7c00fef72..f9d36183a 100644 --- a/cmd/server/user_mgmt_config_test.go +++ b/cmd/server/user_mgmt_config_test.go @@ -10,6 +10,7 @@ import ( func validUM() *UserManagementConfig { return &UserManagementConfig{ Enabled: true, + DatabaseURL: "postgresql://accounts@localhost/users", AdminEmails: []string{" Boss@Example.org "}, PublicBaseURL: "https://scope.example.org/", Mail: UserMailConfig{BrevoAPIKey: "xkeysib-abc", FromEmail: "noreply@example.org"}, @@ -19,12 +20,12 @@ func validUM() *UserManagementConfig { func noEnv(string) string { return "" } func TestResolveUserManagementDefaults(t *testing.T) { - set, err := resolveUserManagement(validUM(), filepath.Join("data", "meshcore.db"), noEnv) + set, err := resolveUserManagement(validUM(), "data", noEnv) if err != nil { t.Fatal(err) } - if set.dbPath != filepath.Join("data", "users.db") { - t.Errorf("dbPath = %q", set.dbPath) + if set.databaseURL != validUM().DatabaseURL { + t.Errorf("dbPath = %q", set.databaseURL) } if !set.adminEmails["boss@example.org"] { t.Errorf("adminEmails not normalized: %v", set.adminEmails) @@ -38,11 +39,14 @@ func TestResolveUserManagementDefaults(t *testing.T) { } func TestResolveUserManagementEnvWins(t *testing.T) { - env := map[string]string{"CORESCOPE_BREVO_API_KEY": "from-env", "CORESCOPE_BREVO_WEBHOOK_SECRET": "env-secret-0123456789"} + env := map[string]string{"CORESCOPE_USERS_DATABASE_URL": "postgresql://accounts@localhost/from_env", "CORESCOPE_BREVO_API_KEY": "from-env", "CORESCOPE_BREVO_WEBHOOK_SECRET": "env-secret-0123456789"} set, err := resolveUserManagement(validUM(), "meshcore.db", func(k string) string { return env[k] }) if err != nil { t.Fatal(err) } + if set.databaseURL != env["CORESCOPE_USERS_DATABASE_URL"] { + t.Fatal("account database environment override not applied") + } if set.brevoAPIKey != "from-env" || set.webhookSecret != "env-secret-0123456789" { t.Fatalf("env not applied: %q %q", set.brevoAPIKey, set.webhookSecret) } @@ -199,7 +203,7 @@ func TestResolveNotifications(t *testing.T) { func TestResolveUsersBackup(t *testing.T) { u := validUM() - meas := filepath.Join("data", "meshcore.db") + meas := "data" defaults := backupSettings{enabled: true, dir: filepath.Join("data", "backups"), keep: 7} set, err := resolveUserManagement(u, meas, noEnv) if err != nil { @@ -208,9 +212,10 @@ func TestResolveUsersBackup(t *testing.T) { if set.backup != defaults { t.Fatalf("absent block = %+v; want %+v", set.backup, defaults) } - u.DBPath = filepath.Join("accounts", "users.db") - if set, _ = resolveUserManagement(u, meas, noEnv); set.backup.dir != filepath.Join("accounts", "backups") { - t.Fatalf("dir follows dbPath: %q", set.backup.dir) + u.DBPath = "private-users.db" + u.DatabaseURL = "" + if set, err := resolveUserManagement(u, meas, noEnv); err != nil || set.databaseURL != u.DBPath { + t.Fatalf("legacy account path lost: %v", err) } u.DBPath = "" off, on := false, true diff --git a/cmd/server/user_mgmt_off_test.go b/cmd/server/user_mgmt_off_test.go index 7fa23c437..5f5395f5b 100644 --- a/cmd/server/user_mgmt_off_test.go +++ b/cmd/server/user_mgmt_off_test.go @@ -73,30 +73,30 @@ func TestAuthRoutesThroughRealRouterAreNoStore(t *testing.T) { } func TestInitUserManagementRefusesMeasurementDB(t *testing.T) { - dir := t.TempDir() - measurement := filepath.Join(dir, "meshcore.db") - os.WriteFile(measurement, nil, 0o644) - srv := &Server{cfg: &Config{UserManagement: &UserManagementConfig{ - Enabled: true, DBPath: measurement, PublicBaseURL: testBase, - Mail: UserMailConfig{BrevoAPIKey: "k", FromEmail: "noreply@example.org"}, - }}} - err := srv.initUserManagement(measurement) - if err == nil || !strings.Contains(err.Error(), "measurement database") { + a, _ := newTestAuthService(t) + u := validUM() + u.DatabaseURL = a.set.databaseURL + srv := &Server{cfg: &Config{UserManagement: u}} + if err := srv.initUserManagement(u.DatabaseURL); err == nil || !strings.Contains(err.Error(), "measurement database") { t.Fatalf("err = %v", err) } } -func TestInitUserManagementCreatesUsersDB(t *testing.T) { - dir := t.TempDir() - srv := &Server{cfg: &Config{UserManagement: &UserManagementConfig{ - Enabled: true, PublicBaseURL: testBase, - Mail: UserMailConfig{BrevoAPIKey: "k", FromEmail: "noreply@example.org"}, - }}} - if err := srv.initUserManagement(filepath.Join(dir, "meshcore.db")); err != nil { +func TestInitUserManagementRequiresBootstrappedDatabase(t *testing.T) { + postgresOnly(t) + u := validUM() + u.DatabaseURL = testDatabaseDSN(t) + srv := &Server{cfg: &Config{UserManagement: u}} + if err := srv.initUserManagement(testDatabaseDSN(t)); err == nil { + t.Fatal("runtime created missing account schema") + } + db, err := openFixtureSQL(u.DatabaseURL) + if err != nil { t.Fatal(err) } - defer srv.closeUserManagement() - if _, err := os.Stat(filepath.Join(dir, "users.db")); err != nil { - t.Fatalf("users.db not created next to the measurement DB: %v", err) + defer db.Close() + var tables int + if err := db.QueryRow(testNativeSQL(`SELECT COUNT(*) FROM sqlite_master WHERE type='table'`, `SELECT COUNT(*) FROM information_schema.tables WHERE table_schema='public'`)).Scan(&tables); err != nil || tables != 0 { + t.Fatalf("runtime created tables: %d, %v", tables, err) } } diff --git a/cmd/server/users_backup.go b/cmd/server/users_backup.go index bebe788da..182577554 100644 --- a/cmd/server/users_backup.go +++ b/cmd/server/users_backup.go @@ -14,6 +14,7 @@ import ( "strings" "time" + "github.com/meshcore-analyzer/dbconfig" "github.com/meshcore-analyzer/users" ) @@ -26,17 +27,23 @@ const ( // usersBackupName matches the snapshot files this code writes and rotates. // Apart from orphaned temporary files (usersBackupTemp), nothing else in // the directory is ever touched. -var usersBackupName = regexp.MustCompile(`^users-\d{8}-\d{6}\.db$`) +var usersBackupName = regexp.MustCompile(`^users-\d{8}-\d{6}\.(?:db|dump)$`) // usersBackupTemp matches the temporary name a snapshot is written under. // One is only left behind when the process died mid-write. -var usersBackupTemp = regexp.MustCompile(`^users-\d{8}-\d{6}\.db\.tmp$`) +var usersBackupTemp = regexp.MustCompile(`^users-\d{8}-\d{6}\.(?:db|dump)\.tmp$`) // usersBackupTempMaxAge is how old an orphaned temporary snapshot must be // before the sweep removes it; no snapshot write takes this long. const usersBackupTempMaxAge = 24 * time.Hour -func usersBackupFile(t time.Time) string { return "users-" + t.UTC().Format(usersBackupLayout) + ".db" } +func usersBackupFile(t time.Time, backend ...dbconfig.Backend) string { + extension := ".dump" + if len(backend) > 0 && backend[0] == dbconfig.SQLite { + extension = ".db" + } + return "users-" + t.UTC().Format(usersBackupLayout) + extension +} // listUsersBackups returns the snapshot file names in dir, oldest first // (the UTC timestamp sorts by name). A missing dir has none. @@ -64,7 +71,7 @@ func listUsersBackups(dir string) ([]string, error) { // back) are skipped so they cannot stop the backups. func usersBackupDue(names []string, now time.Time) bool { for i := len(names) - 1; i >= 0; i-- { - t, err := time.Parse(usersBackupLayout, strings.TrimSuffix(strings.TrimPrefix(names[i], "users-"), ".db")) + t, err := time.Parse(usersBackupLayout, strings.TrimSuffix(strings.TrimSuffix(strings.TrimPrefix(names[i], "users-"), ".dump"), ".db")) if err != nil || t.After(now) { continue } @@ -79,7 +86,7 @@ func writeUsersBackup(st *users.Store, dir string, now time.Time) (string, int64 if err := os.MkdirAll(dir, 0o700); err != nil { return "", 0, err } - path := filepath.Join(dir, usersBackupFile(now)) + path := filepath.Join(dir, usersBackupFile(now, st.Backend())) tmp := path + ".tmp" if err := st.Snapshot(tmp); err != nil { return "", 0, err @@ -187,7 +194,7 @@ func (a *authService) maybeBackup(now time.Time) { // handleAdminUsersBackup streams a fresh users.db snapshot from a temp // directory, removed afterwards. The file holds password hashes and // addresses: admin only, audited as user.backup. -func (s *Server) handleAdminUsersBackup(w http.ResponseWriter, _ *http.Request, admin *users.User, _ *users.Session) { +func (s *Server) handleAdminUsersBackup(w http.ResponseWriter, r *http.Request, admin *users.User, _ *users.Session) { a := s.auth tmpDir, err := os.MkdirTemp("", "corescope-users-") if err != nil { @@ -200,9 +207,9 @@ func (s *Server) handleAdminUsersBackup(w http.ResponseWriter, _ *http.Request, log.Printf("[users] backup download cleanup: %v", err) } }() - name := "corescope-users-" + time.Now().UTC().Format(usersBackupLayout) + ".db" + name := "corescope-" + usersBackupFile(time.Now(), a.st.Backend()) path := filepath.Join(tmpDir, name) - if err := a.st.Snapshot(path); err != nil { + if err := a.st.SnapshotContext(r.Context(), path); err != nil { log.Printf("[users] backup download: %v", err) writeError(w, http.StatusInternalServerError, "internal error") return diff --git a/cmd/server/users_backup_test.go b/cmd/server/users_backup_test.go index c57117d25..55b17a0f4 100644 --- a/cmd/server/users_backup_test.go +++ b/cmd/server/users_backup_test.go @@ -2,7 +2,6 @@ package main import ( "bytes" - "database/sql" "os" "path/filepath" "reflect" @@ -21,7 +20,7 @@ var backupNow = time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC) // created directory. func backupService(t *testing.T, keep int) (*authService, string) { t.Helper() - a, _ := newTestAuthService(t) + a, _ := newTestBackupAuthService(t) dir := filepath.Join(t.TempDir(), "backups") a.set.backup = backupSettings{enabled: true, dir: dir, keep: keep} return a, dir @@ -53,13 +52,10 @@ func TestUsersBackupWrittenWhenNoneExists(t *testing.T) { } a.maybeBackup(backupNow) names := snapshotNames(t, dir) - if len(names) != 1 || names[0] != usersBackupFile(backupNow) || names[0] != "users-20261008-120000.db" { + if len(names) != 1 || names[0] != testUsersBackupFile(backupNow) || names[0] != testNativeSQL("users-20261008-120000.db", "users-20261008-120000.dump") { t.Fatalf("snapshots = %v", names) } - db, err := sql.Open("sqlite", filepath.Join(dir, names[0])) - if err != nil { - t.Fatal(err) - } + db := restorePostgresTestBackup(t, filepath.Join(dir, names[0])) defer db.Close() var n int if err := db.QueryRow(`SELECT COUNT(*) FROM users WHERE email = 'a@example.org'`).Scan(&n); err != nil || n != 1 { @@ -76,7 +72,7 @@ func TestUsersBackupFileMode(t *testing.T) { } a, dir := backupService(t, 7) a.maybeBackup(backupNow) - fi, err := os.Stat(filepath.Join(dir, usersBackupFile(backupNow))) + fi, err := os.Stat(filepath.Join(dir, testUsersBackupFile(backupNow))) if err != nil { t.Fatal(err) } @@ -94,7 +90,7 @@ func TestUsersBackupFileMode(t *testing.T) { func TestUsersBackupSkippedWhenFresh(t *testing.T) { a, dir := backupService(t, 7) - fresh := usersBackupFile(backupNow.Add(-time.Hour)) + fresh := testUsersBackupFile(backupNow.Add(-time.Hour)) writeBackupFile(t, dir, fresh) a.maybeBackup(backupNow) a.maybeBackup(backupNow.Add(22 * time.Hour)) @@ -105,20 +101,20 @@ func TestUsersBackupSkippedWhenFresh(t *testing.T) { func TestUsersBackupWrittenWhenStale(t *testing.T) { a, dir := backupService(t, 7) - stale := usersBackupFile(backupNow.Add(-25 * time.Hour)) + stale := testUsersBackupFile(backupNow.Add(-25 * time.Hour)) writeBackupFile(t, dir, stale) a.maybeBackup(backupNow) - if names := snapshotNames(t, dir); !reflect.DeepEqual(names, []string{stale, usersBackupFile(backupNow)}) { + if names := snapshotNames(t, dir); !reflect.DeepEqual(names, []string{stale, testUsersBackupFile(backupNow)}) { t.Fatalf("snapshots = %v", names) } } func TestUsersBackupFutureSnapshotDoesNotBlock(t *testing.T) { a, dir := backupService(t, 7) - future := usersBackupFile(backupNow.Add(48 * time.Hour)) + future := testUsersBackupFile(backupNow.Add(48 * time.Hour)) writeBackupFile(t, dir, future) a.maybeBackup(backupNow) - want := []string{usersBackupFile(backupNow), future} + want := []string{testUsersBackupFile(backupNow), future} if names := snapshotNames(t, dir); !reflect.DeepEqual(names, want) { t.Fatalf("snapshots = %v; want %v", names, want) } @@ -131,10 +127,10 @@ func TestUsersBackupFutureSnapshotDoesNotBlock(t *testing.T) { func TestUsersBackupRotationNeverDeletesTheNewSnapshot(t *testing.T) { a, dir := backupService(t, 1) - future := usersBackupFile(backupNow.Add(48 * time.Hour)) + future := testUsersBackupFile(backupNow.Add(48 * time.Hour)) writeBackupFile(t, dir, future) a.maybeBackup(backupNow) - want := []string{usersBackupFile(backupNow)} + want := []string{testUsersBackupFile(backupNow)} if names := snapshotNames(t, dir); !reflect.DeepEqual(names, want) { t.Fatalf("snapshots = %v; want %v", names, want) } @@ -144,16 +140,16 @@ func TestUsersBackupRotationKeepsNewestAndForeignFiles(t *testing.T) { a, dir := backupService(t, 3) var old []string // oldest first for i := 5; i >= 1; i-- { - n := usersBackupFile(backupNow.Add(-time.Duration(i) * 25 * time.Hour)) + n := testUsersBackupFile(backupNow.Add(-time.Duration(i) * 25 * time.Hour)) writeBackupFile(t, dir, n) old = append(old, n) } - foreign := []string{"notes.txt", "users-before-upgrade.db", "users.db", usersBackupFile(backupNow.Add(-300*time.Hour)) + ".tmp"} + foreign := []string{"notes.txt", "users-before-upgrade.db", "users.db", testUsersBackupFile(backupNow.Add(-300*time.Hour)) + ".tmp"} for _, n := range foreign { writeBackupFile(t, dir, n) } a.maybeBackup(backupNow) - want := []string{old[3], old[4], usersBackupFile(backupNow)} + want := []string{old[3], old[4], testUsersBackupFile(backupNow)} if names := snapshotNames(t, dir); !reflect.DeepEqual(names, want) { t.Fatalf("after rotation = %v; want %v", names, want) } @@ -173,21 +169,21 @@ func TestUsersBackupSweepsOldTempFiles(t *testing.T) { t.Fatal(err) } } - orphan := usersBackupFile(backupNow.Add(-30*time.Hour)) + ".tmp" - recent := usersBackupFile(backupNow.Add(-time.Hour)) + ".tmp" + orphan := testUsersBackupFile(backupNow.Add(-30*time.Hour)) + ".tmp" + recent := testUsersBackupFile(backupNow.Add(-time.Hour)) + ".tmp" foreign := []string{"other.db.tmp", "users-20261001-000000.db.tmp.bak", "users-before-upgrade.db.tmp"} for _, n := range append([]string{orphan, recent}, foreign...) { writeBackupFile(t, dir, n) setAge(n, 25*time.Hour) } setAge(recent, time.Hour) - dirLike := usersBackupFile(backupNow.Add(-40*time.Hour)) + ".tmp" + dirLike := testUsersBackupFile(backupNow.Add(-40*time.Hour)) + ".tmp" if err := os.Mkdir(filepath.Join(dir, dirLike), 0o700); err != nil { t.Fatal(err) } setAge(dirLike, 25*time.Hour) // A fresh snapshot: no new one is due, the sweep still runs. - writeBackupFile(t, dir, usersBackupFile(backupNow.Add(-time.Hour))) + writeBackupFile(t, dir, testUsersBackupFile(backupNow.Add(-time.Hour))) a.maybeBackup(backupNow) if _, err := os.Stat(filepath.Join(dir, orphan)); !os.IsNotExist(err) { @@ -202,7 +198,7 @@ func TestUsersBackupSweepsOldTempFiles(t *testing.T) { func TestUsersBackupFailureKeepsOldSnapshots(t *testing.T) { a, dir := backupService(t, 1) - stale := usersBackupFile(backupNow.Add(-48 * time.Hour)) + stale := testUsersBackupFile(backupNow.Add(-48 * time.Hour)) writeBackupFile(t, dir, stale) a.st.Close() // every store call fails from here; the fixture's cleanup closes again, which is a no-op a.maybeBackup(backupNow) @@ -229,25 +225,27 @@ func TestUsersBackupDisabled(t *testing.T) { } func TestInitUserManagementBacksUpAtStartup(t *testing.T) { - dir := t.TempDir() - srv := &Server{cfg: &Config{UserManagement: &UserManagementConfig{ - Enabled: true, PublicBaseURL: testBase, - Mail: UserMailConfig{BrevoAPIKey: "k", FromEmail: "noreply@example.org"}, - }}} - if err := srv.initUserManagement(filepath.Join(dir, "meshcore.db")); err != nil { + a, _ := newTestBackupAuthService(t) + dir := filepath.Join(t.TempDir(), "backups") + u := validUM() + u.DatabaseURL = a.set.databaseURL + u.Backup = &UsersBackupConfig{Dir: dir} + srv := &Server{cfg: &Config{UserManagement: u}} + if err := srv.initUserManagement(testDatabaseDSN(t)); err != nil { t.Fatal(err) } - // The janitor's first pass (prune, then backup) runs before it sees stop. srv.closeUserManagement() - if names := snapshotNames(t, filepath.Join(dir, "backups")); len(names) != 1 { + if names := snapshotNames(t, dir); len(names) != 1 { t.Fatalf("snapshots after startup = %v; want 1", names) } } -var usersBackupFilenameRE = regexp.MustCompile(`^attachment; filename="corescope-users-\d{8}-\d{6}\.db"$`) +var usersBackupFilenameRE = regexp.MustCompile(`^attachment; filename="corescope-users-\d{8}-\d{6}\.(?:db|dump)"$`) func TestAdminUsersBackupDownload(t *testing.T) { - f, boss, uma := adminFixture(t) + f := newAuthFixtureWithURL(t, selectedBackupTestDSN(t), "boss@example.org") + boss := f.registerAndActivate(t, "boss@example.org", "Boss", pw) + uma := f.registerAndActivate(t, "uma@example.org", "Uma", pw) tmp := t.TempDir() for _, k := range []string{"TMPDIR", "TMP", "TEMP"} { t.Setenv(k, tmp) @@ -267,20 +265,17 @@ func TestAdminUsersBackupDownload(t *testing.T) { t.Errorf("Content-Disposition = %q", cd) } body := w.Body.Bytes() - if !bytes.HasPrefix(body, []byte("SQLite format 3\x00")) { - t.Fatalf("body is not a SQLite file (%d bytes)", len(body)) + if !bytes.HasPrefix(body, []byte(postgresMagic)) { + t.Fatalf("body is not a PostgreSQL custom archive (%d bytes)", len(body)) } if cl := w.Header().Get("Content-Length"); cl != strconv.Itoa(len(body)) { t.Errorf("Content-Length = %q; body is %d bytes", cl, len(body)) } - path := filepath.Join(t.TempDir(), "download.db") + path := filepath.Join(t.TempDir(), "download.dump") if err := os.WriteFile(path, body, 0o600); err != nil { t.Fatal(err) } - db, err := sql.Open("sqlite", path) - if err != nil { - t.Fatal(err) - } + db := restorePostgresTestBackup(t, path) defer db.Close() var n int if err := db.QueryRow(`SELECT COUNT(*) FROM users`).Scan(&n); err != nil || n != 2 { @@ -297,3 +292,5 @@ func TestAdminUsersBackupDownload(t *testing.T) { t.Fatalf("user.backup row = %+v", e) } } + +func testUsersBackupFile(stamp time.Time) string { return usersBackupFile(stamp, testBackendValue()) } diff --git a/cmd/server/websocket.go b/cmd/server/websocket.go index 14befa64c..121906cc0 100644 --- a/cmd/server/websocket.go +++ b/cmd/server/websocket.go @@ -287,7 +287,7 @@ func (c *Client) writePump(pingInterval time.Duration) { } } -// Poller watches for new transmissions in SQLite and broadcasts them. +// Poller watches for new transmissions in PostgreSQL and broadcasts them. type Poller struct { db *DB hub *Hub @@ -331,9 +331,9 @@ func (p *Poller) Start() { // Ingest new observations for existing transmissions (fixes #174) nextObsID := lastObsID if err := p.db.conn.QueryRow(` - SELECT COALESCE(MAX(id), ?) FROM ( + SELECT COALESCE(MAX(id), $1) FROM ( SELECT id FROM observations - WHERE id > ? + WHERE id > $2 ORDER BY id ASC LIMIT 500 )`, lastObsID, lastObsID).Scan(&nextObsID); err != nil { diff --git a/cmd/server/websocket_test.go b/cmd/server/websocket_test.go index fbdd62106..68e0cf6be 100644 --- a/cmd/server/websocket_test.go +++ b/cmd/server/websocket_test.go @@ -285,7 +285,7 @@ func TestPollerBroadcastsMultipleObservations(t *testing.T) { now := time.Now().UTC().Format(time.RFC3339) if _, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES ('FACE', 'starbursthash237a', ?, 1, 4, '{"pubKey":"aabbccdd11223344","type":"ADVERT"}')`, now); err != nil { + VALUES ('FACE', 'starbursthash237a', $1, 1, 4, '{"pubKey":"aabbccdd11223344","type":"ADVERT"}')`, now); err != nil { t.Fatalf("insert tx failed: %v", err) } var txID int @@ -294,9 +294,9 @@ func TestPollerBroadcastsMultipleObservations(t *testing.T) { } ts := time.Now().Unix() if _, err := db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (?, 1, 14.0, -82, '["aa"]', ?), - (?, 2, 10.5, -90, '["aa","bb"]', ?), - (?, 1, 7.0, -96, '["aa","bb","cc"]', ?)`, + VALUES ($1, 1, 14.0, -82, '["aa"]', $2), + ($3, 2, 10.5, -90, '["aa","bb"]', $4), + ($5, 1, 7.0, -96, '["aa","bb","cc"]', $6)`, txID, ts, txID, ts+1, txID, ts+2); err != nil { t.Fatalf("insert observations failed: %v", err) } @@ -367,8 +367,8 @@ func TestIngestNewObservationsBroadcast(t *testing.T) { maxObs := db.GetMaxObservationID() now := time.Now().Unix() if _, err := db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (1, 2, 6.0, -100, '["aa","zz"]', ?), - (1, 1, 5.0, -101, '["aa","yy"]', ?)`, now, now+1); err != nil { + VALUES (1, 2, 6.0, -100, '["aa","zz"]', $1), + (1, 1, 5.0, -101, '["aa","yy"]', $2)`, now, now+1); err != nil { t.Fatalf("insert new observations failed: %v", err) } diff --git a/cmd/server/zerohop_hashsize_test.go b/cmd/server/zerohop_hashsize_test.go index 6509989e6..f40a2dd39 100644 --- a/cmd/server/zerohop_hashsize_test.go +++ b/cmd/server/zerohop_hashsize_test.go @@ -25,7 +25,7 @@ func insertAdvert(t *testing.T, db *DB, id int, rawHex, hash, pubKey, name strin t.Helper() if _, err := db.conn.Exec( `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json) - VALUES (?, ?, ?, ?, 4, ?)`, + VALUES ($1, $2, $3, $4, 4, $5)`, rawHex, hash, seen, routeType, fmt.Sprintf(`{"pubKey":"%s","name":"%s","type":"ADVERT"}`, pubKey, name), ); err != nil { @@ -33,7 +33,7 @@ func insertAdvert(t *testing.T, db *DB, id int, rawHex, hash, pubKey, name strin } if _, err := db.conn.Exec( `INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) - VALUES (?, 1, 10.0, -90, '[]', ?)`, id, seenEpoch, + VALUES ($1, 1, 10.0, -90, '[]', $2)`, id, seenEpoch, ); err != nil { t.Fatalf("insert observation %s: %v", hash, err) } @@ -45,7 +45,7 @@ func zeroHopStore(t *testing.T) (*DB, string, int64) { now := time.Now().UTC().Add(-1 * time.Hour) recent := now.Format(time.RFC3339) db.conn.Exec(`INSERT INTO observers (id, name, iata, last_seen, first_seen, packet_count) - VALUES ('obs1', 'Obs', 'PRG', ?, '2026-01-01T00:00:00Z', 10)`, recent) + VALUES ('obs1', 'Obs', 'PRG', $1, '2026-01-01T00:00:00Z', 10)`, recent) return db, recent, now.Unix() } @@ -161,7 +161,7 @@ func TestZeroHopDeclaredSizeReachesMultiByteCapability(t *testing.T) { const pk = "5361aa08929b6248bb61cb1fab0fdb97d6dba8ae44f53a44427f63e7798a6b6b" db.conn.Exec(`INSERT INTO nodes (public_key, name, role, last_seen, first_seen, advert_count) - VALUES (?, 'tth-hrebecna', 'repeater', ?, ?, 1)`, pk, recent, recent) + VALUES ($1, 'tth-hrebecna', 'repeater', $2, $3, 1)`, pk, recent, recent) insertAdvert(t, db, 1, "1240"+pk+"cafe", "zh_cap", pk, "tth-hrebecna", 2, recent, epoch) store := NewPacketStore(db, nil) diff --git a/config.example.json b/config.example.json index 847d41115..02dcce9ca 100644 --- a/config.example.json +++ b/config.example.json @@ -1,6 +1,11 @@ { "port": 3000, "apiKey": "your-secret-api-key-here", + "dbPath": "", + "databaseURL": "", + "stateDir": "data", + "_comment_databaseURL": "PostgreSQL telemetry URL. Prefer environment credentials: CORESCOPE_READER_DATABASE_URL for the server, CORESCOPE_WRITER_DATABASE_URL for the ingestor, or CORESCOPE_DATABASE_URL when running a single process. Runtime roles are separate from the bootstrap owner. Used only when PostgreSQL is selected. SQLite uses dbPath/DB_PATH. After installation the private storage-selection.json record wins over bootstrap paths/backend settings; change engines only through the offline operator workflow in docs/storage.md.", + "_comment_stateDir": "Directory shared by the server and ingestor for file queues, statistics and account backups; CORESCOPE_STATE_DIR overrides it. This is a filesystem path, never a database URL. The default is data; containers use /app/data.", "nodeBlacklist": [], "_comment_nodeBlacklist": "Public keys of nodes to hide from all API responses. Use for trolls, offensive names, or nodes reporting false data that operators refuse to fix.", "hiddenNamePrefixes": ["🚫"], @@ -17,17 +22,18 @@ "clientRfDays": 30, "clientRegionsDays": 30, "hopIndexDays": 0, - "_comment": "nodeDays: nodes not seen in N days moved to inactive_nodes (default 7). observerDays: observers not sending data in N days are removed (-1 = keep forever, default 14). observerPurgeDays: observers already marked inactive AND not seen in N days are hard-deleted, but only when no observation, observer_metrics or dropped_packets row still references them (0 = disabled, the default). Set it above both observerDays and packetDays — below those the reference guards keep every row anyway. packetDays: transmissions older than N days are deleted (0 = disabled). clientRxDays: mobile client-RX coverage rows (client_receptions/client_observers) older than N days are deleted (0 = disabled) — bounds the opt-in coverage tables (#1727). clientRxObsDays: diagnostic client_rx_observations rows (by rx_at) older than N days are deleted (0 = disabled) — bounds the opt-in clientRxObservations table. clientRfDays: RF environment sample rows (client_rf_samples, by sampled_at) older than N days are deleted (0 = disabled) — bounds the opt-in clientRfSamples table; unset means unbounded growth at roughly 240 rows/hour per driver. clientRegionsDays: region-discovery answer rows (node_declared_regions, by observed_at) older than N days are deleted (0 = disabled) — bounds the opt-in clientRegions table. hopIndexDays: opt-in index that speeds up the node reach panel (/api/nodes/{pk}/reach) on large databases; 0 = off (the default): no table, no triggers. Negative values are rejected at startup (unlike observerDays, -1 does not mean keep forever). N > 0 indexes the last N days (14 covers the UI's 7-day default with margin). Costs roughly 200 bytes per observation (about 30% of the observations table) and a trigger on every observation insert; reach windows longer than N days use the slower path_json scan. Changes apply at the next ingestor start: raising N backfills the older days in the background, lowering it trims, and 0 removes the table and triggers (about a minute on a large database; the ingestor is already subscribed and buffers packets meanwhile, up to ingestBufferSize). Before rolling back to a version without this setting, set it to 0 and start the ingestor once; an older ingestor keeps the triggers but never trims, so the table would grow without limit. NOTE (#1283): all retention fields are consumed by the INGESTOR process. The server is read-only and never prunes." + "_comment": "nodeDays: nodes not seen in N days moved to inactive_nodes (default 7). observerDays: observers not sending data in N days are removed (-1 = keep forever, default 14). observerPurgeDays: observers already marked inactive AND not seen in N days are hard-deleted, but only when no observation, observer_metrics or dropped_packets row still references them (0 = disabled, the default). Set it above both observerDays and packetDays — below those the reference guards keep every row anyway. packetDays: transmissions older than N days are deleted (0 = disabled). clientRxDays: mobile client-RX coverage rows (client_receptions/client_observers) older than N days are deleted (0 = disabled) — bounds the opt-in coverage tables (#1727). clientRxObsDays: diagnostic client_rx_observations rows (by rx_at) older than N days are deleted (0 = disabled) — bounds the opt-in clientRxObservations table. clientRfDays: RF environment sample rows (client_rf_samples, by sampled_at) older than N days are deleted (0 = disabled) — bounds the opt-in clientRfSamples table; unset means unbounded growth at roughly 240 rows/hour per driver. clientRegionsDays: region-discovery answer rows (node_declared_regions, by observed_at) older than N days are deleted (0 = disabled) — bounds the opt-in clientRegions table. hopIndexDays: SQLite-only opt-in index that speeds up the node reach panel (/api/nodes/{pk}/reach) on large databases; 0 = off (the default): no table, no triggers. Negative values are rejected at startup (unlike observerDays, -1 does not mean keep forever). N > 0 indexes the last N days (14 covers the UI's 7-day default with margin). PostgreSQL logs that it uses the native reach scan instead; this option installs no PostgreSQL hop-index table or triggers. Costs roughly 200 bytes per observation (about 30% of the observations table) and a trigger on every observation insert; reach windows longer than N days use the slower path_json scan. Changes apply at the next ingestor start: raising N backfills the older days in the background, lowering it trims, and 0 removes the table and triggers (about a minute on a large database; the ingestor is already subscribed and buffers packets meanwhile, up to ingestBufferSize). Before rolling back to a version without this setting, set it to 0 and start the ingestor once; an older ingestor keeps the triggers but never trims, so the table would grow without limit. NOTE (#1283): all retention fields are consumed by the INGESTOR process. The server is read-only and never prunes." }, "db": { + "backend": "sqlite", "vacuumOnStartup": false, "incrementalVacuumPages": 1024, "load": { "chunkSize": 10000, "_comment": "chunkSize: rows fetched per chunk by PacketStore.LoadChunked during startup (#1009). Default 10000. Lower values surface the early-HTTP-readiness signal sooner (the listener binds after the first chunk) at the cost of more SQL round-trips. Higher values reduce per-chunk overhead but delay first-chunk readiness. The X-CoreScope-Load-Status response header reports loading|ready and progress= until the load completes." }, - "_comment": "vacuumOnStartup: run one-time full VACUUM to enable incremental auto-vacuum on existing DBs. Executed by the INGESTOR at startup, BEFORE the MQTT subscriber starts (#1283), so there is no contention with concurrent writes. Blocks ingestor startup for minutes on large DBs; requires 2x DB file size in free disk space. incrementalVacuumPages: free pages returned to OS after each retention reaper cycle (default 1024). See #919. load.chunkSize: see nested _comment (#1009).", - "_comment_slowWriterMs": "#1340 — SQLite writer-lock log threshold (default 500). Any wrapped writer call (tagged neighbor_builder, mqtt_handler, prune_packets, prune_observers, prune_metrics, vacuum) whose hold_ms exceeds this emits a single [db-slow-writer] log line. Configured per-process via the CORESCOPE_DB_SLOW_WRITER_MS environment variable on the INGESTOR (e.g. CORESCOPE_DB_SLOW_WRITER_MS=200 for tighter alerting). Per-component wait_ms / hold_ms / contention_total histograms are surfaced via /api/perf/write-sources under .writer_perf regardless of this threshold." + "_comment": "backend selects a fresh install: sqlite (default) or postgres; CORESCOPE_DB_BACKEND supplies a bootstrap environment choice. Installed storage-selection.json is authoritative, including when older config still says sqlite after a verified PostgreSQL conversion. Backend changes are offline operator actions, not live customizer toggles. SQLite vacuumOnStartup is opt-in, can block startup for minutes and needs working disk space; incrementalVacuumPages defaults to 1024. These SQLite maintenance controls are ignored by PostgreSQL, which manages VACUUM/ANALYZE/checkpoints. Keep PostgreSQL autovacuum enabled. load.chunkSize applies to both engines.", + "_comment_slowWriterMs": "Writer-lock log threshold (default 500 ms). Set CORESCOPE_DB_SLOW_WRITER_MS on the ingestor to adjust logging. Per-component wait/hold time and contention counters are available at /api/perf/write-sources. These counters describe the application writer, not PostgreSQL cluster-wide commits or WAL flushes." }, "listLimits": { "packetsMax": 10000, @@ -37,7 +43,7 @@ "bulkHealthMax": 200, "_comment": "Maximum row counts returned by list API endpoints. These enforce a DoS-bounded ceiling for both UI and external requests. Operators with small/embedded deployments can tighten these; operators running large regional meshes can raise them. bulkHealthMax is intentionally separate from nodesMax: /api/nodes/bulk-health is per-row much heavier than /api/nodes (it joins per-node observer health + recent-packet stats) so its ceiling stays low (default 200) even if nodesMax is raised." }, - "_comment_ingestorStats": "Ingestor publishes a 1-Hz stats snapshot consumed by the server's /api/perf/io and /api/perf/write-sources endpoints (#1120). Path is configured via the CORESCOPE_INGESTOR_STATS environment variable on the INGESTOR process. Default: /tmp/corescope-ingestor-stats.json. The writer uses O_NOFOLLOW + 0o600, so a pre-planted symlink in /tmp cannot be used to clobber an arbitrary file. SECURITY: in shared-tmp environments (multi-tenant hosts), point CORESCOPE_INGESTOR_STATS at a private directory like /var/lib/corescope/ingestor-stats.json that only the corescope user can write to.", + "_comment_ingestorStats": "Ingestor publishes a 1-Hz snapshot consumed by /api/perf/io and /api/perf/write-sources. The default is ingestor-stats.json under stateDir. CORESCOPE_INGESTOR_STATS overrides it and must select the same file for both processes. Keep the state directory writable only by the instance operator.", "corsAllowedOrigins": [], "_comment_corsAllowedOrigins": "Cross-origin allowlist for embed scenarios (#1369) AND for /ws WebSocket upgrades (#1793 — CSWSH defense per OWASP WebSocket Security Cheat Sheet). Exact-match origins, e.g. [\"https://blog.example.com\", \"https://embed.example.com\"]. Same-origin requests (Origin host == request Host) and non-browser clients (no Origin header) are always allowed for /ws. When empty (default), no Access-Control-* headers are sent and browsers enforce same-origin; cross-origin /ws upgrades are rejected. When non-empty, only the listed origins receive CORS headers, and Access-Control-Allow-Methods is limited to GET, HEAD, OPTIONS (the cross-domain surface is read-only — same-origin admin writes are unaffected). One exception, with userManagement.enabled: the CoreDrive RX device-token routes (/api/auth/device-token, /api/auth/me, /api/auth/logout, /api/account/settings, /api/account/companions*) also allow POST, PUT, DELETE and the Authorization and Content-Type headers for listed origins, so an RX build on another origin can log in and link companions with a bearer token. Use [\"*\"] to allow any origin for CORS XHR (NOT recommended for write-capable deployments); note that \"*\" is deliberately NOT honored for /ws upgrades — list explicit origins to permit cross-origin WebSocket clients (OWASP guidance: avoid wildcards in CSWSH defense). Operators can override per-deployment with the CORS_ALLOWED_ORIGINS environment variable (comma-separated). No credentialed CORS is enabled. To embed the map or channels pages cross-domain, add the embedding origin here and use the URL pattern '/#/map?embed=1' or '/#/channels?embed=1' — embed mode hides the top-nav, bottom-nav, and side drawer for full-bleed iframe rendering.", "webSocket": { @@ -453,6 +459,8 @@ "userManagement": { "enabled": false, "dbPath": "", + "databaseURL": "", + "approvedChannelsDatabaseURL": "", "adminEmails": ["operator@example.org"], "publicBaseUrl": "https://corescope.example.org", "sessionDays": 30, @@ -485,10 +493,10 @@ "keep": 7 } }, - "_comment_userManagement": "Optional accounts (off by default; see docs/user-guide/accounts.md). When enabled: visitors can register with email + password and activate through a mailed link; addresses in adminEmails become admins; admins manage users at #/admin/users and can use the operator endpoints without the apiKey. The dashboard stays public. dbPath defaults to users.db next to the analyzer database: a separate file, the server never writes measurement data. publicBaseUrl is required and is used for every link in every mail. Mail goes through Brevo's transactional API (a free account is enough); brevoApiKey and webhookSecret may instead come from CORESCOPE_BREVO_API_KEY and CORESCOPE_BREVO_WEBHOOK_SECRET. Set webhookSecret (16+ chars) and point a Brevo transactional webhook at /api/mail/brevo/webhook with bearer auth to see delivery status per user. Alternatively set mail.provider to \"postal\" to send through your own Postal server: postalBaseUrl (https) and postalApiKey (or CORESCOPE_POSTAL_API_KEY) are then required, and the webhook is https://postal:@/api/mail/postal/webhook (secret: letters, digits, . _ - only; or CORESCOPE_POSTAL_WEBHOOK_SECRET). trustedProxies (CIDRs) lets login rate limits use X-Forwarded-For behind a reverse proxy. Startup fails if enabled with an incomplete mail setup.", - "_comment_channelProposals": "Optional, needs userManagement.enabled (docs/user-guide/accounts.md, Channel proposals). Logged-in users propose hashtag channels; admins approve, reject or revoke them on the Proposals tab. An approved channel is decrypted by the ingestor from its next refresh (once a minute, no restart) and listed for every visitor on the Channels page. maxApproved bounds the decryption work (every approved key is tried on every group message); perUserPerDay and maxPending limit proposals. The ingestor reads users.db read-only from userManagement.dbPath, or next to the analyzer database; set dbPath explicitly when the server and the ingestor are not given the same analyzer database path.", + "_comment_userManagement": "Optional accounts (off by default; see docs/user-guide/accounts.md). When enabled: visitors can register with email + password and activate through a mailed link; addresses in adminEmails become admins; admins manage users at #/admin/users and can use the operator endpoints without the apiKey. The dashboard stays public. SQLite dbPath defaults to users.db beside telemetry; PostgreSQL databaseURL (or CORESCOPE_USERS_DATABASE_URL) uses a separate account database with a restricted writer role. The recorded account target persists while this feature is disabled and participates in an offline backend conversion. The server never writes telemetry. Packaged setup validates/initializes the selected stores before startup. publicBaseUrl is required and is used for every link in every mail. Mail goes through Brevo's transactional API (a free account is enough); brevoApiKey and webhookSecret may instead come from CORESCOPE_BREVO_API_KEY and CORESCOPE_BREVO_WEBHOOK_SECRET. Set webhookSecret (16+ chars) and point a Brevo transactional webhook at /api/mail/brevo/webhook with bearer auth to see delivery status per user. Alternatively set mail.provider to \"postal\" to send through your own Postal server: postalBaseUrl (https) and postalApiKey (or CORESCOPE_POSTAL_API_KEY) are then required, and the webhook is https://postal:@/api/mail/postal/webhook (secret: letters, digits, . _ - only; or CORESCOPE_POSTAL_WEBHOOK_SECRET). trustedProxies (CIDRs) lets login rate limits use X-Forwarded-For behind a reverse proxy. Startup fails if enabled with an incomplete mail setup.", + "_comment_channelProposals": "Optional, needs userManagement.enabled (docs/user-guide/accounts.md, Channel proposals). Logged-in users propose hashtag channels; admins approve, reject or revoke them on the Proposals tab. An approved channel is decrypted by the ingestor from its next refresh (once a minute, no restart) and listed for every visitor on the Channels page. maxApproved bounds the decryption work (every approved key is tried on every group message); perUserPerDay and maxPending limit proposals. The ingestor reads the approved_channels view using userManagement.approvedChannelsDatabaseURL or CORESCOPE_APPROVED_CHANNELS_DATABASE_URL. The same restricted reader exposes only pubkeys through linked_companions when clientRxCoverage.requireLinkedCompanion is enabled. It must have no access to account tables or write privileges.", "_comment_notifications": "Optional, needs userManagement.enabled (docs/user-guide/accounts.md, Node notifications). Logged-in users watch nodes and get one mail per check when a watched node goes offline, comes back or reports a low battery; admins can also watch for new foreign nodes and observers going offline. Thresholds are healthThresholds and batteryThresholds. A check runs every intervalMinutes; at most perUserPerDay mails per user and maxMailsPerDay in total go out per 24 hours (over a limit the change is recorded and dropped, never queued). maxMailsPerDay counts notification mail only: keep it below the mail provider's daily quota so activation, password-reset and address-change mail still go out (the default 100 leaves 200 of Brevo's free 300 a day); maxWatchesPerUser bounds each watch list. Every mail has a one-click unsubscribe. Optional externalAlerts {url, maxAgeHours (48), label} offers the node.external event, fed by a JSON alerts feed computed outside CoreScope (docs/user-guide/accounts.md, External node alerts).", - "_comment_usersBackup": "Part of userManagement (docs/user-guide/accounts.md, Backups). With accounts on, the server snapshots users.db at startup when the newest snapshot is older than 24 hours, then about every 24 hours, to users-.db (UTC, mode 0600) in dir (empty: backups/ next to users.db), and deletes the oldest beyond keep (default 7), never the one just written. On unless enabled is false. Snapshots hold password hashes, and deleted accounts stay in them for up to keep days: protect the directory like users.db. A dir that already exists is not tightened to 0700: set that yourself. Admins download a fresh snapshot at /api/admin/users-backup. A restore brings back accounts deleted after the snapshot (also self-deletions), old passwords, revoked sessions and used links: follow the Restore steps in the doc (note the deletions from the current audit log first, then disable those accounts and clear sessions and tokens before starting, and delete them again in the admin area).", + "_comment_usersBackup": "With accounts enabled, the server creates a consistent native snapshot at startup when the newest snapshot is older than 24 hours, then daily. SQLite snapshots use .db; PostgreSQL custom-format archives use .dump. Files are users- under backup.dir (default: backups under stateDir). keep limits retained snapshots (default 7). PostgreSQL requires pg_dump18; the supplied image includes it. Archives contain authentication state: protect the directory and follow the account restore procedure before restarting, because restoring also restores old passwords, sessions, used tokens and deleted accounts.", "customizer": { "disabledTabs": [], "_comment_disabledTabs": "Issue #1508. List of customizer-modal tab ids to hide from end users. Useful when operators want to expose only daily-use viewer controls and keep one-shot admin chrome out of the way. Recognized ids: branding, theme, nodes, home, display, geofilter, export. Example: [\"branding\", \"geofilter\", \"export\"] hides the admin tabs and leaves theme/colors/home/display visible. Default [] keeps the legacy behavior (all tabs visible). Operators edit this in config.json directly — there is no in-app UI for this list (by design)." diff --git a/docker-compose.example.postgres.yml b/docker-compose.example.postgres.yml new file mode 100644 index 000000000..1d7392eb9 --- /dev/null +++ b/docker-compose.example.postgres.yml @@ -0,0 +1,28 @@ +# Optional PostgreSQL storage. Combine with the matching SQLite-default base +# using -f base.yml -f this-file.yml. No profiles: unused PG credentials and +# services are absent from the base model. Changing files is not a conversion; +# use the documented offline storage switch before changing an installed backend. +services: + postgres: + extends: + file: docker/postgres.compose.yml + service: postgres + volumes: + - ${POSTGRES_DATA_DIR:-./postgres-data}:/var/lib/postgresql + bootstrap: + extends: + file: docker/postgres.compose.yml + service: bootstrap + image: ${CORESCOPE_IMAGE:-ghcr.io/kpa-clawbot/corescope:latest} + volumes: + - ${DATA_DIR:-./data}:/app/data + corescope: + depends_on: + bootstrap: + condition: service_completed_successfully + environment: + CORESCOPE_DB_BACKEND: postgres + CORESCOPE_READER_DATABASE_URL: postgresql://corescope_reader:${CORESCOPE_READER_PASSWORD:?Set CORESCOPE_READER_PASSWORD}@postgres:5432/corescope_telemetry?sslmode=disable&search_path=public + CORESCOPE_WRITER_DATABASE_URL: postgresql://corescope_writer:${CORESCOPE_WRITER_PASSWORD:?Set CORESCOPE_WRITER_PASSWORD}@postgres:5432/corescope_telemetry?sslmode=disable&search_path=public + CORESCOPE_USERS_DATABASE_URL: postgresql://corescope_accounts:${CORESCOPE_ACCOUNTS_PASSWORD:?Set CORESCOPE_ACCOUNTS_PASSWORD}@postgres:5432/corescope_accounts?sslmode=disable&search_path=public + CORESCOPE_APPROVED_CHANNELS_DATABASE_URL: postgresql://corescope_channels:${CORESCOPE_CHANNELS_PASSWORD:?Set CORESCOPE_CHANNELS_PASSWORD}@postgres:5432/corescope_accounts?sslmode=disable&search_path=public diff --git a/docker-compose.example.yml b/docker-compose.example.yml index e600becbb..54df15b75 100644 --- a/docker-compose.example.yml +++ b/docker-compose.example.yml @@ -1,10 +1,9 @@ -# CoreScope — simple single-container deployment using the pre-built -# image from GHCR. +# CoreScope — SQLite by default, using the pre-built +# application image from GHCR. # # AUDIENCE: third-party operators running a self-hosted CoreScope on -# ONE host. Ships with the built-in Mosquitto broker DISABLED by -# default (DISABLE_MOSQUITTO defaults to `false` here, meaning -# mosquitto RUNS in-container — the historical shape); flip to `true` +# ONE host. Ships with the built-in Mosquitto broker enabled +# (DISABLE_MOSQUITTO defaults to `false`); flip to `true` # if you already have an external MQTT broker to point CoreScope at. # # For the CoreScope maintainers' internal staging deployment (with a @@ -12,21 +11,26 @@ # `meshcore-net`), see `docker-compose.staging.yml`. # # Usage: docker compose -f docker-compose.example.yml up -d +# Optional PostgreSQL: add -f docker-compose.example.postgres.yml after setup. # Docs: https://github.com/Kpa-clawbot/CoreScope/blob/master/DEPLOY.md +name: corescope + services: corescope: - image: ghcr.io/kpa-clawbot/corescope:latest + image: ${CORESCOPE_IMAGE:-ghcr.io/kpa-clawbot/corescope:latest} ports: - "${HTTP_PORT:-80}:80" volumes: - ${DATA_DIR:-./data}:/app/data environment: - - DISABLE_CADDY=${DISABLE_CADDY:-true} + - CORESCOPE_DB_BACKEND=sqlite + - CORESCOPE_STATE_DIR=${CORESCOPE_STATE_DIR:-/app/data} + - DISABLE_CADDY=${DISABLE_CADDY:-false} - DISABLE_MOSQUITTO=${DISABLE_MOSQUITTO:-false} restart: unless-stopped healthcheck: - test: ["CMD", "wget", "-qO-", "http://localhost:3000/api/stats"] + test: ["CMD", "wget", "-qO-", "http://localhost:3000/api/healthz"] interval: 30s timeout: 5s retries: 3 diff --git a/docker-compose.no-mosquitto.yml b/docker-compose.no-mosquitto.yml index d8b114080..1d6f8a3f1 100644 --- a/docker-compose.no-mosquitto.yml +++ b/docker-compose.no-mosquitto.yml @@ -13,6 +13,8 @@ # deploy that keeps the built-in broker, see # `docker-compose.example.yml`. +name: corescope + services: prod: build: @@ -36,10 +38,12 @@ services: - ${PROD_DATA_DIR:-~/meshcore-data}:/app/data - caddy-data:/data/caddy environment: + - CORESCOPE_DB_BACKEND=sqlite + - CORESCOPE_STATE_DIR=${CORESCOPE_STATE_DIR:-/app/data} - NODE_ENV=production - DISABLE_MOSQUITTO=true healthcheck: - test: ["CMD", "wget", "-qO-", "http://localhost:3000/api/stats"] + test: ["CMD", "wget", "-qO-", "http://localhost:3000/api/healthz"] interval: 30s timeout: 5s retries: 3 diff --git a/docker-compose.postgres.yml b/docker-compose.postgres.yml new file mode 100644 index 000000000..132310170 --- /dev/null +++ b/docker-compose.postgres.yml @@ -0,0 +1,28 @@ +# Optional PostgreSQL storage. Combine with the matching SQLite-default base +# using -f base.yml -f this-file.yml. No profiles: unused PG credentials and +# services are absent from the base model. Changing files is not a conversion; +# use the documented offline storage switch before changing an installed backend. +services: + postgres: + extends: + file: docker/postgres.compose.yml + service: postgres + volumes: + - ${PROD_POSTGRES_DATA_DIR:-~/corescope-postgres}:/var/lib/postgresql + bootstrap: + extends: + file: docker/postgres.compose.yml + service: bootstrap + image: corescope:latest + volumes: + - ${PROD_DATA_DIR:-~/meshcore-data}:/app/data + prod: + depends_on: + bootstrap: + condition: service_completed_successfully + environment: + CORESCOPE_DB_BACKEND: postgres + CORESCOPE_READER_DATABASE_URL: postgresql://corescope_reader:${CORESCOPE_READER_PASSWORD:?Set CORESCOPE_READER_PASSWORD}@postgres:5432/corescope_telemetry?sslmode=disable&search_path=public + CORESCOPE_WRITER_DATABASE_URL: postgresql://corescope_writer:${CORESCOPE_WRITER_PASSWORD:?Set CORESCOPE_WRITER_PASSWORD}@postgres:5432/corescope_telemetry?sslmode=disable&search_path=public + CORESCOPE_USERS_DATABASE_URL: postgresql://corescope_accounts:${CORESCOPE_ACCOUNTS_PASSWORD:?Set CORESCOPE_ACCOUNTS_PASSWORD}@postgres:5432/corescope_accounts?sslmode=disable&search_path=public + CORESCOPE_APPROVED_CHANNELS_DATABASE_URL: postgresql://corescope_channels:${CORESCOPE_CHANNELS_PASSWORD:?Set CORESCOPE_CHANNELS_PASSWORD}@postgres:5432/corescope_accounts?sslmode=disable&search_path=public diff --git a/docker-compose.staging.no-mosquitto.yml b/docker-compose.staging.no-mosquitto.yml index 281164d19..402c3f6ef 100644 --- a/docker-compose.staging.no-mosquitto.yml +++ b/docker-compose.staging.no-mosquitto.yml @@ -12,6 +12,8 @@ # `docker-compose.example.yml` (or `docker-compose.no-mosquitto.yml` # for a prod-flavoured variant with an external broker). +name: corescope-staging + services: staging-go: build: @@ -40,11 +42,13 @@ services: - ${STAGING_DATA_DIR:-~/meshcore-staging-data}:/app/data - caddy-data-staging-go:/data/caddy environment: + - CORESCOPE_DB_BACKEND=sqlite + - CORESCOPE_STATE_DIR=${CORESCOPE_STATE_DIR:-/app/data} - NODE_ENV=staging - ENABLE_PPROF=true - DISABLE_MOSQUITTO=true healthcheck: - test: ["CMD", "wget", "-qO-", "http://localhost:3000/api/stats"] + test: ["CMD", "wget", "-qO-", "http://localhost:3000/api/healthz"] interval: 30s timeout: 10s retries: 5 diff --git a/docker-compose.staging.postgres.yml b/docker-compose.staging.postgres.yml new file mode 100644 index 000000000..a716170f2 --- /dev/null +++ b/docker-compose.staging.postgres.yml @@ -0,0 +1,28 @@ +# Optional PostgreSQL storage. Combine with the matching SQLite-default base +# using -f base.yml -f this-file.yml. No profiles: unused PG credentials and +# services are absent from the base model. Changing files is not a conversion; +# use the documented offline storage switch before changing an installed backend. +services: + postgres: + extends: + file: docker/postgres.compose.yml + service: postgres + volumes: + - ${STAGING_POSTGRES_DATA_DIR:-~/corescope-staging-postgres}:/var/lib/postgresql + bootstrap: + extends: + file: docker/postgres.compose.yml + service: bootstrap + image: corescope-go:latest + volumes: + - ${STAGING_DATA_DIR:-~/meshcore-staging-data}:/app/data + staging-go: + depends_on: + bootstrap: + condition: service_completed_successfully + environment: + CORESCOPE_DB_BACKEND: postgres + CORESCOPE_READER_DATABASE_URL: postgresql://corescope_reader:${CORESCOPE_READER_PASSWORD:?Set CORESCOPE_READER_PASSWORD}@postgres:5432/corescope_telemetry?sslmode=disable&search_path=public + CORESCOPE_WRITER_DATABASE_URL: postgresql://corescope_writer:${CORESCOPE_WRITER_PASSWORD:?Set CORESCOPE_WRITER_PASSWORD}@postgres:5432/corescope_telemetry?sslmode=disable&search_path=public + CORESCOPE_USERS_DATABASE_URL: postgresql://corescope_accounts:${CORESCOPE_ACCOUNTS_PASSWORD:?Set CORESCOPE_ACCOUNTS_PASSWORD}@postgres:5432/corescope_accounts?sslmode=disable&search_path=public + CORESCOPE_APPROVED_CHANNELS_DATABASE_URL: postgresql://corescope_channels:${CORESCOPE_CHANNELS_PASSWORD:?Set CORESCOPE_CHANNELS_PASSWORD}@postgres:5432/corescope_accounts?sslmode=disable&search_path=public diff --git a/docker-compose.staging.yml b/docker-compose.staging.yml index 6002fb916..18d77a424 100644 --- a/docker-compose.staging.yml +++ b/docker-compose.staging.yml @@ -20,6 +20,8 @@ # # Override defaults via .env or environment variables. +name: corescope-staging + services: staging-go: build: @@ -55,6 +57,8 @@ services: - ${STAGING_DATA_DIR:-~/meshcore-staging-data}:/app/data - caddy-data-staging-go:/data/caddy environment: + - CORESCOPE_DB_BACKEND=sqlite + - CORESCOPE_STATE_DIR=${CORESCOPE_STATE_DIR:-/app/data} - NODE_ENV=staging - ENABLE_PPROF=true # Default TRUE: the standalone mqtt-broker container owns MQTT on @@ -72,7 +76,7 @@ services: # via docker DNS without going through the host. meshcore-net: {} healthcheck: - test: ["CMD", "wget", "-qO-", "http://localhost:3000/api/stats"] + test: ["CMD", "wget", "-qO-", "http://localhost:3000/api/healthz"] interval: 30s timeout: 10s retries: 5 diff --git a/docker-compose.yml b/docker-compose.yml index 3fb4b8bbd..754a9fc48 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,6 +3,8 @@ # CRITICAL: All data mounts use bind mounts (~/path), NOT named volumes. # This ensures the DB and theme are visible on the host filesystem for backup. +name: corescope + services: prod: build: @@ -27,11 +29,13 @@ services: - ${PROD_DATA_DIR:-~/meshcore-data}:/app/data - caddy-data:/data/caddy environment: + - CORESCOPE_DB_BACKEND=sqlite + - CORESCOPE_STATE_DIR=${CORESCOPE_STATE_DIR:-/app/data} - NODE_ENV=production - DISABLE_MOSQUITTO=${DISABLE_MOSQUITTO:-false} - DISABLE_CADDY=${DISABLE_CADDY:-false} healthcheck: - test: ["CMD", "wget", "-qO-", "http://localhost:3000/api/stats"] + test: ["CMD", "wget", "-qO-", "http://localhost:3000/api/healthz"] interval: 30s timeout: 5s retries: 3 diff --git a/docker/entrypoint-go.sh b/docker/entrypoint-go.sh index 2c95907e3..77a972e24 100644 --- a/docker/entrypoint-go.sh +++ b/docker/entrypoint-go.sh @@ -1,4 +1,5 @@ #!/bin/sh +set -eu # Config lives in the data directory (bind-mounted from host) # The Go server already searches /app/data/config.json via LoadConfig @@ -24,6 +25,14 @@ fi . /select-service-user.sh prepare_service_user || exit 1 +# Setup may create SQLite databases and the private selection record. Create +# them as the services' user, after the existing-data ownership hand-over. +if [ "$CORESCOPE_SERVICE_USER" = root ]; then + /app/storage.sh start +else + su -s /bin/sh -c '/app/storage.sh start' "$CORESCOPE_SERVICE_USER" +fi + SUPERVISORD_CONF="/etc/supervisor/conf.d/supervisord.conf" if [ "${DISABLE_MOSQUITTO:-false}" = "true" ] && [ "${DISABLE_CADDY:-false}" = "true" ]; then echo "[config] internal MQTT broker disabled (DISABLE_MOSQUITTO=true)" diff --git a/docker/postgres-bootstrap.sh b/docker/postgres-bootstrap.sh new file mode 100755 index 000000000..675a02ef6 --- /dev/null +++ b/docker/postgres-bootstrap.sh @@ -0,0 +1,44 @@ +#!/bin/sh +set -eu + +# Only the owner service may initialize/adopt PostgreSQL. Existing SQLite data +# requires the explicit verified switch; a missing record is never a fallback. +/app/storage.sh postgres-setup +/app/storage.sh managed-postgres +# Legacy check-ready remains a read-only schema check; use owner-only environment +# for this child without changing the runtime account role URL or leaking argv. +databases=corescope_telemetry +account_owner= +case $(/app/storage.sh field has_accounts) in + true) + databases="$databases corescope_accounts" + account_owner=${CORESCOPE_USERS_OWNER_DATABASE_URL:?set account owner URL} + ;; + false) ;; + *) echo 'Cannot read the recorded account target.' >&2; exit 1 ;; +esac +if readiness=$(CORESCOPE_USERS_DATABASE_URL="$account_owner" /app/corescope-migrate -check-ready 2>&1); then + printf '%s\n' "$readiness" +else + # Only a recorded, existing PostgreSQL target can be upgraded. Unlike setup, + # this action cannot create a missing database or add an absent account store. + # Its offline lease refuses active writers; failed upgrades block startup. + # Supported old versions are expected here. The owner upgrade reports their + # version transition; retain provisional diagnostics only if it fails. + if ! /app/storage.sh upgrade; then + printf '%s\n' "$readiness" >&2 + echo 'PostgreSQL schema upgrade failed; runtime services remain stopped.' >&2 + exit 1 + fi + if ! CORESCOPE_USERS_DATABASE_URL="$account_owner" /app/corescope-migrate -check-ready; then + echo 'PostgreSQL schema verification failed; runtime services remain stopped.' >&2 + exit 1 + fi +fi + +export PGHOST=postgres PGPORT=5432 PGSSLMODE=disable +export PGUSER=corescope_owner PGPASSWORD="${CORESCOPE_OWNER_PASSWORD:?set CORESCOPE_OWNER_PASSWORD}" +for PGDATABASE in $databases; do + export PGDATABASE + psql -X -v ON_ERROR_STOP=1 -f /app/postgres-grants.sql +done diff --git a/docker/postgres-grants.sql b/docker/postgres-grants.sql new file mode 100644 index 000000000..138dc0576 --- /dev/null +++ b/docker/postgres-grants.sql @@ -0,0 +1,23 @@ +\set ON_ERROR_STOP on +-- Execute as the migration owner after schema installation/import, once in +-- each database. Runtime credentials never own objects or change readiness. +SELECT kind FROM corescope_schema \gset +SELECT :'kind' = 'telemetry' AS telemetry \gset +REVOKE CREATE ON SCHEMA public FROM PUBLIC; +\if :telemetry +GRANT USAGE ON SCHEMA public TO corescope_reader, corescope_writer; +GRANT SELECT ON ALL TABLES IN SCHEMA public TO corescope_reader; +GRANT SELECT ON ALL SEQUENCES IN SCHEMA public TO corescope_reader; +GRANT SELECT ON corescope_schema TO corescope_writer; +GRANT SELECT ON packets_v TO corescope_writer; +SELECT format('GRANT SELECT,INSERT,UPDATE,DELETE ON TABLE %I.%I TO corescope_writer',schemaname,tablename) +FROM pg_tables WHERE schemaname='public' AND tablename NOT LIKE 'corescope\_%' ESCAPE '\' \gexec +GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO corescope_writer; +\else +GRANT USAGE ON SCHEMA public TO corescope_accounts, corescope_channels; +GRANT SELECT ON ALL TABLES IN SCHEMA public TO corescope_accounts; +SELECT format('GRANT SELECT,INSERT,UPDATE,DELETE ON TABLE %I.%I TO corescope_accounts',schemaname,tablename) +FROM pg_tables WHERE schemaname='public' AND tablename NOT LIKE 'corescope\_%' ESCAPE '\' AND tablename <> 'schema_version' \gexec +GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO corescope_accounts; +GRANT SELECT ON approved_channels,linked_companions,corescope_schema TO corescope_channels; +\endif diff --git a/docker/postgres-init.sh b/docker/postgres-init.sh new file mode 100755 index 000000000..5d855ed0d --- /dev/null +++ b/docker/postgres-init.sh @@ -0,0 +1,40 @@ +#!/bin/sh +# Runs once, against an empty PostgreSQL data directory. Passwords stay in the +# environment; psql reads them without putting them in process arguments. +set -eu + +for password in "${CORESCOPE_OWNER_PASSWORD:?set CORESCOPE_OWNER_PASSWORD}" \ + "${CORESCOPE_READER_PASSWORD:?set CORESCOPE_READER_PASSWORD}" \ + "${CORESCOPE_WRITER_PASSWORD:?set CORESCOPE_WRITER_PASSWORD}" \ + "${CORESCOPE_ACCOUNTS_PASSWORD:?set CORESCOPE_ACCOUNTS_PASSWORD}" \ + "${CORESCOPE_CHANNELS_PASSWORD:?set CORESCOPE_CHANNELS_PASSWORD}"; do + case "$password" in + *[!a-fA-F0-9]*|'') echo 'Database role passwords must be at least 32 hexadecimal characters; use openssl rand -hex 32.' >&2; exit 1 ;; + esac + if [ "${#password}" -lt 32 ]; then + echo 'Database role passwords must be at least 32 hexadecimal characters.' >&2 + exit 1 + fi +done +unset password + +psql -X -v ON_ERROR_STOP=1 --username "${POSTGRES_USER:-postgres}" --dbname postgres <<'SQL' +\getenv owner_password CORESCOPE_OWNER_PASSWORD +\getenv reader_password CORESCOPE_READER_PASSWORD +\getenv writer_password CORESCOPE_WRITER_PASSWORD +\getenv accounts_password CORESCOPE_ACCOUNTS_PASSWORD +\getenv channels_password CORESCOPE_CHANNELS_PASSWORD +CREATE ROLE corescope_owner LOGIN PASSWORD :'owner_password'; +CREATE ROLE corescope_reader LOGIN PASSWORD :'reader_password'; +CREATE ROLE corescope_writer LOGIN PASSWORD :'writer_password'; +CREATE ROLE corescope_accounts LOGIN PASSWORD :'accounts_password'; +CREATE ROLE corescope_channels LOGIN PASSWORD :'channels_password'; +CREATE DATABASE corescope_telemetry OWNER corescope_owner TEMPLATE template0 ENCODING 'UTF8' LOCALE 'C'; +CREATE DATABASE corescope_accounts OWNER corescope_owner TEMPLATE template0 ENCODING 'UTF8' LOCALE 'C'; +REVOKE ALL ON DATABASE corescope_telemetry FROM PUBLIC; +REVOKE ALL ON DATABASE corescope_accounts FROM PUBLIC; +GRANT CONNECT ON DATABASE corescope_telemetry TO corescope_reader, corescope_writer; +GRANT CONNECT ON DATABASE corescope_accounts TO corescope_accounts, corescope_channels; +ALTER ROLE corescope_reader SET default_transaction_read_only = on; +ALTER ROLE corescope_channels SET default_transaction_read_only = on; +SQL diff --git a/docker/postgres.compose.yml b/docker/postgres.compose.yml new file mode 100644 index 000000000..9d8d508d3 --- /dev/null +++ b/docker/postgres.compose.yml @@ -0,0 +1,40 @@ +# Shared services extended by the deployment variants. Relative bind sources +# here resolve beside this file. Runtime services receive no owner password. +services: + postgres: + image: postgres:18.6-alpine3.24 + restart: unless-stopped + stop_grace_period: 60s + shm_size: 128mb + environment: + POSTGRES_PASSWORD: ${POSTGRES_ADMIN_PASSWORD:?Set POSTGRES_ADMIN_PASSWORD} + CORESCOPE_OWNER_PASSWORD: ${CORESCOPE_OWNER_PASSWORD:?Set CORESCOPE_OWNER_PASSWORD} + CORESCOPE_READER_PASSWORD: ${CORESCOPE_READER_PASSWORD:?Set CORESCOPE_READER_PASSWORD} + CORESCOPE_WRITER_PASSWORD: ${CORESCOPE_WRITER_PASSWORD:?Set CORESCOPE_WRITER_PASSWORD} + CORESCOPE_ACCOUNTS_PASSWORD: ${CORESCOPE_ACCOUNTS_PASSWORD:?Set CORESCOPE_ACCOUNTS_PASSWORD} + CORESCOPE_CHANNELS_PASSWORD: ${CORESCOPE_CHANNELS_PASSWORD:?Set CORESCOPE_CHANNELS_PASSWORD} + volumes: + - ${POSTGRES_DATA_DIR:-../postgres-data}:/var/lib/postgresql + - ./postgres-init.sh:/docker-entrypoint-initdb.d/10-corescope.sh:ro + healthcheck: + test: ["CMD-SHELL", "pg_isready -h 127.0.0.1 -U postgres -d postgres"] + interval: 5s + timeout: 5s + retries: 20 + bootstrap: + image: ${CORESCOPE_IMAGE:-corescope:latest} + entrypoint: ["/bin/sh", "/app/postgres-bootstrap.sh"] + restart: "no" + depends_on: + postgres: + condition: service_healthy + environment: + CORESCOPE_DB_BACKEND: postgres + CORESCOPE_STATE_DIR: ${CORESCOPE_STATE_DIR:-/app/data} + CORESCOPE_DATABASE_URL: postgresql://corescope_owner:${CORESCOPE_OWNER_PASSWORD:?Set CORESCOPE_OWNER_PASSWORD}@postgres:5432/corescope_telemetry?sslmode=disable&search_path=public + CORESCOPE_USERS_OWNER_DATABASE_URL: postgresql://corescope_owner:${CORESCOPE_OWNER_PASSWORD:?Set CORESCOPE_OWNER_PASSWORD}@postgres:5432/corescope_accounts?sslmode=disable&search_path=public + CORESCOPE_READER_DATABASE_URL: postgresql://corescope_reader:${CORESCOPE_READER_PASSWORD:?Set CORESCOPE_READER_PASSWORD}@postgres:5432/corescope_telemetry?sslmode=disable&search_path=public + CORESCOPE_WRITER_DATABASE_URL: postgresql://corescope_writer:${CORESCOPE_WRITER_PASSWORD:?Set CORESCOPE_WRITER_PASSWORD}@postgres:5432/corescope_telemetry?sslmode=disable&search_path=public + CORESCOPE_USERS_DATABASE_URL: postgresql://corescope_accounts:${CORESCOPE_ACCOUNTS_PASSWORD:?Set CORESCOPE_ACCOUNTS_PASSWORD}@postgres:5432/corescope_accounts?sslmode=disable&search_path=public + CORESCOPE_APPROVED_CHANNELS_DATABASE_URL: postgresql://corescope_channels:${CORESCOPE_CHANNELS_PASSWORD:?Set CORESCOPE_CHANNELS_PASSWORD}@postgres:5432/corescope_accounts?sslmode=disable&search_path=public + CORESCOPE_OWNER_PASSWORD: ${CORESCOPE_OWNER_PASSWORD:?Set CORESCOPE_OWNER_PASSWORD} diff --git a/docker/storage.sh b/docker/storage.sh new file mode 100755 index 000000000..cf8bc61f6 --- /dev/null +++ b/docker/storage.sh @@ -0,0 +1,87 @@ +#!/bin/sh +# Packaged selection boundary. Go owns discovery, validation and conversion. +set -eu +state_dir=${CORESCOPE_STATE_DIR:-/app/data} +case "$state_dir" in /app/data|/app/data/*) ;; *) + echo 'CORESCOPE_STATE_DIR must be /app/data or a persistent subdirectory of that mounted data directory.' >&2; exit 1 ;; +esac +case "/$state_dir/" in */../*|*/./*) echo 'Use a normalized persistent state directory without dot segments.' >&2; exit 1 ;; esac +# Refuse a plain image directory: losing the selection on recreation is unsafe. +awk '$5 == "/app/data" { found=1 } END { exit !found }' /proc/self/mountinfo || { + echo '/app/data must be a persistent data mount before storage setup.' >&2; exit 1 +} +persistent_root=$(cd /app/data && pwd -P) +existing=$state_dir +while [ ! -e "$existing" ] && [ ! -L "$existing" ]; do existing=${existing%/*}; done +resolved=$(readlink -f "$existing") || { echo 'Cannot resolve the selected state directory.' >&2; exit 1; } +case "$resolved" in "$persistent_root"|"$persistent_root"/*) ;; *) + echo 'The state directory resolves outside the persistent data mount.' >&2; exit 1 ;; +esac +selection_file=$state_dir/storage-selection.json +action=${1:-start} +[ "$#" -eq 0 ] || shift +status() { /app/corescope-migrate -storage-action=status -selection-file "$selection_file"; } +mutate() { + operation=$1; shift + /app/corescope-migrate -storage-action="$operation" -selection-file "$selection_file" \ + -config-dir /app -state-dir "$state_dir/storage-jobs" -offline "$@" +} +blocked() { + job=$(printf '%s\n' "$report" | jq -r '.job_id') + echo "Storage is not ready; keep services stopped and use storage status, then resume or abort job $job." >&2 + exit 1 +} +case "$action" in + status) status ;; + field) + case "${1:-}" in has_accounts|state|backend|generation|job_id|state_dir|source_backend|target_backend|telemetry.sqlite_path|accounts.sqlite_path|telemetry.postgres.host|telemetry.postgres.port|telemetry.postgres.database|telemetry.postgres.schema|accounts.postgres.host|accounts.postgres.port|accounts.postgres.database|accounts.postgres.schema) ;; + *) echo 'Unsupported storage status field.' >&2; exit 1 ;; + esac + report=$(status) || exit 1 + case "$1" in + has_accounts) printf '%s\n' "$report" | jq -r '.accounts != null' ;; + source_backend|target_backend) printf '%s\n' "$report" | jq -r ".$1" ;; + *) printf '%s\n' "$report" | jq -er ".$1 // empty" ;; + esac + ;; + managed-postgres) + report=$(status) || exit 1 + printf '%s\n' "$report" | jq -e ' + .state == "ready" and .backend == "postgres" + and .telemetry.postgres == {host:"postgres",port:5432,database:"corescope_telemetry",schema:"public"} + and (.accounts == null or .accounts.postgres == {host:"postgres",port:5432,database:"corescope_accounts",schema:"public"}) + ' >/dev/null || { + echo 'Managed operations require the recorded local PostgreSQL telemetry/accounts targets. Use explicit owner tooling for custom endpoints; no local substitute was used.' >&2 + exit 1 + } + ;; + start) + report=$(status) || exit 1 + case $(printf '%s\n' "$report" | jq -er '.state') in + ready) ;; # Runtime opens the recorded backend; bootstrap hints cannot replace it. + unrecorded) + [ "${CORESCOPE_DB_BACKEND:-sqlite}" = sqlite ] || { + echo 'PostgreSQL needs its matching bootstrap override before application startup.' >&2; exit 1 + } + # This explicit packaged setup intent still validates all existing data. + # Neither missing selection nor a missing database is a fallback signal. + mutate setup -backend=sqlite + ;; + *) blocked ;; + esac + ;; + postgres-setup) + report=$(status) || exit 1 + case $(printf '%s\n' "$report" | jq -er '.state') in + unrecorded) mutate setup -backend=postgres ;; + ready) + [ "$(printf '%s\n' "$report" | jq -er '.backend')" = postgres ] || { + echo 'SQLite is recorded. Use the explicit offline storage switch before adding PostgreSQL.' >&2; exit 1 + } + ;; + *) blocked ;; + esac + ;; + setup|adopt|init|upgrade|switch|resume|abort) mutate "$action" "$@" ;; + *) echo 'Unsupported packaged storage action.' >&2; exit 1 ;; +esac diff --git a/docker/supervisord-go-no-caddy.conf b/docker/supervisord-go-no-caddy.conf index 3b1047443..39b53a74b 100644 --- a/docker/supervisord-go-no-caddy.conf +++ b/docker/supervisord-go-no-caddy.conf @@ -31,7 +31,7 @@ stderr_logfile_maxbytes=0 [program:corescope-server] user=%(ENV_CORESCOPE_SERVICE_USER)s -command=/app/corescope-server -config-dir /app -db /app/data/meshcore.db -public /app/public -port 3000 +command=/app/corescope-server -config-dir /app -public /app/public -port 3000 directory=/app autostart=true autorestart=true diff --git a/docker/supervisord-go-no-mosquitto-no-caddy.conf b/docker/supervisord-go-no-mosquitto-no-caddy.conf index 8c8c98598..33e1a503d 100644 --- a/docker/supervisord-go-no-mosquitto-no-caddy.conf +++ b/docker/supervisord-go-no-mosquitto-no-caddy.conf @@ -22,7 +22,7 @@ stderr_logfile_maxbytes=0 [program:corescope-server] user=%(ENV_CORESCOPE_SERVICE_USER)s -command=/app/corescope-server -config-dir /app -db /app/data/meshcore.db -public /app/public -port 3000 +command=/app/corescope-server -config-dir /app -public /app/public -port 3000 directory=/app autostart=true autorestart=true diff --git a/docker/supervisord-go-no-mosquitto.conf b/docker/supervisord-go-no-mosquitto.conf index 3c673f2ec..ceca39990 100644 --- a/docker/supervisord-go-no-mosquitto.conf +++ b/docker/supervisord-go-no-mosquitto.conf @@ -22,7 +22,7 @@ stderr_logfile_maxbytes=0 [program:corescope-server] user=%(ENV_CORESCOPE_SERVICE_USER)s -command=/app/corescope-server -config-dir /app -db /app/data/meshcore.db -public /app/public -port 3000 +command=/app/corescope-server -config-dir /app -public /app/public -port 3000 directory=/app autostart=true autorestart=true diff --git a/docker/supervisord-go.conf b/docker/supervisord-go.conf index 18a8f4d70..dcd7d9777 100644 --- a/docker/supervisord-go.conf +++ b/docker/supervisord-go.conf @@ -31,7 +31,7 @@ stderr_logfile_maxbytes=0 [program:corescope-server] user=%(ENV_CORESCOPE_SERVICE_USER)s -command=/app/corescope-server -config-dir /app -db /app/data/meshcore.db -public /app/public -port 3000 +command=/app/corescope-server -config-dir /app -public /app/public -port 3000 directory=/app autostart=true autorestart=true diff --git a/docs/backend-switching.md b/docs/backend-switching.md new file mode 100644 index 000000000..d2c701640 --- /dev/null +++ b/docs/backend-switching.md @@ -0,0 +1,105 @@ +# Choosing and switching storage + +A fresh installation defaults to SQLite. PostgreSQL is an explicit installation choice. An update retains the installed backend and both selected database targets, including an account store whose feature is disabled. + +The stable state directory contains one `storage-selection.json`. After setup, that record is authoritative for the backend and SQLite file paths. Config/env backend selectors and file paths are bootstrap inputs, so an old `db.backend: sqlite` does not undo a completed PostgreSQL switch. PostgreSQL credentials and TLS settings remain outside the record; they may rotate but must still identify the recorded host, port, database and schema. + +`accounts: null` means no account store has been initialized. A non-null target remains recorded even when account features are disabled. Runtime opens never create a missing account database. Fresh SQLite setup initializes its account store upfront; legacy adoption preserves any valid existing account file but records null when it is absent. Backend switches preserve that absence. Explicit setup can later initialize the first account target under an exclusive installation lease and atomically record it. PostgreSQL initialization requires supplied account owner and runtime credentials; ordinary setup with neither preserves null. A previously recorded missing target always fails instead of being replaced. + +Keep the runtime state-directory setting stable across updates and switches. The migration tool's `-selection-file` identifies that record. Its separate `-state-dir` chooses private recovery/job storage; it defaults to `migration` beside the selection record. + +## Inspect and initialize + +The packaged startup runs guarded setup before either service starts. Direct installations can use the same command while every service is stopped: + +```sh +corescope-migrate -storage-action=status \ + -selection-file /app/data/storage-selection.json + +corescope-migrate -storage-action=setup -offline \ + -selection-file /app/data/storage-selection.json \ + -config-dir /app +``` + +Status reads only local metadata. It needs no PostgreSQL credentials or connection and returns one JSON object. `ready`, `unrecorded`, and `pending` return exit status 0; `corrupt` returns 1. **Unrecorded is not proof of a fresh installation.** Pending or corrupt state must never trigger an empty SQLite fallback or a new installation. Status includes credential-free selected targets and the stable state directory for native backup tools. + +Setup validates an existing recorded choice, adopts a supported legacy SQLite installation, or initializes provably empty targets. `-storage-action=init` refuses existing schemas; `adopt` requires an existing supported telemetry store. A legacy SQLite writer/setup may apply supported native schema upgrades before recording its choice. Conversion copies, in contrast, leave their sources unchanged. + +Use `-config FILE` for an explicit configuration file; missing or invalid explicit files fail. Otherwise `-config-dir DIR` checks `config.json`, then `data/config.json`, continuing only when a candidate is absent. An existing unreadable or malformed primary configuration stops setup. Explicit relative telemetry/account paths resolve from the process working directory. An omitted telemetry path defaults to `config-dir/data/meshcore.db`; accounts default to `users.db` beside telemetry. On unrecorded adoption, conflicting config `dbPath` and `DB_PATH` fail unless `-sqlite-path` explicitly resolves the conflict. `-users-sqlite-path` is the corresponding account override. + +For a fresh PostgreSQL install, add `-backend=postgres`. This explicit choice overrides bootstrap config/env defaults. Setup refuses to initialize an empty PostgreSQL target while known primary SQLite data exists. A ready PostgreSQL installation with retained SQLite files needs a matching completed-import record or explicit adoption; switching live data uses the verified procedure below. + +## Prepare an offline switch + +Stop **every** writer and reader, including other hosts, scheduled jobs, and administrative database clients. Keep both original databases and their native backups. The tool takes local installation leases and native source locks; a local lease is not a distributed deployment coordinator. + +Prepare fresh, unselected destination targets. Existing SQLite files are never overwritten, and existing PostgreSQL schema/data is not an import destination. Telemetry and accounts remain separate databases/files. Conversion includes every recorded account store even when account features are disabled. + +Supply PostgreSQL credentials through the private environment, not process arguments or logs: + +| Variable | New storage actions | +|---|---| +| `CORESCOPE_DATABASE_URL` | Telemetry migration owner for the PostgreSQL side | +| `CORESCOPE_USERS_OWNER_DATABASE_URL` | Account migration owner for the PostgreSQL side | +| `CORESCOPE_READER_DATABASE_URL` | Restricted telemetry reader | +| `CORESCOPE_WRITER_DATABASE_URL` | Restricted telemetry writer | +| `CORESCOPE_USERS_DATABASE_URL` | Restricted account runtime writer | +| `CORESCOPE_APPROVED_CHANNELS_DATABASE_URL` | Optional restricted approved-channel projection reader | + +The owner creates/imports destination schemas, applies the existing narrow runtime grant policy, then verifies actual runtime access before selection changes. Runtime roles never gain schema ownership or DML on readiness/import metadata; the account writer cannot modify `schema_version`. Credentials are not stored in selection records, job manifests or command output. Role URL target overrides through query parameters or service settings are not supported. + +SQLite to PostgreSQL: + +```sh +corescope-migrate -storage-action=switch -backend=postgres -offline \ + -selection-file /app/data/storage-selection.json \ + -state-dir /app/data/migration +``` + +PostgreSQL to new SQLite files: + +```sh +corescope-migrate -storage-action=switch -backend=sqlite -offline \ + -selection-file /app/data/storage-selection.json \ + -sqlite-path /app/data/returned/meshcore.db \ + -users-sqlite-path /app/data/returned/users.db \ + -state-dir /app/data/migration +``` + +For either direction, the owner URLs address the PostgreSQL side: the destination for the first command, the source for the second. Do not pre-bootstrap an import destination. Original invocation without `-storage-action` retains its prior PostgreSQL bootstrap/forward-import behavior, including its legacy account-owner environment convention. + +## What verification covers + +Native SQLite recovery snapshots retain WAL contents and hidden rowids; PostgreSQL recovery is a private `pg_dump` custom archive. Copying uses bounded batches and a 64 MiB per-row safety limit. Every application table is compared by complete row count and ordered canonical digest, including NULL/empty distinctions, raw text/JSON/timestamps and representable fractional values. Unsupported values or custom source schemas fail before selection instead of being coerced or discarded. + +Identity verification includes retained IDs, deleted high-water marks, historical references and backfill cursors. PostgreSQL `last_value` and `is_called` are retained in the private manifest; SQLite non-reusing allocators are seeded above every retained/reference maximum. No dummy rows are inserted. Account SQLite schema 6 uses explicit `mail_events.id`; PostgreSQL keeps its native account schema version 5. Those schema version values describe the engine, not copied account data. + +Reverse conversion adds required SQLite physical migration markers while preserving every source ledger entry. The private reverse manifest lists those additions. It builds the native observer/time index offline even if the retained async ledger already says the task is done. PostgreSQL-specific readiness/import bookkeeping is retained in its native recovery archive, not copied into application SQLite tables. + +SQLite reverse targets contain `corescope_reverse_progress` until complete data verification. Native open/setup/adoption paths reject that marker, including through another state directory. Runtime schema readiness alone is never treated as proof that an import completed. The PostgreSQL source's managed table/column/constraint/index/view layout is checked against a versioned native manifest; custom extensions require explicit review. + +Only after both stores verify and destination runtime access succeeds does one atomic replacement select the new backend and both targets together. Original files/databases and recovery copies remain available. This is a planned-downtime switch, not live replication or an automatic rollback service. + +## Interruption and recovery + +Inspect status to obtain the pending `job_id`. Keep services stopped and retain the same selection path, recovery directory and destination targets. Supply credentials again, then resume: + +```sh +corescope-migrate -storage-action=resume -offline \ + -selection-file /app/data/storage-selection.json \ + -state-dir /app/data/migration -job-id JOB_ID +``` + +Resume rechecks the exact source, sequences, native recovery copy, destination and committed copy progress. A changed source or target is refused. An interruption before a complete preparation manifest may require preserving the failed job and starting a new job with fresh targets after abort; partial artifacts are not silently reused. + +First account initialization uses the same journal, with identical source and target backends. Resume can finish an existing ready account target, without PostgreSQL credentials for SQLite. It never recreates a missing target or guesses that an empty target is fresh. If initialization stopped before readiness, preserve/inspect it, abort the uncommitted metadata update, then rerun explicit setup. A journal with no staged destination permits only metadata-only abort; no database connection is needed. + +Before selection replacement, abort can reopen the original selected installation: + +```sh +corescope-migrate -storage-action=abort -offline \ + -selection-file /app/data/storage-selection.json \ + -state-dir /app/data/migration -job-id JOB_ID +``` + +Abort preserves staged databases and recovery files. After selection replacement, abort refuses to restore an old selection; resume verifies and finishes that job instead. Inspect status after a cleanup error, since the verified new selection may already be committed. Returning later to the previous backend is another explicit verified conversion into fresh targets, preserving data written since the first switch. diff --git a/docs/deployment.md b/docs/deployment.md index 0eb8c2178..fdb5c249e 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -18,70 +18,39 @@ Comprehensive guide to deploying and operating CoreScope. For a quick start, see ## System Requirements -| Resource | Minimum | Recommended | -|----------|---------|-------------| -| RAM | 256 MB | 512 MB+ | -| Disk | 500 MB (image + DB) | 2 GB+ for long-term data | -| CPU | 1 core | 2+ cores | -| Architecture | `linux/amd64`, `linux/arm64` | — | -| Docker | 20.10+ | Latest stable | - -CoreScope runs well on Raspberry Pi 4/5 (ARM64). The Go server uses ~300 MB RAM for 56K+ packets. +The supplied images support `linux/amd64` and `linux/arm64`. Use Docker with the Compose plugin. SQLite is the default; PostgreSQL18.6 is an optional separate service. Budget for the application and database together: earlier SQLite memory figures do not size a PostgreSQL installation. Measure the retained dataset, packet cache, database indexes/WAL and backup storage before choosing capacity. ---- +For migration, retain the original data plus recovery and normalized snapshots while PostgreSQL builds its own data and indexes. Required disk space and downtime must be measured on a representative copy. See [offline upgrade and recovery](postgresql-upgrade.md). ## Docker Deployment -### Quick Start (one command) - -```bash -docker run -d --name corescope \ - -p 80:80 \ - -v corescope-data:/app/data \ - ghcr.io/kpa-clawbot/corescope:latest -``` +### Complete Compose checkout -Open `http://localhost` — you'll see an empty dashboard ready to receive packets. +Use [DEPLOY.md](../DEPLOY.md#quick-start) to clone and pin the complete repository, populate the private `.env` and select a matching reviewed image. Keep the matching checkout, including optional PostgreSQL overrides and their initialization scripts. -No `config.json` is required. The server starts with sensible defaults: -- HTTP on port 3000 (Caddy proxies port 80 → 3000 internally) -- Internal Mosquitto MQTT broker on port 1883 -- Ingestor connects to `mqtt://localhost:1883` automatically -- SQLite database at `/app/data/meshcore.db` - -### Full `docker run` Reference (recommended) +```sh +docker compose -f docker-compose.example.yml up -d +docker compose -f docker-compose.example.yml ps +``` -The bare `docker run` command is the primary deployment method. One image, documented parameters — run it however you want. +The default app validates/adopts existing SQLite storage or initializes a provably fresh install before starting. The optional `docker-compose.example.postgres.yml` override adds PostgreSQL and owner bootstrap. An installed selection is authoritative; a missing connection or an interrupted conversion never permits SQLite fallback. -```bash -docker run -d --name corescope \ - --restart=unless-stopped \ - -p 80:80 -p 443:443 -p 1883:1883 \ - -e DISABLE_MOSQUITTO=false \ - -e DISABLE_CADDY=false \ - -v /your/data:/app/data \ - -v /your/Caddyfile:/etc/caddy/Caddyfile:ro \ - -v /your/caddy-data:/data/caddy \ - ghcr.io/kpa-clawbot/corescope:latest -``` +The application container runs the server and ingestor, optional Mosquitto, and optional Caddy. When selected, PostgreSQL runs separately. Its runtime services receive restricted roles; the owner belongs only to setup/conversion. -#### Parameters +| Setting | Purpose | +|---|---| +| `HTTP_PORT` | Example deployment's host HTTP port (default 80) | +| `DATA_DIR` | Application state: config, theme, queues, statistics and account backups | +| `POSTGRES_DATA_DIR` | Separate PostgreSQL storage; never copy live files as a logical backup | +| `CORESCOPE_IMAGE` | Reviewed application image tag or digest matching the checkout | +| `DISABLE_MOSQUITTO` | Use an external broker when true | +| `DISABLE_CADDY` | Use an external reverse proxy when true | +| `RUN_AS_UID` / `RUN_AS_GID` | Opt-in service account; set in the container environment, not the data-directory `.env` | +| `CORESCOPE_*_PASSWORD` | Distinct owner and runtime credentials; only for PostgreSQL, merge `.env.postgres.example` | -| Parameter | Required | Description | -|-----------|----------|-------------| -| `-p 80:80` | Yes | HTTP web UI | -| `-p 443:443` | No | HTTPS (only if using built-in Caddy with a domain) | -| `-p 1883:1883` | No | MQTT broker (expose if external gateways connect directly) | -| `-v /your/data:/app/data` | Yes | Persistent data: SQLite DB, config.json, theme.json | -| `-v /your/Caddyfile:/etc/caddy/Caddyfile:ro` | No | Custom Caddyfile for HTTPS | -| `-v /your/caddy-data:/data/caddy` | No | Caddy TLS certificate storage | -| `-e DISABLE_MOSQUITTO=true` | No | Skip the internal Mosquitto broker (use your own) | -| `-e DISABLE_CADDY=true` | No | Skip the built-in Caddy reverse proxy | -| `-e MQTT_BROKER=mqtt://host:1883` | No | Override MQTT broker URL | -| `-e RUN_AS_UID=1000` | No | Run the Go services as this uid instead of root — see [Running the Go services unprivileged](#running-the-go-services-unprivileged) | -| `-e RUN_AS_GID=1000` | No | Group for `RUN_AS_UID` (defaults to it) | +The production and staging variants use their own directory/port variables listed in `.env.example`. `manage.sh` operates those variants; `./manage.sh start --with-staging` clones telemetry only into an empty staging database and keeps staging accounts independent. -#### `/app/data/.env` convenience file +### `/app/data/.env` convenience file Instead of passing `-e` flags, you can drop a `.env` file in your data volume: @@ -107,55 +76,16 @@ directory the Go services can write once they run unprivileged: reports (`docker exec stat -c '%u' /app/data/.env`) and give the host file whichever owner id maps to container uid 0. -### Docker Compose (legacy alternative) - -Docker Compose files are maintained for backward compatibility but are no longer the recommended approach. - -```bash -curl -sL https://raw.githubusercontent.com/Kpa-clawbot/CoreScope/master/docker-compose.example.yml \ - -o docker-compose.yml -docker compose up -d -``` - -#### Compose environment variables - -| Variable | Default | Description | -|----------|---------|-------------| -| `HTTP_PORT` | `80` | Host port for the web UI | -| `DATA_DIR` | `./data` | Host path for persistent data | -| `DISABLE_MOSQUITTO` | `false` | Set `true` to use an external MQTT broker | -| `DISABLE_CADDY` | `false` | Set `true` to skip the built-in Caddy proxy | - -### manage.sh (legacy alternative) - -The `manage.sh` wrapper script provides a setup wizard and convenience commands. It uses Docker Compose internally. See [DEPLOY.md](../DEPLOY.md) for usage. New deployments should prefer bare `docker run`. - -### Image tags - -| Tag | Use case | -|-----|----------| -| `v3.4.1` | Pinned release — recommended for production | -| `v3.4` | Latest patch in the v3.4.x series | -| `v3` | Latest minor+patch in v3.x | -| `latest` | Latest release tag | -| `edge` | Built from master on every push — unstable | - ### Updating -```bash -docker compose pull -docker compose up -d -``` - -For `docker run` users: +Preserve a native backup and private configuration, pin the new checkout and matching image, then use the same Compose variant: -```bash -docker pull ghcr.io/kpa-clawbot/corescope:latest -docker stop corescope && docker rm corescope -docker run -d --name corescope ... # same flags as before +```sh +docker compose -f docker-compose.example.yml pull +docker compose -f docker-compose.example.yml up -d ``` -Data is preserved in the volume — updates are non-destructive. +A backend change in either direction requires the [offline storage switch](storage.md), not only an image update. Ordinary updates keep the recorded choice. Keep the previous application and recovery files until validation is complete. ### Running the Go services unprivileged @@ -232,6 +162,11 @@ Notes: - A fresh install is root-owned and so starts as root, like any other. Set `RUN_AS_UID` from the first run to avoid a hand-over later. +Packaged SQLite validation/setup also runs as the resolved service account, so +a fresh database and its private storage-selection record remain readable by +the services. PostgreSQL owner bootstrap still runs separately; the application +only validates the recorded selection with runtime credentials. + The uid selection lives in `docker/select-service-user.sh`, with its branches covered by `docker/test-select-service-user.sh`. @@ -241,9 +176,11 @@ covered by `docker/test-select-service-user.sh`. CoreScope uses a layered configuration system (highest priority wins): -1. **Environment variables** — `MQTT_BROKER`, `DB_PATH`, etc. +1. **Environment variables** — `MQTT_BROKER`, `CORESCOPE_READER_DATABASE_URL`, etc. 2. **`/app/data/config.json`** — full config file (volume-mounted) -3. **Built-in defaults** — work out of the box with no config +3. **Built-in defaults** — SQLite for a provably fresh installation; PostgreSQL requires its role URLs + +After setup, `storage-selection.json` overrides bootstrap backend/path hints. Credentials and TLS remain private config/environment settings. Keep the same shared state directory on restart and update. ### Environment variable overrides @@ -251,21 +188,17 @@ CoreScope uses a layered configuration system (highest priority wins): |----------|---------|-------------| | `MQTT_BROKER` | `mqtt://localhost:1883` | MQTT broker URL (overrides config file) | | `MQTT_TOPIC` | `meshcore/#` | MQTT topic subscription pattern | -| `DB_PATH` | `data/meshcore.db` | SQLite database path | +| `CORESCOPE_READER_DATABASE_URL` | Required for PostgreSQL | Restricted telemetry reader URL | +| `CORESCOPE_WRITER_DATABASE_URL` | Required for PostgreSQL | Restricted telemetry writer URL | +| `CORESCOPE_USERS_DATABASE_URL` | Required when accounts are enabled | Separate account database writer URL | +| `CORESCOPE_APPROVED_CHANNELS_DATABASE_URL` | Required when proposals are enabled | Restricted approved-channel view reader URL | +| `CORESCOPE_STATE_DIR` | `data` for native binaries | Shared filesystem state, separate from database URLs | | `DISABLE_MOSQUITTO` | `false` | Skip the internal Mosquitto broker | | `DISABLE_CADDY` | `false` | Skip the built-in Caddy reverse proxy | ### config.json -For advanced configuration, create a `config.json` and mount it at `/app/data/config.json`: - -```bash -docker run -d --name corescope \ - -p 80:80 \ - -v corescope-data:/app/data \ - -v ./config.json:/app/data/config.json:ro \ - ghcr.io/kpa-clawbot/corescope:latest -``` +Place `config.json` in the mounted state directory (`DATA_DIR` in the example deployment). Use private environment configuration for database passwords; do not publish rendered Compose configuration or URLs. Supported keys include `db.backend`, SQLite `dbPath`/`DB_PATH` and `userManagement.dbPath`, optional PostgreSQL role URLs, and `stateDir`. Explicit relative SQLite paths use the process working directory; containers use `/app`. Keep container databases/state inside `/app/data` so the bind mount persists them. Set `CORESCOPE_STATE_DIR=/app/data/` in the private `.env` when using a custom container state directory; every service shares it. See `config.example.json` in the repository for all available options including: - MQTT sources (multiple brokers) @@ -329,13 +262,7 @@ CoreScope receives MeshCore packets via MQTT. The container ships with an intern The built-in Mosquitto broker listens on port 1883 inside the container. Point your MeshCore gateways at it: -```bash -# Expose MQTT port for external gateways -docker run -d --name corescope \ - -p 80:80 -p 1883:1883 \ - -v corescope-data:/app/data \ - ghcr.io/kpa-clawbot/corescope:latest -``` +Publish `1883:1883` on the application service in your Compose override when external gateways need to reach the built-in broker. Keep `DISABLE_MOSQUITTO=false`. The database service must remain on the internal network. ### External broker @@ -343,12 +270,12 @@ To use your own MQTT broker (Mosquitto, EMQX, HiveMQ, etc.): 1. Disable the internal broker: ```bash - -e DISABLE_MOSQUITTO=true + DISABLE_MOSQUITTO=true ``` 2. Point the ingestor at your broker: ```bash - -e MQTT_BROKER=mqtt://your-broker:1883 + MQTT_BROKER=mqtt://your-broker:1883 ``` Or via `config.json`: @@ -433,15 +360,7 @@ The container includes Caddy for automatic Let's Encrypt certificates: } ``` -2. Mount it and expose TLS ports: - ```bash - docker run -d --name corescope \ - -p 80:80 -p 443:443 \ - -v corescope-data:/app/data \ - -v caddy-certs:/data/caddy \ - -v ./Caddyfile:/etc/caddy/Caddyfile:ro \ - ghcr.io/kpa-clawbot/corescope:latest - ``` +2. Set `DISABLE_CADDY=false` for the application service, publish ports 80 and 443, mount the Caddyfile at `/etc/caddy/Caddyfile:ro` and persist certificate storage at `/data/caddy`. Make these changes in the same Compose deployment so the PostgreSQL dependency and role configuration remain intact. Caddy handles certificate issuance and renewal automatically. @@ -637,41 +556,19 @@ docker stats corescope ## Backup & Restore -### Backup +Use native SQLite snapshots for SQLite or custom-format PostgreSQL archives for PostgreSQL. The authenticated `GET /api/backup` and admin `GET /api/admin/users-backup` endpoints download `.dump` files. The server needs PostgreSQL 18 `pg_dump` on `PATH`; the supplied image includes it. -All persistent data lives in `/app/data`. The critical file is the SQLite database: - -```bash -# Copy from the Docker volume -docker cp corescope:/app/data/meshcore.db ./backup-$(date +%Y%m%d).db +For the production variant managed by `manage.sh`: -# Or if using a bind mount -cp ./data/meshcore.db ./backup-$(date +%Y%m%d).db +```sh +./manage.sh backup ./backups/instance ``` -Optional files to back up: -- `config.json` — custom configuration -- `theme.json` — custom theme/branding - -### Restore - -```bash -# Stop the container -docker stop corescope +This saves `telemetry.dump`, `accounts.dump` and available config/theme/Caddy files. Each archive is consistent, but two sequential dumps are not a shared cross-database snapshot; stop both writers for a coordinated pair. Preserve credentials, queues/state and TLS material separately. Encrypt private backups off-host. -# Replace the database -docker cp ./backup.db corescope:/app/data/meshcore.db - -# Restart -docker start corescope -``` +`./manage.sh restore ` requires the application stopped and fresh/empty native targets. SQLite restores stage and validate the full state bundle in a new empty state directory. PostgreSQL restores require both databases empty. It reapplies runtime grants and leaves the application stopped for validation. It never replaces a nonempty PostgreSQL database or overwrites an existing SQLite state directory. Use a fresh PostgreSQL storage location while retaining the previous cluster for recovery. -### Automated backups - -```bash -# cron: daily backup at 3 AM, keep 7 days -0 3 * * * docker cp corescope:/app/data/meshcore.db /backups/corescope-$(date +\%Y\%m\%d).db && find /backups -name "corescope-*.db" -mtime +7 -delete -``` +Account backups contain password hashes, addresses, sessions and tokens. Restoration can revive deleted accounts or revoked credentials. Review the [account recovery procedure](user-guide/accounts.md#backups) before reopening the service. See [native backup and cutover boundaries](postgresql-upgrade.md#native-backups-and-restores) for downloaded archives and rollback. --- @@ -720,16 +617,16 @@ The in-memory packet store grows with retained packets. Configure retention limi `packetStore.maxMemoryMB` bounds the store **and the caches that belong to it** — the decoded-packet cache and per-packet index entries, not just the stored rows. It is enforced in two places: the startup load stops at the budget, and the store evicts oldest-first when it exceeds it. Leaving it unset means no limit. Actual usage is on `/api/perf` as `packetStore.trackedMB`, next to `maxMB`. -### Database locked errors +### Database readiness or privilege errors -SQLite doesn't support concurrent writers well. Ensure only one CoreScope instance accesses the database file. If running multiple containers, each needs its own database. +Confirm the PostgreSQL service is healthy, the owner bootstrap/import finished, and each process uses its intended runtime role. An incomplete import stays closed until resume verifies every requested store. Do not grant owner privileges to runtime processes or manually flip readiness markers. The ingestor owns telemetry writes and retention; multiple active ingestors targeting one store are refused. ### Container unhealthy Check logs: `docker compose logs --tail 50`. Common causes: - Port 3000 already in use inside the container -- Database file permissions (must be writable by the container user) -- Corrupted database — restore from backup +- Missing/wrong runtime credentials, privilege grants or schema readiness +- Database failure — investigate logs and restore a verified archive into a fresh target ### ARM / Raspberry Pi issues diff --git a/docs/performance/optional-backend-b5-2026-10-09.md b/docs/performance/optional-backend-b5-2026-10-09.md new file mode 100644 index 000000000..28bc9c542 --- /dev/null +++ b/docs/performance/optional-backend-b5-2026-10-09.md @@ -0,0 +1,91 @@ +# Optional backend: two five-pair B experiments, 9 October 2026 + +Both experiments passed the benchmark data and coverage gates for `c03a2585eef3819cf06640911b2f7308d99339db`. They do **not** support a general speed claim. The measured c03 SQLite candidate has a material `/api/observers` regression, with a later correction separately evidenced by the native control below. Its Linux numbers remain unchanged. Candidate PostgreSQL greatly shortens this retention workload and its associated ingest queue, but costs more CPU, memory and storage and slows many reads. + +**Exact scope.** Baseline: `3e21b179ecccae005aa5f6f45bc095353c94b343`; measured application candidate: `c03a2585eef3819cf06640911b2f7308d99339db`. These are separate same-host paired studies, with different runners and HTTP mixes. Never rank candidate PostgreSQL against candidate SQLite by combining their numbers. The SQLite mix adds paths, neighbors and health requests and changes endpoint weights. Later test fixes and the observer correction were not measured by these Linux runs. + +| Study | Successful benchmark job / sanitized artifact | Harness | +|---|---|---| +| Native SQLite before/after | [Run 37985636346, job 114006837291](https://github.com/Kpa-clawbot/CoreScope/actions/runs/37985636346/job/114006837291), artifact `11645874366` | `4de419413c0a6fd93ecc2361cd704cd2a59e673e`; SHA-256 `f06f836de930a16d428b7c644231d8460e1a7ab736eed855e5d3eae7f2e23e96` | +| Upstream SQLite versus candidate PostgreSQL | [Run 37985643046, job 114006882924](https://github.com/Kpa-clawbot/CoreScope/actions/runs/37985643046/job/114006882924), artifact `11647406303` | Harness from `c03a2585eef3819cf06640911b2f7308d99339db`; SHA-256 `7c234ba61c93351688d7068c9d8405f7d2906558d797854738eb0ff059f35d84` | + +The PostgreSQL benchmark job succeeded independently of the workflow’s unrelated test-fixture failures. Smoke/pilot runs and previous failed studies are excluded. + +**Method and acceptance.** Each study has five alternating pairs, Linux amd64 / Go 1.27.2, enforced 3 CPU / 6 GiB for the whole measured stack, a 2 GiB packet-store limit and 3 GiB server memory setting. PostgreSQL is 18.6. The B corpus contains 128,000 transmissions and 2,048,000 observations over eight days. Every full and hot startup loaded the complete retained 112,000 / 1,792,000 records, with zero startup evictions. Accounts are disabled. + +Each of the ten legs per study completed all 12,000 offered ingest events and 4,800 HTTP requests: 50 ingest/s and 20 HTTP/s, 60 s warmup plus 180 s measured. Independently reconstructed schedules, warmup flags, replay identities, full WebSocket receipt coverage, errors and drops all passed. Post-replay validation recorded 118,000 transmissions / 1,802,200 observations in every leg after deleting 16,000 / 256,000; the empty second cleanup deleted zero. All 18 table digests match within each pair: full initial data and logical post-replay data. Post-replay comparison excludes only the declared operational receipt/maintenance timestamps listed in the JSON. SQLite comparison normalizes only the exact, verified `observers_identity_autoincrement_v1` migration marker; observer rowids and all other logical application values remain checked. + +All 1,000 paired isolated SQL-control responses per study matched in bytes and SHA-256. Static mixed-request controls (metrics, rx-coverage, packet detail and unfiltered RF) also matched. Mixed live response bodies are not universally byte-identical: public artifacts retain hashes, sizes and structural/count checks, not bodies sufficient to explain every difference. This is not proof of universal API response equality. + +Tables show medians of per-leg quantiles and the median of five candidate/baseline ratios, with their observed range; these are not confidence intervals or ratios of pooled requests. Lower is better for times and resource use. Service time excludes dispatch/replay queue wait; scheduled-to-complete includes it. HTTP service includes response reading and validation. p99 is omitted below 1,000 samples per leg; the 900 late-ingest events, individual HTTP classes and 100-per-class SQL controls therefore have no p99 claim. + +**SQLite before/after.** All five observer p95 comparisons regress (180 measured requests per leg). Its roughly 53× increase is the principal performance blocker for this measured candidate. Ingest scheduled latency improves, but duplicate service p99 increases and several reads regress. + +| Metric (ms) | Upstream SQLite | Candidate SQLite | Median paired ratio [range] | +|---|---:|---:|---:| +| Observers HTTP p95 | 3.653 | 194.9 | 53.36 [50.05, 54.08] | +| Additional ingest scheduled p95 | 2203 | 1583 | 0.7466 [0.6328, 0.9491] | +| Additional ingest scheduled p99 | 8066 | 6803 | 0.8422 [0.7889, 0.9025] | +| Additional ingest service p95 | 1.132 | 0.7674 | 0.6956 [0.6357, 0.7325] | +| New ingest service p95 | 1.572 | 1.174 | 0.7446 [0.7076, 0.763] | +| Duplicate ingest service p99 | 1.392 | 2.067 | 1.704 [0.5793, 2.745] | +| Health HTTP p95 | 73.54 | 1.288 | 0.01737 [0.003842, 0.939] | +| Paths HTTP p95 | 46.28 | 47.79 | 1.03 [0.9701, 1.038] | +| Reach HTTP p95 | 0.7175 | 0.8227 | 1.098 [0.947, 1.203] | +| Topology HTTP p95 | 37.84 | 37.72 | 1.102 [0.7627, 1.387] | +| Uncached channel SQL p95 | 90.02 | 96.89 | 1.075 [1.02, 1.082] | +| WebSocket signed return-to-receipt p99 | 1976 | 1317 | 0.5422 [0.4464, 0.9699] | + +Cleanup took 11.231 → 11.091 s (paired ratio 0.9949); it remains a substantial queue interruption. Full-start ready took 28.268 → 27.848 s (0.9891), but **candidate storage adoption separately took 42.03–42.59 s, median 42.50 s, before startup timing began**. The baseline needed no adoption. Neither leg drained ingest or HTTP work past the scheduled window’s end. + +| Metric (unit in row) | Upstream SQLite | Candidate SQLite | Median paired ratio [range] | +|---|---:|---:|---:| +| CPU consumed (s) | 54.68 | 53.52 | 0.9787 [0.9597, 1.023] | +| Cgroup sampled peak (GiB) | 3.988 | 3.944 | 0.9944 [0.9891, 1.002] | +| PSS sampled peak (GiB) | 2.897 | 2.87 | 0.9947 [0.9904, 1.005] | +| Selected DB/cluster + WAL peak (GiB) | 1.071 | 1.069 | 0.9987 [0.9896, 1] | +| WAL sampled peak (MiB) | 38.7 | 29.6 | 0.9496 [0.6983, 1.003] | +| Device 8:0 write delta (GiB) | 2.31 | 2.24 | 0.9694 [0.9652, 0.9812] | + +Memory and CPU differences are small: the normalized average-CPU paired ratio is 0.9837 and its range crosses 1. All SQLite measured memory-limit/OOM deltas were zero; candidate whole-leg kernel peaks were 4.045–4.120 GiB. This is not a large retained-memory win. + +**PostgreSQL comparison.** These ratios use the upstream SQLite leg of the PostgreSQL study, not the candidate SQLite experiment above. Cleanup fell from 11.268 to 0.974 s (paired ratio 0.08708). Scheduled ingest tails fell sharply while additional and duplicate service p95 increased, so the queue-inclusive result must not be presented as a universal per-write speedup. + +| Metric (ms) | Upstream SQLite | Candidate PostgreSQL | Median paired ratio [range] | +|---|---:|---:|---:| +| Additional ingest scheduled p95 | 2262 | 3.044 | 0.001346 [0.001186, 0.001457] | +| Additional ingest scheduled p99 | 8075 | 5.57 | 0.0006807 [0.0006152, 0.0007697] | +| Additional ingest service p95 | 1.627 | 2.239 | 1.373 [0.956, 1.513] | +| Duplicate ingest service p95 | 1.049 | 1.874 | 1.713 [1.626, 1.94] | +| New ingest service p95 | 2.689 | 2.238 | 0.8562 [0.6949, 0.9195] | +| New ingest service p99 | 5.492 | 4.041 | 0.7722 [0.6216, 0.8537] | +| Observers HTTP p95 | 3.762 | 4.06 | 1.079 [1.071, 1.154] | +| Nodes (50) HTTP p95 | 5.241 | 10.55 | 1.784 [1.656, 2.156] | +| Messages (offset 0) HTTP p95 | 69.11 | 114.9 | 1.654 [1.626, 2.519] | +| Packet detail, memory HTTP p95 | 1.644 | 4.013 | 2.407 [2.298, 2.553] | +| Uncached channel SQL p95 | 95.29 | 122.6 | 1.278 [1.219, 1.54] | +| WebSocket signed return-to-receipt p99 | 2035 | 998 | 0.5318 [0.4524, 0.5928] | + +Full-start ready was 28.965 → 27.712 s (0.9701), while hot-start first-ready was 9.768 → 11.271 s (1.1427). The separately measured import, grants and validation stage took a median **148.887 s**; neither startup number includes it. There was no work draining past the scheduled measured endpoint. + +| Metric (unit in row) | Upstream SQLite | Candidate PostgreSQL | Median paired ratio [range] | +|---|---:|---:|---:| +| CPU consumed (s) | 45.02 | 59.23 | 1.329 [1.31, 1.519] | +| Average CPU cores | 0.2506 | 0.3311 | 1.323 [1.308, 1.525] | +| Cgroup sampled peak (GiB) | 3.954 | 5.536 | 1.422 [1.371, 1.527] | +| PSS sampled peak (GiB) | 2.868 | 3.194 | 1.11 [1.088, 1.148] | +| Selected DB/cluster + WAL peak (GiB) | 1.072 | 2.273 | 2.119 [2.099, 2.146] | +| WAL sampled peak (MiB) | 30.71 | 1024 | 33.35 [25.54, 40.79] | +| Device 8:0 write delta (GiB) | 2.315 | 0.3907 | 0.1702 [0.1619, 0.2152] | + +**PostgreSQL memory pressure is a sizing constraint.** All five descriptor-local whole-leg kernel peaks reached the 6 GiB limit, with small kernel-accounting overshoots of 4–116 KiB. Measured-window `memory.events max` increments were `[338, 0, 522, 0, 0]`; OOM and OOM-kill increments were zero. The lowest sampled headroom was only 20 KiB. An absence of killed processes does not establish spare capacity. Cgroup memory includes file cache. PSS apportions shared pages; summed RSS (paired 1.853× here) double-counts shared PostgreSQL mappings and must not be interpreted as unique resident memory. + +PostgreSQL retained **1 GiB of WAL throughout every measured window after migration**. Its storage peak therefore includes that carryover and is not all newly generated runtime data. PostgreSQL storage counts the private cluster; SQLite counts its database and sidecars. Conversion source/recovery copies are additional. Lower windowed write accounting does not measure later checkpoints or steady-state write amplification. Minimum sampled free disk was 72.11 GiB in the PostgreSQL legs and 75.81 GiB in candidate SQLite’s separate study; these well-provisioned runners do not establish minimum operator disk sizing. + +**Limits.** Resource deltas use the first and last sample inside the exact 180 s measured window, not startup or warmup; sampled spans are about 178–180 s. Gauge peaks can miss subsecond spikes; whole-leg kernel peaks are separately reset/read through the same retained file descriptor and include setup/startup/validation. `memory.events` raw final counters are cumulative; only explicit differences are per-window events. I/O is device-level cgroup accounting, not a physical-device write-amplification measurement. Query plans were collected after retention. WebSocket timing is signed receipt minus ingest-return; no negative measured samples occurred here. OS cache is warm, tests are synthetic, accounts are disabled, and neither capacity, OS-cold behavior, a long soak, Windows performance, nor a generally superior backend follows. + +All per-run metrics, paired distributions, body-hash comparison counts, settings, source/binary identities and raw-file SHA-256 values are in the [archived JSON at commit d132e582](https://github.com/n30nex/CoreScope/blob/d132e582e6260897cde43c8041f104ba53bde0ff/docs/performance/optional-backend-b5-2026-10-09.json). Production hashes and both harness hashes were recomputed from immutable Git objects; public database digest attestations were checked without reopening private payload databases. + +**Separate observer follow-up control.** A separate Windows amd64 / Go 1.27.2 native SQLite response-build control used 2,048,000 observations, 128,000 transmissions, 128 observers and eight days, WAL/FULL, a 2,000 KiB reader page cache, GOMAXPROCS=2 and GOMEMLIMIT=512MiB. Each stage timed 50 uncached buildObserversDefaultResponse calls with warm OS/SQLite page caches. Before correction the upgraded layout had p50/p95 169.0/195.7 ms versus its same-run legacy control 19.46/20.29 ms. After the writer-side observer-statistics correction, upgraded p50/p95 were 19.59/20.42 ms versus same-run legacy 19.68/21.21 ms. Exact recent counts and full typed observer responses matched, excluding server_time. This supports the local planner repair; it is not full HTTP service, concurrent replay, PostgreSQL, or a replacement Linux B5 result. + +Source/evidence: base `c6fee4e8872b6c0caf7af0aadc3013d48b704882`; before tree `0041649915878be1782e7d620a9336207493b83d`; corrected tree `1d9a08258b1350e512528075ca70ad988d569af7`. Frozen logs `observers-native-b-before.log` (SHA-256 `859686c1b225c5044e7ffa3175a8e7efc35576584cecd43f5b4c0601ca58cc4b`) and `observers-native-b-after.log` (`bb362f9d4544bef9bf31c0a2682346ef9dd1808f51d2d950c0e99dc9a5d0e50f`) were hash-checked and read. diff --git a/docs/performance/postgresql-ce023e0-details.md b/docs/performance/postgresql-ce023e0-details.md new file mode 100644 index 000000000..d1b7b234f --- /dev/null +++ b/docs/performance/postgresql-ce023e0-details.md @@ -0,0 +1,273 @@ +# PostgreSQL ce023: complete paired results + +[Summary](../postgresql-performance.md) · [Complete JSON at commit d132e582](https://github.com/n30nex/CoreScope/blob/d132e582e6260897cde43c8041f104ba53bde0ff/docs/performance/postgresql-ce023e0-details.json) · [Run](https://github.com/Kpa-clawbot/CoreScope/actions/runs/37914893623) · [Raw public artifact](https://github.com/Kpa-clawbot/CoreScope/actions/runs/37914893623/artifacts/11613951167) + +All generated tables below are preserved from the validated final analysis. +The artifact-relative source directory is `corescope-bench-37914893623-1/public`. +The JSON's `analysis_reference` identifies the analyzer's source-review anchor +(`c315d7d`); its `provenance.candidate_sha` identifies the measured candidate +(`ce023e0`). The review checked 189 input hashes; the raw public bundle contains +234 files. No host-local paths or private payload databases are published here. + +The [intermediate c315 run](https://github.com/Kpa-clawbot/CoreScope/actions/runs/37901327267) +was diagnostic evidence for the RX generic-plan correction. It ran on another +Actions runner; no quantified cross-run speedup is claimed. All paired ratios +below compare ce023 with the unchanged upstream on the final run's same host. + +Candidate: ce023e0d623116fe66987761d06016590f98fb11 + +Completed eligible B; five pairs. Values are median [min, max] across runs. Ratios are paired PostgreSQL/SQLite; no request populations are pooled. + +## Per-run latency quantiles + +| Scenario / class / timing | SQLite p50 ms | PostgreSQL p50 ms | SQLite p95 ms | PostgreSQL p95 ms | Paired p50 ratio (n) | Paired p95 ratio (n) | +|---|---:|---:|---:|---:|---:|---:| +| durable-ingest / additional / end_to_end | 1.461 [1.375, 1.517] | 2.049 [2.006, 2.15] | 1453 [1394, 1630] | 2.844 [2.788, 2.956] | 1.446 [1.335, 1.459] (n=5) | 0.001965 [0.001722, 0.002046] (n=5) | +| durable-ingest / additional / queue | 0.6397 [0.6228, 0.6449] | 0.621 [0.6148, 0.6294] | 1433 [1394, 1610] | 1.076 [1.069, 1.08] | 0.9763 [0.9611, 1.005] (n=5) | 0.0007504 [0.0006689, 0.0007669] (n=5) | +| durable-ingest / additional / service | 0.7615 [0.7034, 0.8227] | 1.367 [1.334, 1.45] | 1.267 [1.159, 1.353] | 2.041 [1.988, 2.185] | 1.86 [1.735, 1.896] (n=5) | 1.615 [1.502, 1.76] (n=5) | +| durable-ingest / duplicate / end_to_end | 1.328 [1.229, 1.419] | 1.897 [1.851, 1.979] | 1394 [1315, 1571] | 2.489 [2.426, 2.629] | 1.481 [1.304, 1.509] (n=5) | 0.00185 [0.001544, 0.00188] (n=5) | +| durable-ingest / duplicate / queue | 0.635 [0.6221, 0.6538] | 0.6051 [0.5988, 0.6299] | 1394 [1315, 1571] | 1.073 [1.065, 1.081] | 0.9709 [0.9194, 1.013] (n=5) | 0.0007642 [0.000688, 0.0008153] (n=5) | +| durable-ingest / duplicate / service | 0.6438 [0.5763, 0.6995] | 1.252 [1.211, 1.314] | 1.006 [0.9305, 1.109] | 1.674 [1.612, 1.761] | 1.993 [1.84, 2.101] (n=5) | 1.682 [1.589, 1.733] (n=5) | +| durable-ingest / late / end_to_end | 1.625 [1.535, 1.814] | 2.188 [2.12, 2.266] | 1355 [1276, 1531] | 2.904 [2.752, 2.953] | 1.347 [1.249, 1.415] (n=5) | 0.002143 [0.001854, 0.002244] (n=5) | +| durable-ingest / late / queue | 0.6344 [0.5942, 0.7032] | 0.6373 [0.5996, 0.6439] | 1354 [1275, 1531] | 1.072 [1.058, 1.085] | 0.9553 [0.889, 1.073] (n=5) | 0.0007966 [0.0007087, 0.0008381] (n=5) | +| durable-ingest / late / service | 0.9377 [0.849, 1.004] | 1.546 [1.505, 1.631] | 1.464 [1.361, 1.52] | 2.071 [1.919, 2.123] | 1.679 [1.616, 1.773] (n=5) | 1.408 [1.35, 1.464] (n=5) | +| durable-ingest / new / end_to_end | 1.671 [1.602, 1.831] | 2.18 [2.131, 2.311] | 1304 [1237, 1516] | 2.858 [2.831, 2.997] | 1.275 [1.25, 1.361] (n=5) | 0.00229 [0.001885, 0.002309] (n=5) | +| durable-ingest / new / queue | 0.6307 [0.6293, 0.6446] | 0.6211 [0.6192, 0.6458] | 1284 [1217, 1496] | 1.074 [1.07, 1.084] | 0.9845 [0.9779, 1.012] (n=5) | 0.0008446 [0.000718, 0.0008798] (n=5) | +| durable-ingest / new / service | 1.003 [0.9395, 1.073] | 1.515 [1.48, 1.607] | 1.634 [1.525, 1.803] | 2.084 [2.028, 2.211] | 1.542 [1.47, 1.575] (n=5) | 1.241 [1.223, 1.359] (n=5) | +| handler-control / 200_valid_envelopes / service | 0.8129 [0.8029, 0.8751] | 1.576 [1.546, 1.728] | 2.036 [1.799, 2.169] | 2.274 [2.113, 2.395] | 1.939 [1.903, 1.974] (n=5) | 1.148 [1.054, 1.264] (n=5) | +| mixed-http / analytics_channels / end_to_end | 30.11 [29.54, 30.58] | 31 [30.71, 31.11] | 34.08 [33.49, 35.53] | 36.57 [34.5, 41.08] | 1.03 [1.004, 1.053] (n=5) | 1.03 [1.025, 1.201] (n=5) | +| mixed-http / analytics_channels / queue | 0.3555 [0.2994, 0.4261] | 0.3569 [0.3157, 0.5119] | 1.033 [0.8971, 1.078] | 1.04 [0.9728, 1.065] | 1.062 [0.9062, 1.44] (n=5) | 1.021 [0.928, 1.157] (n=5) | +| mixed-http / analytics_channels / service | 29.51 [29.13, 30.11] | 30.49 [30.08, 30.61] | 33.77 [32.46, 34.82] | 36.47 [33.86, 40.57] | 1.033 [1.014, 1.051] (n=5) | 1.048 [1.005, 1.19] (n=5) | +| mixed-http / analytics_rf / end_to_end | 2.565 [2.541, 2.609] | 2.337 [2.259, 2.374] | 3.277 [3.126, 3.432] | 3.175 [3.015, 3.337] | 0.896 [0.8807, 0.9297] (n=5) | 0.9342 [0.9182, 1.037] (n=5) | +| mixed-http / analytics_rf / queue | 0.8237 [0.6097, 0.8441] | 0.6058 [0.5712, 0.6508] | 1.069 [1.052, 1.108] | 1.058 [1.021, 1.074] | 0.7696 [0.7355, 0.9646] (n=5) | 0.9893 [0.9418, 1.015] (n=5) | +| mixed-http / analytics_rf / service | 1.749 [1.742, 1.783] | 1.595 [1.576, 1.666] | 2.581 [2.472, 2.659] | 2.588 [2.394, 2.618] | 0.9115 [0.8988, 0.9344] (n=5) | 1.004 [0.9116, 1.021] (n=5) | +| mixed-http / analytics_rf_filtered / end_to_end | 1.898 [1.863, 2.016] | 2.225 [2.164, 2.325] | 582.5 [482.5, 783.3] | 576.1 [549.2, 622.3] | 1.158 [1.14, 1.2] (n=5) | 1.062 [0.7365, 1.141] (n=5) | +| mixed-http / analytics_rf_filtered / queue | 0.1195 [0.1159, 0.147] | 0.1377 [0.127, 0.2074] | 0.8623 [0.8086, 0.9341] | 0.9849 [0.9468, 1.054] | 1.188 [1.033, 1.411] (n=5) | 1.142 [1.119, 1.19] (n=5) | +| mixed-http / analytics_rf_filtered / service | 1.711 [1.653, 1.775] | 1.965 [1.897, 1.996] | 582 [482.3, 783.2] | 576 [548.4, 621.3] | 1.12 [1.114, 1.208] (n=5) | 1.062 [0.7364, 1.141] (n=5) | +| mixed-http / analytics_topology / end_to_end | 30.33 [30.18, 31.06] | 30.59 [30.55, 31.15] | 33.42 [32.16, 37.02] | 39.21 [33.59, 48.29] | 1.007 [0.9931, 1.029] (n=5) | 1.088 [0.9074, 1.445] (n=5) | +| mixed-http / analytics_topology / queue | 0.7453 [0.7158, 0.8084] | 0.6501 [0.5254, 0.7381] | 1.022 [0.9774, 1.087] | 1.036 [1.003, 1.074] | 0.8723 [0.6499, 1.016] (n=5) | 1.018 [0.9234, 1.072] (n=5) | +| mixed-http / analytics_topology / service | 29.64 [29.57, 30.39] | 30.08 [29.98, 30.58] | 32.52 [31.25, 36.5] | 38.55 [32.52, 47.81] | 1.013 [0.9994, 1.032] (n=5) | 1.103 [0.8908, 1.47] (n=5) | +| mixed-http / channels / end_to_end | 1.297 [1.262, 1.322] | 1.307 [1.266, 1.385] | 1.768 [1.748, 1.823] | 1.853 [1.794, 2.156] | 1.009 [0.9578, 1.097] (n=5) | 1.05 [1.015, 1.183] (n=5) | +| mixed-http / channels / queue | 0.5876 [0.5363, 0.6329] | 0.587 [0.5739, 0.637] | 1.046 [1.028, 1.076] | 1.058 [1.03, 1.063] | 1.036 [0.9067, 1.084] (n=5) | 1.002 [0.9846, 1.013] (n=5) | +| mixed-http / channels / service | 0.6692 [0.653, 0.6971] | 0.6724 [0.6581, 0.7051] | 0.9293 [0.8749, 1.064] | 1.058 [0.9314, 1.458] | 1.008 [0.9902, 1.021] (n=5) | 1.209 [0.9345, 1.371] (n=5) | +| mixed-http / messages_0 / end_to_end | 1.952 [1.936, 2.064] | 2.017 [1.963, 2.117] | 60.09 [58.82, 64.02] | 113.3 [108.3, 179] | 1.026 [1.014, 1.057] (n=5) | 1.907 [1.782, 2.796] (n=5) | +| mixed-http / messages_0 / queue | 0.549 [0.4973, 0.6505] | 0.5701 [0.5473, 0.6505] | 1.038 [1.015, 1.054] | 1.049 [1.031, 1.063] | 1.012 [0.9968, 1.206] (n=5) | 1.016 [0.9921, 1.024] (n=5) | +| mixed-http / messages_0 / service | 1.338 [1.32, 1.365] | 1.411 [1.354, 1.477] | 59.91 [58.73, 63.54] | 112.7 [107.7, 178.3] | 1.054 [1.008, 1.082] (n=5) | 1.9 [1.785, 2.806] (n=5) | +| mixed-http / messages_1000 / end_to_end | 2.011 [1.929, 2.142] | 2.089 [2.065, 2.199] | 83.21 [82.65, 86.91] | 114.9 [111.9, 165.4] | 1.031 [0.9738, 1.1] (n=5) | 1.376 [1.345, 1.903] (n=5) | +| mixed-http / messages_1000 / queue | 0.6226 [0.5858, 0.7269] | 0.6336 [0.5732, 0.6619] | 1.054 [1.039, 1.088] | 1.051 [1.04, 1.075] | 0.9207 [0.8867, 1.118] (n=5) | 1.002 [0.9653, 1.021] (n=5) | +| mixed-http / messages_1000 / service | 1.377 [1.344, 1.409] | 1.411 [1.39, 1.455] | 82.79 [81.99, 86.2] | 114.1 [111.4, 165.2] | 1.036 [1.002, 1.044] (n=5) | 1.379 [1.344, 1.917] (n=5) | +| mixed-http / metrics_dense_24h / end_to_end | 3.906 [3.845, 3.95] | 4.405 [4.335, 4.495] | 4.845 [4.579, 4.922] | 5.237 [5.152, 7.076] | 1.138 [1.102, 1.146] (n=5) | 1.073 [1.053, 1.545] (n=5) | +| mixed-http / metrics_dense_24h / queue | 0.5668 [0.4823, 0.6363] | 0.5668 [0.5126, 0.5854] | 1.048 [1.039, 1.057] | 1.039 [1.01, 1.072] | 1.01 [0.8887, 1.214] (n=5) | 0.9975 [0.9635, 1.013] (n=5) | +| mixed-http / metrics_dense_24h / service | 3.327 [3.284, 3.395] | 3.851 [3.796, 3.871] | 4.126 [4.075, 4.211] | 4.688 [4.503, 6.618] | 1.144 [1.14, 1.173] (n=5) | 1.13 [1.07, 1.624] (n=5) | +| mixed-http / metrics_sparse_7d / end_to_end | 2.044 [1.948, 2.16] | 2.283 [2.24, 2.337] | 2.503 [2.452, 2.898] | 2.819 [2.775, 2.987] | 1.115 [1.082, 1.188] (n=5) | 1.132 [0.9759, 1.172] (n=5) | +| mixed-http / metrics_sparse_7d / queue | 0.6256 [0.5906, 0.6783] | 0.5361 [0.5013, 0.576] | 1.054 [1.037, 1.064] | 1.017 [0.9639, 1.053] | 0.8013 [0.7605, 0.9753] (n=5) | 0.9681 [0.9145, 0.9953] (n=5) | +| mixed-http / metrics_sparse_7d / service | 1.369 [1.353, 1.43] | 1.736 [1.709, 1.787] | 1.606 [1.554, 2.147] | 1.961 [1.921, 2.119] | 1.261 [1.25, 1.268] (n=5) | 1.256 [0.9135, 1.298] (n=5) | +| mixed-http / nodes_2000 / end_to_end | 62.5 [61.84, 63.6] | 64.35 [63.35, 64.7] | 119.6 [69.93, 122.1] | 104.7 [72.78, 127] | 1.024 [1.014, 1.032] (n=5) | 0.8723 [0.6087, 1.727] (n=5) | +| mixed-http / nodes_2000 / queue | 0.6 [0.4865, 0.6183] | 0.5109 [0.4756, 0.5571] | 1.033 [0.9864, 1.081] | 1.045 [1.02, 1.06] | 0.8477 [0.7692, 1.132] (n=5) | 0.9911 [0.9756, 1.06] (n=5) | +| mixed-http / nodes_2000 / service | 61.85 [61.2, 62.99] | 63.77 [62.76, 64.36] | 119.4 [69.45, 121.7] | 103.8 [72.35, 126.9] | 1.027 [1.015, 1.033] (n=5) | 0.869 [0.6057, 1.73] (n=5) | +| mixed-http / nodes_50 / end_to_end | 3.661 [3.537, 3.765] | 5.526 [5.443, 5.646] | 8.125 [5.561, 8.896] | 9.892 [7.729, 9.995] | 1.509 [1.49, 1.552] (n=5) | 1.217 [0.8688, 1.797] (n=5) | +| mixed-http / nodes_50 / queue | 0.2297 [0.1344, 0.2799] | 0.1754 [0.1382, 0.1957] | 1.029 [0.9864, 1.051] | 1.016 [0.9824, 1.066] | 0.7635 [0.6017, 1.309] (n=5) | 1 [0.9348, 1.06] (n=5) | +| mixed-http / nodes_50 / service | 3.332 [3.238, 3.414] | 5.231 [5.068, 5.322] | 7.376 [4.911, 8.105] | 9.457 [6.928, 9.643] | 1.566 [1.539, 1.583] (n=5) | 1.294 [0.8548, 1.925] (n=5) | +| mixed-http / nodes_region / end_to_end | 5.921 [5.747, 6.172] | 8.268 [7.988, 8.485] | 12.2 [9.103, 14.11] | 15.57 [14.59, 16.3] | 1.376 [1.363, 1.439] (n=5) | 1.277 [1.035, 1.791] (n=5) | +| mixed-http / nodes_region / queue | 0.4812 [0.4291, 0.5252] | 0.4942 [0.4748, 0.5549] | 1.051 [1.036, 1.065] | 1.046 [1.024, 1.067] | 1.079 [0.9041, 1.152] (n=5) | 0.9877 [0.9702, 1.03] (n=5) | +| mixed-http / nodes_region / service | 5.476 [5.336, 5.637] | 7.724 [7.528, 7.812] | 11.61 [8.665, 13.74] | 15.28 [14.15, 15.75] | 1.402 [1.386, 1.431] (n=5) | 1.316 [1.03, 1.817] (n=5) | +| mixed-http / observers / end_to_end | 3.476 [3.44, 3.605] | 3.519 [3.463, 3.543] | 4.572 [4.434, 4.9] | 4.884 [4.819, 5.227] | 1.002 [0.9828, 1.023] (n=5) | 1.074 [1.054, 1.096] (n=5) | +| mixed-http / observers / queue | 0.5647 [0.5399, 0.5779] | 0.5578 [0.522, 0.5812] | 1.033 [1.019, 1.056] | 1.043 [1.034, 1.064] | 1.015 [0.9033, 1.065] (n=5) | 1.006 [0.9924, 1.03] (n=5) | +| mixed-http / observers / service | 2.846 [2.815, 2.967] | 2.932 [2.867, 2.991] | 3.912 [3.757, 4.061] | 4.267 [4.18, 4.618] | 1.027 [1.003, 1.051] (n=5) | 1.135 [1.066, 1.15] (n=5) | +| mixed-http / packet_detail_memory / end_to_end | 1.867 [1.846, 1.919] | 3.145 [3.069, 3.348] | 2.407 [2.345, 2.501] | 3.763 [3.732, 4.101] | 1.679 [1.65, 1.792] (n=5) | 1.55 [1.538, 1.749] (n=5) | +| mixed-http / packet_detail_memory / queue | 0.5601 [0.531, 0.5866] | 0.5716 [0.5094, 0.6415] | 1.028 [0.9905, 1.057] | 1.031 [1.012, 1.039] | 1.013 [0.9593, 1.094] (n=5) | 1.002 [0.9787, 1.039] (n=5) | +| mixed-http / packet_detail_memory / service | 1.264 [1.212, 1.369] | 2.582 [2.495, 2.779] | 1.586 [1.505, 1.955] | 3.093 [3.044, 3.304] | 2.043 [1.961, 2.17] (n=5) | 1.936 [1.69, 2.189] (n=5) | +| mixed-http / packet_detail_pruned / end_to_end | 1.275 [1.236, 1.431] | 1.254 [1.217, 1.357] | 1.734 [1.725, 1.791] | 1.823 [1.694, 2.103] | 0.9835 [0.8647, 1.074] (n=5) | 1.057 [0.9802, 1.174] (n=5) | +| mixed-http / packet_detail_pruned / queue | 0.5449 [0.4803, 0.7149] | 0.5003 [0.4245, 0.6195] | 1.062 [1.021, 1.072] | 1.036 [1.009, 1.051] | 0.9181 [0.6686, 0.9498] (n=5) | 0.9744 [0.9431, 1.029] (n=5) | +| mixed-http / packet_detail_pruned / service | 0.6807 [0.6523, 0.6935] | 0.7155 [0.6777, 0.7325] | 0.7926 [0.7728, 0.8374] | 1.051 [0.8693, 1.374] | 1.051 [0.9831, 1.108] (n=5) | 1.333 [1.097, 1.778] (n=5) | +| mixed-http / packet_detail_sql / end_to_end | 1.727 [1.68, 1.757] | 2.145 [2.106, 2.316] | 2.197 [2.151, 2.224] | 2.729 [2.662, 3.427] | 1.233 [1.217, 1.366] (n=5) | 1.261 [1.212, 1.541] (n=5) | +| mixed-http / packet_detail_sql / queue | 0.5701 [0.5563, 0.6822] | 0.6203 [0.5669, 0.6349] | 1.046 [1.033, 1.058] | 1.046 [1, 1.069] | 1.088 [0.8995, 1.122] (n=5) | 0.9991 [0.9564, 1.015] (n=5) | +| mixed-http / packet_detail_sql / service | 1.09 [1.066, 1.155] | 1.524 [1.509, 1.626] | 1.33 [1.293, 1.352] | 1.934 [1.79, 2.642] | 1.407 [1.388, 1.43] (n=5) | 1.446 [1.354, 1.954] (n=5) | +| mixed-http / packets_memory / end_to_end | 30.33 [30.07, 33.27] | 35.34 [34.69, 36.85] | 32.37 [31.87, 35.97] | 38.32 [37.21, 40.13] | 1.16 [1.107, 1.195] (n=5) | 1.167 [1.115, 1.229] (n=5) | +| mixed-http / packets_memory / queue | 0.6212 [0.5473, 0.6726] | 0.5808 [0.5089, 0.5875] | 1.056 [1.041, 1.06] | 1.015 [1.012, 1.055] | 0.8832 [0.826, 1.073] (n=5) | 0.9624 [0.9551, 1] (n=5) | +| mixed-http / packets_memory / service | 29.8 [29.35, 32.67] | 34.74 [34.08, 36.26] | 31.86 [31.5, 35.26] | 37.45 [36.81, 39.24] | 1.161 [1.11, 1.197] (n=5) | 1.168 [1.11, 1.246] (n=5) | +| mixed-http / reach / end_to_end | 1.394 [1.291, 1.424] | 1.413 [1.393, 1.476] | 1.811 [1.794, 1.885] | 1.877 [1.842, 1.927] | 1.01 [0.9778, 1.107] (n=5) | 1.034 [0.9878, 1.069] (n=5) | +| mixed-http / reach / queue | 0.5871 [0.5545, 0.6118] | 0.5899 [0.5659, 0.633] | 1.032 [0.9735, 1.056] | 1.053 [1.011, 1.067] | 1.035 [0.9639, 1.068] (n=5) | 1.005 [0.969, 1.096] (n=5) | +| mixed-http / reach / service | 0.802 [0.7769, 0.8079] | 0.8281 [0.8187, 0.8414] | 0.932 [0.9122, 0.9496] | 0.9602 [0.9469, 0.9832] | 1.044 [1.013, 1.066] (n=5) | 1.028 [1.004, 1.077] (n=5) | +| mixed-http / rx_coverage / end_to_end | 2.164 [2.137, 2.2] | 2.331 [2.277, 2.421] | 2.689 [2.623, 2.723] | 2.872 [2.745, 3.056] | 1.087 [1.051, 1.113] (n=5) | 1.076 [1.021, 1.122] (n=5) | +| mixed-http / rx_coverage / queue | 0.5906 [0.5682, 0.6063] | 0.5677 [0.5278, 0.594] | 1.045 [1.004, 1.054] | 1.038 [1.002, 1.048] | 0.9442 [0.8937, 1.028] (n=5) | 0.9927 [0.9507, 1.034] (n=5) | +| mixed-http / rx_coverage / service | 1.556 [1.539, 1.604] | 1.799 [1.681, 1.846] | 1.769 [1.765, 1.83] | 1.959 [1.842, 2.229] | 1.155 [1.092, 1.157] (n=5) | 1.107 [1.043, 1.218] (n=5) | +| mixed-http / stats / end_to_end | 1.11 [1.069, 1.145] | 1.075 [1.034, 1.143] | 2.746 [2.704, 3.13] | 4.481 [4.163, 5.316] | 0.9915 [0.9413, 1.006] (n=5) | 1.516 [1.431, 1.966] (n=5) | +| mixed-http / stats / queue | 0.5857 [0.5192, 0.6159] | 0.5614 [0.5127, 0.5887] | 1.035 [0.9994, 1.054] | 1.032 [1.013, 1.052] | 0.952 [0.8325, 1.134] (n=5) | 0.9919 [0.9628, 1.051] (n=5) | +| mixed-http / stats / service | 0.4691 [0.4541, 0.4871] | 0.4602 [0.4424, 0.4626] | 2.279 [2.1, 2.712] | 3.787 [3.708, 4.776] | 0.9791 [0.9269, 1.002] (n=5) | 1.662 [1.397, 2.075] (n=5) | +| sql-miss / channel_messages_sql / service | 76.11 [66.72, 81.74] | 106.3 [102.2, 118.3] | 85.14 [79.01, 103] | 122.7 [111.9, 183.9] | 1.433 [1.368, 1.531] (n=5) | 1.387 [1.292, 1.786] (n=5) | +| sql-miss / observer_metrics_sql / service | 0.7654 [0.7299, 0.8456] | 0.8473 [0.8147, 0.8923] | 1.314 [1.305, 1.467] | 1.382 [1.21, 1.409] | 1.071 [1.007, 1.166] (n=5) | 1.013 [0.8741, 1.072] (n=5) | +| websocket / new_transmission / receipt_after_ingest_return | 507.1 [506.1, 534.6] | 511.2 [505.5, 519.7] | 985.5 [969.2, 991.8] | 959.6 [946.8, 971] | 1.004 [0.9698, 1.027] (n=5) | 0.969 [0.9608, 0.9916] (n=5) | + +## Eligible p99 + +| Scenario / class / timing | SQLite ms | PostgreSQL ms | Paired ratio (n) | +|---|---:|---:|---:| +| durable-ingest / additional / end_to_end | 7115 [6890, 7476] | 4.16 [3.517, 4.915] | 0.0005931 [0.0004829, 0.0006907] (n=5) | +| durable-ingest / additional / queue | 7076 [6890, 7308] | 1.124 [1.118, 1.137] | 0.0001588 [0.0001535, 0.0001651] (n=5) | +| durable-ingest / additional / service | 3.273 [3.075, 3.537] | 3.024 [2.628, 3.568] | 0.8551 [0.8029, 1.027] (n=5) | +| durable-ingest / duplicate / end_to_end | 7204 [6832, 7437] | 3.076 [2.793, 4.713] | 0.0004278 [0.0003756, 0.0006898] (n=5) | +| durable-ingest / duplicate / queue | 7196 [6812, 7436] | 1.118 [1.112, 1.14] | 0.0001553 [0.0001504, 0.0001673] (n=5) | +| durable-ingest / duplicate / service | 1.603 [1.484, 2.024] | 2.028 [1.936, 3.13] | 1.31 [1.153, 1.546] (n=5) | +| durable-ingest / new / end_to_end | 7113 [7046, 7302] | 3.862 [3.416, 4.425] | 0.0005296 [0.0004739, 0.000628] (n=5) | +| durable-ingest / new / queue | 7113 [7035, 7282] | 1.126 [1.115, 1.152] | 0.0001583 [0.0001532, 0.0001635] (n=5) | +| durable-ingest / new / service | 3.69 [3.642, 4.041] | 2.785 [2.578, 2.993] | 0.7308 [0.6877, 0.8112] (n=5) | +| websocket / new_transmission / receipt_after_ingest_return | 1389 [1330, 1892] | 999 [986.7, 1569] | 0.7334 [0.5279, 1.12] (n=5) | + +## Durations, throughput and sampled resources + +| Metric | Unit | SQLite | PostgreSQL | Paired ratio (n) | +|---|---|---:|---:|---:| +| durable-ingest/additional/completed_per_second_including_drain | per_second | 15.02 [15.02, 15.02] | 15.02 [15.02, 15.02] | 1 [1, 1] (n=5) | +| durable-ingest/duplicate/completed_per_second_including_drain | per_second | 7.515 [7.515, 7.515] | 7.515 [7.515, 7.515] | 1 [1, 1] (n=5) | +| durable-ingest/late/completed_per_second_including_drain | per_second | 2.506 [2.506, 2.506] | 2.506 [2.506, 2.506] | 1 [1, 1] (n=5) | +| durable-ingest/new/completed_per_second_including_drain | per_second | 25.03 [25.03, 25.03] | 25.03 [25.03, 25.03] | 1 [1, 1] (n=5) | +| migration/elapsed_ns | s | — | 147.6 [144.7, 151] | — (n=0) | +| migration/harness_validation_ns | s | — | 54.06 [53.28, 56.62] | — (n=0) | +| migration/migration_tool_ns | s | — | 90.99 [89.94, 94.13] | — (n=0) | +| migration/role_grants_ns | s | — | 0.7733 [0.7025, 0.8264] | — (n=0) | +| mixed-http/analytics_channels/completed_per_second_including_drain | per_second | 0.4109 [0.4109, 0.4109] | 0.4109 [0.4109, 0.4109] | 1 [1, 1] (n=5) | +| mixed-http/analytics_rf/completed_per_second_including_drain | per_second | 0.6157 [0.6157, 0.6157] | 0.6157 [0.6157, 0.6157] | 1 [1, 1] (n=5) | +| mixed-http/analytics_rf_filtered/completed_per_second_including_drain | per_second | 0.411 [0.411, 0.411] | 0.411 [0.411, 0.411] | 1 [1, 1] (n=5) | +| mixed-http/analytics_topology/completed_per_second_including_drain | per_second | 0.6156 [0.6156, 0.6156] | 0.6156 [0.6156, 0.6156] | 1 [1, 1] (n=5) | +| mixed-http/channels/completed_per_second_including_drain | per_second | 1.433 [1.433, 1.433] | 1.433 [1.433, 1.433] | 1 [1, 1] (n=5) | +| mixed-http/messages_0/completed_per_second_including_drain | per_second | 1.433 [1.433, 1.433] | 1.433 [1.433, 1.433] | 1 [1, 1] (n=5) | +| mixed-http/messages_1000/completed_per_second_including_drain | per_second | 1.229 [1.229, 1.229] | 1.229 [1.228, 1.229] | 1 [0.9993, 1] (n=5) | +| mixed-http/metrics_dense_24h/completed_per_second_including_drain | per_second | 1.026 [1.026, 1.026] | 1.026 [1.026, 1.026] | 1 [1, 1] (n=5) | +| mixed-http/metrics_sparse_7d/completed_per_second_including_drain | per_second | 1.026 [1.026, 1.026] | 1.026 [1.026, 1.026] | 1 [1, 1] (n=5) | +| mixed-http/nodes_2000/completed_per_second_including_drain | per_second | 1.432 [1.432, 1.432] | 1.432 [1.432, 1.432] | 1 [0.9998, 1] (n=5) | +| mixed-http/nodes_50/completed_per_second_including_drain | per_second | 1.433 [1.433, 1.433] | 1.433 [1.433, 1.433] | 1 [1, 1] (n=5) | +| mixed-http/nodes_region/completed_per_second_including_drain | per_second | 1.229 [1.229, 1.229] | 1.229 [1.229, 1.229] | 1 [1, 1] (n=5) | +| mixed-http/observers/completed_per_second_including_drain | per_second | 2.052 [2.052, 2.052] | 2.052 [2.052, 2.052] | 1 [1, 1] (n=5) | +| mixed-http/packet_detail_memory/completed_per_second_including_drain | per_second | 1.025 [1.025, 1.025] | 1.025 [1.025, 1.025] | 1 [1, 1] (n=5) | +| mixed-http/packet_detail_pruned/completed_per_second_including_drain | per_second | 1.108 [1.108, 1.108] | 1.108 [1.108, 1.108] | 1 [1, 1] (n=5) | +| mixed-http/packet_detail_sql/completed_per_second_including_drain | per_second | 1.034 [1.034, 1.034] | 1.034 [1.034, 1.034] | 1 [1, 1] (n=5) | +| mixed-http/packets_memory/completed_per_second_including_drain | per_second | 1.025 [1.025, 1.025] | 1.025 [1.025, 1.025] | 1 [1, 1] (n=5) | +| mixed-http/reach/completed_per_second_including_drain | per_second | 1.026 [1.026, 1.026] | 1.026 [1.026, 1.026] | 1 [1, 1] (n=5) | +| mixed-http/rx_coverage/completed_per_second_including_drain | per_second | 1.026 [1.026, 1.026] | 1.026 [1.026, 1.026] | 1 [1, 1] (n=5) | +| mixed-http/stats/completed_per_second_including_drain | per_second | 1.027 [1.027, 1.027] | 1.027 [1.027, 1.027] | 1 [1, 1] (n=5) | +| resource/aggregate_accounted_io_dbytes | MiB | 0 [0, 0] | 0.003906 [0, 0.003906] | — (n=0) | +| resource/aggregate_accounted_io_dios | operations | 0 [0, 0] | 1 [0, 1] | — (n=0) | +| resource/aggregate_accounted_io_rbytes | MiB | 0 [0, 0] | 5.762 [5.57, 6.875] | — (n=0) | +| resource/aggregate_accounted_io_rios | operations | 0 [0, 0] | 658 [641, 780] | — (n=0) | +| resource/aggregate_accounted_io_wbytes | MiB | 2360 [2335, 2378] | 433.6 [376.7, 507.6] | 0.1857 [0.1596, 0.2153] (n=5) | +| resource/aggregate_accounted_io_wios | operations | 2.187e+05 [2.14e+05, 2.236e+05] | 2.173e+04 [1.797e+04, 2.53e+04] | 0.1016 [0.08217, 0.1162] (n=5) | +| resource/average_cores_over_sampled_span | cores | 0.2422 [0.2295, 0.2484] | 0.3191 [0.3099, 0.3706] | 1.324 [1.28, 1.492] (n=5) | +| resource/cpu_delta_usec | CPU s | 43.45 [41.18, 44.58] | 56.86 [55.35, 66.47] | 1.332 [1.274, 1.491] (n=5) | +| resource/observed_peak_cgroup_memory_bytes | MiB | 4043 [4013, 4078] | 5681 [5581, 6144] | 1.415 [1.389, 1.518] (n=5) | +| resource/observed_peak_pss_bytes | MiB | 2922 [2892, 2925] | 3247 [3200, 3321] | 1.11 [1.1, 1.136] (n=5) | +| resource/observed_peak_rss_bytes | MiB | 2926 [2896, 2929] | 5055 [4965, 5221] | 1.728 [1.696, 1.803] (n=5) | +| resource/observed_peak_storage_bytes | MiB | 1098 [1095, 1122] | 2327 [2326, 2327] | 2.12 [2.073, 2.126] (n=5) | +| resource/observed_peak_wal_bytes | MiB | 40.01 [28.92, 53.02] | 1024 [1024, 1024] | 25.6 [19.31, 35.41] (n=5) | +| retention-empty/elapsed_ns | s | 0.0002677 [0.0002576, 0.000307] | 0.001732 [0.001672, 0.001941] | 6.471 [5.936, 7.459] (n=5) | +| retention/elapsed_ns | s | 10.49 [10.45, 10.73] | 0.905 [0.8444, 0.9622] | 0.08626 [0.0805, 0.09126] (n=5) | +| sql-miss/channel_messages_sql/completed_per_second_including_drain | per_second | 13.8 [12.73, 14.12] | 8.948 [7.699, 9.504] | 0.6471 [0.6049, 0.6733] (n=5) | +| sql-miss/observer_metrics_sql/completed_per_second_including_drain | per_second | 13.92 [12.83, 14.23] | 9.089 [7.87, 9.657] | 0.652 [0.6133, 0.6784] (n=5) | +| startup-hot1/first_http_ns | s | 9.565 [9.467, 10.18] | 11.48 [10.98, 11.68] | 1.187 [1.148, 1.223] (n=5) | +| startup-hot1/full_ready_ns | s | 20.52 [20.13, 21.75] | 20.15 [19.92, 20.65] | 0.9707 [0.9494, 1.006] (n=5) | +| startup-hot1/ready_ns | s | 9.565 [9.467, 10.18] | 11.48 [10.98, 11.68] | 1.187 [1.148, 1.223] (n=5) | +| startup/first_http_ns | s | 28.04 [27.44, 30.18] | 27.23 [27.14, 28.55] | 0.9713 [0.9383, 0.9896] (n=5) | +| startup/full_ready_ns | s | 28.06 [27.45, 30.19] | 27.44 [27.22, 29.31] | 0.9783 [0.9403, 0.9989] (n=5) | +| startup/ready_ns | s | 28.04 [27.44, 30.18] | 27.23 [27.14, 28.55] | 0.9713 [0.9383, 0.9896] (n=5) | + +## Measured-window resource coverage + +| Pair / backend | Samples | Sampled span / window s | Boundary gaps first / last s | Max interval s | +|---|---:|---:|---:|---:| +| 0 / sqlite | 180 | 179.425 / 180 | 0.318 / 0.257 | 1.006 | +| 0 / postgres | 175 | 179.331 / 180 | 0.298 / 0.371 | 1.058 | +| 1 / sqlite | 180 | 179.443 / 180 | 0.181 / 0.376 | 1.007 | +| 1 / postgres | 174 | 178.207 / 180 | 0.875 / 0.918 | 1.075 | +| 2 / sqlite | 180 | 179.432 / 180 | 0.207 / 0.361 | 1.009 | +| 2 / postgres | 174 | 178.591 / 180 | 0.875 / 0.534 | 1.102 | +| 3 / sqlite | 180 | 179.444 / 180 | 0.383 / 0.173 | 1.007 | +| 3 / postgres | 175 | 179.154 / 180 | 0.109 / 0.737 | 1.087 | +| 4 / sqlite | 179 | 178.460 / 180 | 0.833 / 0.707 | 1.026 | +| 4 / postgres | 175 | 179.421 / 180 | 0.295 / 0.284 | 1.118 | + +## Acceptance counts + +| Pair / backend | Scheduled / completed ingest | HTTP successes total / measured | Measured WS sentinels / missing / negative | Errors / drops | +|---|---:|---:|---:|---:| +| 0 / sqlite | 12000 / 12000 | 4800 / 3600 | 4500 / 0 / 1 | 0 / 0 | +| 0 / postgres | 12000 / 12000 | 4800 / 3600 | 4500 / 0 / 0 | 0 / 0 | +| 1 / sqlite | 12000 / 12000 | 4800 / 3600 | 4500 / 0 / 0 | 0 / 0 | +| 1 / postgres | 12000 / 12000 | 4800 / 3600 | 4500 / 0 / 0 | 0 / 0 | +| 2 / sqlite | 12000 / 12000 | 4800 / 3600 | 4500 / 0 / 0 | 0 / 0 | +| 2 / postgres | 12000 / 12000 | 4800 / 3600 | 4500 / 0 / 0 | 0 / 0 | +| 3 / sqlite | 12000 / 12000 | 4800 / 3600 | 4500 / 0 / 0 | 0 / 0 | +| 3 / postgres | 12000 / 12000 | 4800 / 3600 | 4500 / 0 / 0 | 0 / 0 | +| 4 / sqlite | 12000 / 12000 | 4800 / 3600 | 4500 / 0 / 0 | 0 / 0 | +| 4 / postgres | 12000 / 12000 | 4800 / 3600 | 4500 / 0 / 0 | 0 / 0 | + +Per-run JSON retains per-device I/O deltas, exact sample bounds, counts, retention deletions, settings, provenance and input hashes. + +## Provenance + +- baseline_sha: 9dbc287579a237ffa744dd0c91fa7227d09763ac +- candidate_sha: ce023e0d623116fe66987761d06016590f98fb11 +- harness_sha256: 207f4317b3dcf0395ca8f76f3a8d4398e2520d2aff429a3381b502becf856f8a +- go_version: go version go1.27.2 linux/amd64 +- postgres_version: postgres (PostgreSQL) 18.6 (Ubuntu 18.6-1.pgdg24.04+2) +- compiler: cc (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0 +- Workload: {"http_concurrency": 64, "http_rate": 20, "ingest_rate": 50, "ingestor_memory_mib": 512, "measured_seconds": 180, "packet_store_mib": 2048, "queue_limit": 1024, "retention_hours": 168, "server_memory_mib": 3072, "sql_reader_pool": 4, "sql_writer_pool": 1, "warmup_seconds": 60} +- Resource budget: {"allowed_cpus": [0, 1, 2, 3], "cpu_limit": 3.0, "enforced": true, "inactive_charge_reset_bytes": 5099520, "memory_limit_bytes": 6442450944, "memory_peak_reset": true, "resource_counters": ["cgroup.procs", "memory.current", "memory.events", "cpu.stat", "io.stat"]} + +## Limits + +- Five paired runs are the independent repetitions; request records are never pooled across runs. +- Absolute entries are medians of per-run quantiles, with observed min/max, not confidence intervals. +- p99 is omitted unless every contributing run has at least 1000 successful samples. +- Ingest service excludes replay queue wait; HTTP service includes response reading and validation. Queue and schedule-to-complete distributions are separate. +- Resource deltas use the first and last samples strictly inside the measured window; no boundary interpolation. Coverage and sampling gaps are reported. +- Memory/RSS/PSS/storage/WAL are observed sampled peaks, not proof against shorter spikes. Storage includes selected database files and WAL. +- I/O is per-device cgroup accounting. The aggregate can count layered devices twice and is not physical storage bytes. +- WebSocket lag is signed receipt minus ingest return; negative lags remain. Nonpositive baseline denominators have no ordinary speed ratio. +- Accounts are disabled by the reviewed harness. The OS cache is warm; startup uses a new process and approximately 100ms controller probes. +- No capacity, OS-cold, 65-minute soak or engine-wide causal claim follows from this report. +- Validation uses public digests and runtime evidence; it does not reopen private databases or payload files. + +## Resource interpretation + +The measured PostgreSQL cgroup-memory peaks for pairs 0–4 were 5581.371, +6143.984, 5742.367, 5680.508 and 5678.938 MiB. Their measured-window +`memory.events.max` deltas were 0, 444, 0, 0 and 0. All 174 samples in pair 1 +were at least 99% of the 6-GiB limit; the nearest sampled headroom was 16 KiB. +SQLite's measured-window memory-event deltas were zero. All final OOM, OOM-kill +and OOM-group-kill counters were zero. Every PostgreSQL full leg approached +6144 MiB during import/startup, so the measured-window table must not be read as +a complete-run headroom or capacity guarantee. + +Native `memory.peak` resets were checked for each cgroup transition; SQLite full +legs measured about 4231–4340 MiB after PostgreSQL legs near 6144 MiB, without +inheriting their prior peaks. PostgreSQL WAL was 1024 MiB in every measured +window, at its start and end: retained segments from import, not measured WAL +generation. Storage includes WAL. Cgroup per-device I/O accounting is not a +physical-disk-write estimate. Observed free filesystem minima (72.065 GiB in +measured windows; 70.817 GiB across full legs) describe this runner only, not a +required operator minimum. + +## Preserved failed qualification + +The [100-event/s run 37897747657](https://github.com/Kpa-clawbot/CoreScope/actions/runs/37897747657) +at `7b2c19a1eaf0050113c1ce8ae73cc69b6e541893` failed its SQLite leg. It completed +23,918 of 24,000 events and dropped 82, all during measured time, with zero +write errors and correct durable effects for completed work. The drop schedule +window was 205.92–206.76 seconds. Retention was scheduled at 195 seconds and took +11.744610088 seconds, deleting 16,000 transmissions and 256,000 observations. +Maximum ingestion queue wait was 11.054 seconds; source and raw timing review +identified maintenance-related queue saturation. The bounded queue held 1024 +events: 10.24 seconds of offered arrivals at 100/s versus 20.48 seconds at 50/s. +This failed artifact supplies no cross-engine conclusion and is not pooled +with the final separately labeled common 50/s experiment. + +## Production and harness hashes + +The application source hash was equal before and after each build/measurement. +Documentation-only publication does not change which application was measured. + +| Material | SHA-256 | +|---|---| +| Candidate production source | `74f4bdc613e1311f65dbbb27d216c186d499d82c05fbb7c9337e239a07b5ae5c` | +| Baseline production source | `7b786814eec6adc6c322a331727d9a4c2bd51a63456b6bb233f2a3327b39381c` | +| Benchmark harness | `207f4317b3dcf0395ca8f76f3a8d4398e2520d2aff429a3381b502becf856f8a` | diff --git a/docs/performance/startup-cache-budget-2026-10-12.md b/docs/performance/startup-cache-budget-2026-10-12.md new file mode 100644 index 000000000..fcb01ec8e --- /dev/null +++ b/docs/performance/startup-cache-budget-2026-10-12.md @@ -0,0 +1,109 @@ +# Startup cache budget repair + +An ARM64 development instance repeatedly reached its 1.5 GiB application +container limit during recovery. The packet cache was configured for 256 MiB, +a one-hour hot startup and 24 hours of cached history. The initial loader used +a transmission-count estimate assuming ten observations per transmission; the +background loader then materialized whole daily windows without applying the +byte budget. Reducing the configured budget alone did not bound that backfill. + +Startup and background loading now share a reader which accounts for actual +observations and publishes only complete transmissions that fit the existing +`packetStore.maxMemoryMB` budget. Bounded loading selects recent traffic first, +including old transmission IDs heard again recently. Reaching the configured +limit is successful cache initialization, with honest coverage reporting and +SQL fallback for omitted history. Live observations and indexes survive an +overlapping history load. Poll watermarks cover the complete database rather +than replaying excluded historical rows. + +There is no schema change or new configuration setting. An unlimited cache +retains primary-key cursor traversal. Memory-capped pages require recency +sorting with the existing indexes; this repair does not add a full index to +every `last_seen` update. A native PostgreSQL plan regression verifies that an +unlimited 100-transmission page seeks past a 20,000-row timestamp tie without +rescanning those earlier rows. + +## Reproduction and measurements + +The committed benchmarks use canonical v3 fixtures with distinct observer/path +pairs, short synthetic packet metadata and twelve-hop paths. The hot window +contains one transmission; the remainder exercises historical backfill. +Both versions use a 16 MiB cache, a one-hour hot window and 24-hour retention. +The fixture has the production index layout, without an extra recency index. + +Baseline server code: `2d1e763a7f2f710066246dcba47ba8cb4f4652e1`, using the +same benchmark fixture through a Go source overlay. Three one-iteration runs +per version/workload were measured on Windows amd64, Intel i5-4690K, Go 1.25.1, +native SQLite, cgo enabled and `GOMAXPROCS=2`. Fixture creation is outside the +timer. Retained Go heap is the change in `HeapAlloc` after explicit GC, keeping +the store alive. Values below are medians of those three runs. + +| Synthetic workload | Startup before | Startup after | Tracked cache before / after | Retained Go heap before / after | +|---|---:|---:|---:|---:| +| 6,000 transmissions, 65 observations each | 5.882 s | 0.416 s | 226.4 / 15.96 MiB | 247.0 / 26.30 MiB | +| 30,000 transmissions, 6 observations each | 2.983 s | 0.277 s | 212.3 / 16.00 MiB | 151.2 / 20.27 MiB | + +Startup times across the three runs were 5.882/5.671/7.192 s versus +0.416/0.435/0.395 s for high fanout, and 3.274/2.983/2.944 s versus +0.304/0.277/0.274 s for typical fanout. This is a cache-budget comparison: +the repaired version intentionally retains less history in RAM. Database +history remains available through SQL. These measurements are not a comparison +of equal in-memory coverage, whole-container memory, or steady-state API and +ingestion performance. The local host was not isolated from other work. + +To repeat the candidate measurements from `cmd/server`: + +```sh +CORESCOPE_TEST_BACKEND=sqlite CGO_ENABLED=1 GOMAXPROCS=2 \ + go test -run '^$' -bench '^BenchmarkStartupMemoryBudget' \ + -benchtime=1x -count=3 -benchmem . +``` + +Regression tests also cover an oversized single transmission, tied timestamps, +ID zero, a recently heard old ID, native PostgreSQL values, complete SQL history +fallback, accurate byte accounting and a live poll interleaved with backfill. +The same startup tests run against both native database backends. + +## Grouped SQL history after bounded startup + +The bounded cache exposes the grouped SQL history path more often. Its old +query formatted every observation timestamp before taking `MAX`, and selected +best-path observations before pagination. The follow-up takes numeric `MAX` +first, then pages only transmission IDs and latest timestamps before joining +payloads and computing observation details. No index or schema change is needed. +The PostgreSQL plan regression checks one backwards indexed timestamp row per +matching transmission and two best-path selections for a two-transmission page. +Filtering, full group counts/IATAs, no-observation timestamp fallback and legacy +v2 results have regression coverage; timestamp ties now use descending ID. + +`BenchmarkGroupedPacketHistory` compares baseline `5633088` with this follow-up +using the same 6,000 transmissions and 390,000 observations, a 24-hour filter, +and three one-iteration runs. Fixture creation is outside the timer. Both +versions return the requested page and report a total of 6,000. The client is +the same Windows amd64/i5-4690K/Go 1.25.1 host with `GOMAXPROCS=2`; SQLite runs +locally, while native PostgreSQL 18 runs in an isolated ARM64 test container on +a Raspberry Pi 5 through a local SSH forward, with normal durability enabled. + +| Backend | Returned page | Before median | After median | +|---|---:|---:|---:| +| SQLite | 1 | 1.654 s | 0.102 s | +| SQLite | 5,000 | 2.693 s | 0.727 s | +| PostgreSQL | 1 | 5.021 s | 0.553 s | +| PostgreSQL | 5,000 | 4.858 s | 4.516 s | + +The PostgreSQL 5,000-row samples overlap substantially: 4.842–5.619 s before, +3.894–6.685 s after. These shared-host measurements support the small-page +improvement, not a robust large-page speedup. A one-row result still ranks all +matching transmissions; the 5,000-row case returns only part of the fixture. +Short synthetic metadata and empty IATA fields are not a live 50,000-row browser +workload. These timings cover database calls and result construction, excluding +HTTP encoding, transfer and browser rendering; they are not a cross-backend +hardware comparison or a whole-process memory measurement. + +Run from `cmd/server`, using `CORESCOPE_TEST_BACKEND=sqlite` or `postgres` with +`CORESCOPE_TEST_POSTGRES_URL` pointing at an isolated test database: + +```sh +CGO_ENABLED=1 GOMAXPROCS=2 go test -run '^$' \ + -bench '^BenchmarkGroupedPacketHistory$' -benchtime=1x -count=3 -benchmem . +``` diff --git a/docs/postgresql-migration-plan.md b/docs/postgresql-migration-plan.md new file mode 100644 index 000000000..4ef123de5 --- /dev/null +++ b/docs/postgresql-migration-plan.md @@ -0,0 +1,55 @@ +# PostgreSQL conversion plan + +Status: implemented and validated at `ce023e0d623116fe66987761d06016590f98fb11`. The [operator guide](postgresql-upgrade.md) covers cutover, interruption recovery and rollback. The [performance report](postgresql-performance.md) records the completed five-pair Linux comparison, including regressions and resource limits. Maintainer acceptance of the replacement remains a separate decision. + +## Goal and baseline + +Deliver one CoreScope PR that makes PostgreSQL the runtime database, supplies a verified upgrade path for existing SQLite instances, and includes reproducible performance results against unmodified upstream commit `9dbc287579a237ffa744dd0c91fa7227d09763ac` (v3.14.0). + +The user has approved a planned downtime window. This plan covers telemetry and the optional account store, not just the main database. The inventory found 18 telemetry tables plus `packets_v`, 12 account tables, four Go commands, and database calls across 50 production Go files. + +Issue #1442 asks for optional PostgreSQL support and contains objections to previous unmeasured engine changes. Reference that discussion without claiming this full replacement satisfies its dual-backend scope. Open a focused conversion/upgrade/benchmark issue before the implementation PR. Acceptance of the proposal by maintainers is not assumed. + +## Design commitments + +- Target supported PostgreSQL 18, pinning the exact tested patch/image in deployment and benchmark evidence. Use pgx through the existing `database/sql` boundaries and explicit PostgreSQL SQL. Preserve the Go services, PacketStore caches, static frontend and polling architecture. +- Keep telemetry and accounts in separate logical databases on one PostgreSQL service by default. Keep separate credentials and handles: telemetry writer, telemetry reader, account writer, and a restricted approved-channel reader for the ingestor. Migration/bootstrap privileges are separate. Test that runtime roles cannot cross these boundaries. +- Retain SQLite only for legacy upgrade/import support and importer test fixtures. Convert server, ingestor, decrypt/migrate commands, active maintenance tools, both backup paths, account export, diagnostics, deployment and CI. +- Preserve public JSON/WebSocket behavior. Explicitly document the necessary changes to database diagnostics and downloadable native backup formats. Introduce a state-directory setting for the existing small file queues/sidecars; a database URL must never become a filesystem path or appear unredacted in logs. +- Preserve observation identity, null behavior, raw values, observer identities, snapshot consistency and committed feed ordering. Do not increase write concurrency merely because PostgreSQL permits it. Replace SQLite-supplied serialization with explicit transactional/locking rules where required, including account compare-and-set and one-time transitions. + +## Milestones and validation + +### 1. PostgreSQL runtime and schema + +Port schema creation/upgrades, connection setup and SQL to PostgreSQL. Replace hidden observer row IDs with explicit stable keys; use returned IDs rather than SQLite insertion APIs. Keep legacy migration history separate from PostgreSQL schema readiness. Move or retire the server's existing attempted telemetry hash writes rather than enabling them through broad credentials. + +Write failing tests before implementation. Validate fresh/repeated/concurrent startup, role-denied writes, deduplication and nulls, time and ordering behavior, delayed commits/ID gaps, snapshot consistency, retention and API/WebSocket parity. Exercise accounts both enabled and disabled, including login/session preservation, single-use tokens, concurrent settings/proposal changes and deletion/anonymization. + +### 2. Existing-instance upgrade + +Provide an explicit offline import command and operator cutover procedure. Stop telemetry and account writers, retain a WAL-aware consistent SQLite recovery snapshot, normalize supported legacy shapes on a working copy, and load an empty PostgreSQL destination in bounded batches. The supported source-layout/version matrix must be documented and covered by fixtures; unknown/newer schemas get an actionable refusal. + +Preserve all supported application data, observer identity links, IDs, sequence high-water marks, password hashes, sessions, preferences, proposals and notifications. Validate counts, stable logical digests and relationships before recording completion. Partial imports must not be accepted by running services; interruption/resume must be deterministic and bound to the same source/target. Never silently drop incompatible records. + +Test source immutability, current and supported older snapshots, accounts absent, null/orphan links, gaps/deleted high IDs, malformed/unsupported data, cancellation, rerun/resume and completion gating. Restore both native PostgreSQL backup types into fresh databases and verify data plus API behavior. Document rollback before PostgreSQL accepts new writes; do not promise lossless reverse conversion after cutover. Document the planned ingestion gap and preservation of existing configuration/key material. + +### 3. Reproducible performance comparison + +Use an isolated Linux runner, running both exact revisions serially on the same machine with the same compiler, workload, logical data, cache policy, retention, resource limits and durability. Keep SQLite WAL/FULL synchronous settings and PostgreSQL durable settings enabled. Count verified database effects and errors; handler returns and the existing WALCommits counter are not proof of durable success. + +The committed 499-transmission fixture is only a correctness input. Add deterministic synthetic corpora with declared distributions: 30K/90K transmissions/observations, 128K/2.048M for the primary comparison, and 1M/16M for larger-data qualification where runner capacity permits. Record measured sizes, not inferred GB labels. Use five paired repetitions for primary comparisons and report resource/cold-cache limitations explicitly. + +Measure durable ingestion, first/full startup readiness, SQL-backed API latency separately from warm memory/cache controls, concurrent ingest/API load, WebSocket visibility lag, bounded retention, whole-stack CPU/memory/storage/WAL, migration duration and disk headroom. Publish absolute values, p50/p95/p99 where sample counts support them, paired variation, errors, regressions, query plans, manifests and raw results. Do not assume or promise a speedup. If the required comparison cannot execute, the performance deliverable remains incomplete. + +The current Windows host has limited free RAM and its existing Podman configuration fails before listing containers. Do not repair or disturb that global environment for this work; use isolated CI resources and bounded local checks. + +### 4. Operational integration and PR review + +Update configuration examples and comments, environment/secret handling, Compose/supervisor variants, native operation, build packaging, migration/backup instructions and database-specific UI labels. Credentials remain deployment configuration. Document pool/timeout controls; track any genuinely user-facing new tuning control for appropriate admin/customizer exposure. + +Run all Go modules, the authoritative standalone frontend suite, PostgreSQL integration tests, imported-fixture browser E2E, account-on/off journeys, backup/restore tests and real-browser API/UI checks. Retain assertion-based red-to-green CI evidence. Perform independent correctness, database/schema, security and TDD reviews; resolve findings, pass exact-head CI and include the benchmark report before marking the single PR ready for maintainer review. + +## Approval boundary + +The implementation plan has received explicit sign-off under repository `AGENTS.md` rule 5. Approval authorizes the implementation and testing above; it does not authorize merging or deploying the PR to live instances. diff --git a/docs/postgresql-performance.md b/docs/postgresql-performance.md new file mode 100644 index 000000000..fdfad9624 --- /dev/null +++ b/docs/postgresql-performance.md @@ -0,0 +1,116 @@ +# PostgreSQL conversion performance + +> Historical measurement of candidate `ce023e0`, before optional backend selection +> and upstream's #2147 concurrency fix. It does not measure the current dual-backend +> implementation. Keep these results separate from the +> [optional-backend comparison against upstream `3e21b17`](performance/optional-backend-b5-2026-10-09.md). + +The PostgreSQL conversion reduced retention-related ingestion delays in this +test. Median writes and several reads were slower, and CPU, memory and storage +costs increased. These results describe the implemented conversion under one +synthetic workload. + +Five alternating pairs completed in [run 37914893623](https://github.com/Kpa-clawbot/CoreScope/actions/runs/37914893623), comparing +[candidate `ce023e0`](https://github.com/Kpa-clawbot/CoreScope/commit/ce023e0d623116fe66987761d06016590f98fb11) +with unmodified upstream +[`9dbc287`](https://github.com/Kpa-clawbot/CoreScope/commit/9dbc287579a237ffa744dd0c91fa7227d09763ac). +The [raw public artifact](https://github.com/Kpa-clawbot/CoreScope/actions/runs/37914893623/artifacts/11613951167), +[complete tables](performance/postgresql-ce023e0-details.md) and +[machine-readable results at commit d132e582](https://github.com/n30nex/CoreScope/blob/d132e582e6260897cde43c8041f104ba53bde0ff/docs/performance/postgresql-ce023e0-details.json) retain +every measured class, regression, range, input hash and limitation. + +Both engines ran serially on the same Linux x86-64 runner, with Go 1.27.2, +PostgreSQL 18.6, durable settings and a common budget of **3 CPUs and 6 GiB** for +application and database. Corpus B +contained 128,000 transmissions, 2,048,000 observations, 2,000 nodes and 128 +observers across eight days. Accounts were disabled; OS caches were warm. +Application startup used a new process; +channel SQL tests explicitly cleared the application cache. + +The supplemental workload offered **50 ingest events/s and 20 HTTP requests/s**, +with 60 seconds of warmup and 180 measured seconds per engine. Every leg completed +12,000 scheduled ingest events and 4,800 expected HTTP responses with zero errors +or dropped schedules. Both startup modes loaded all 112,000 transmissions and +1,792,000 observations in the seven-day retained window, with zero evictions. +All 18-table migration and post-replay digests matched, source files stayed +unchanged, and retention deleted the expected rows. + +Values below are **median [minimum, maximum] across five runs**. The ratio column +is the median and range of the five paired PostgreSQL/SQLite ratios; it is not +the quotient of the displayed medians. Ranges are observed variation, not +confidence intervals. Latencies are milliseconds. Ingest service excludes queue +wait in the replay driver; schedule→return includes it. HTTP timing includes scheduling delay, response +reading and validation. + +| Latency metric (ms) | SQLite | PostgreSQL | Paired PG/SQLite | +|---|---:|---:|---:| +| New ingest: schedule→return p95 | 1304 [1237, 1516] | 2.858 [2.831, 2.997] | 0.00229 [0.001885, 0.002309] | +| New ingest: service p50 | 1.003 [0.9395, 1.073] | 1.515 [1.48, 1.607] | 1.542 [1.47, 1.575] | +| Channel messages: SQL miss p50 | 76.11 [66.72, 81.74] | 106.3 [102.2, 118.3] | 1.433 [1.368, 1.531] | +| Channel messages: HTTP first-page p95 | 60.09 [58.82, 64.02] | 113.3 [108.3, 179] | 1.907 [1.782, 2.796] | +| In-memory packet list: HTTP p50 | 30.33 [30.07, 33.27] | 35.34 [34.69, 36.85] | 1.16 [1.107, 1.195] | +| RX coverage: HTTP p50 | 2.164 [2.137, 2.2] | 2.331 [2.277, 2.421] | 1.087 [1.051, 1.113] | +| RX coverage: HTTP p95 | 2.689 [2.623, 2.723] | 2.872 [2.745, 3.056] | 1.076 [1.021, 1.122] | +| RF analytics: HTTP p50 | 2.565 [2.541, 2.609] | 2.337 [2.259, 2.374] | 0.896 [0.8807, 0.9297] | +| WebSocket receipt after ingest return: p95 | 985.5 [969.2, 991.8] | 959.6 [946.8, 971] | 0.969 [0.9608, 0.9916] | + +Ingestion queue tails improved while median service costs rose across event +classes. Channel SQL misses, HTTP message-page tails and the in-memory packet +list regressed. RF analytics improved; RX coverage remained slightly slower. +WebSocket median lag was +similar (507.1 versus 511.2 ms); its p95 improved modestly. One negative SQLite +receipt-minus-return sample is retained, reflecting visibility before the final +ingest return. The details show p99 only with at least 1,000 samples per run. + +| Duration or resource metric | SQLite | PostgreSQL | Paired PG/SQLite | +|---|---:|---:|---:| +| Retention pass (s) | 10.49 [10.45, 10.73] | 0.905 [0.8444, 0.9622] | 0.08626 [0.0805, 0.09126] | +| Full startup, full readiness (s) | 28.06 [27.45, 30.19] | 27.44 [27.22, 29.31] | 0.9783 [0.9403, 0.9989] | +| Hot startup, initial readiness (s) | 9.565 [9.467, 10.18] | 11.48 [10.98, 11.68] | 1.187 [1.148, 1.223] | +| Hot startup, full background fill (s) | 20.52 [20.13, 21.75] | 20.15 [19.92, 20.65] | 0.9707 [0.9494, 1.006] | +| Measured-window CPU time (CPU s) | 43.45 [41.18, 44.58] | 56.86 [55.35, 66.47] | 1.332 [1.274, 1.491] | +| Sampled cgroup memory peak (MiB) | 4043 [4013, 4078] | 5681 [5581, 6144] | 1.415 [1.389, 1.518] | +| Sampled process PSS peak (MiB) | 2922 [2892, 2925] | 3247 [3200, 3321] | 1.11 [1.1, 1.136] | +| Sampled database + WAL footprint (MiB) | 1098 [1095, 1122] | 2327 [2326, 2327] | 2.12 [2.073, 2.126] | +| Accounted device writes (MiB) | 2360 [2335, 2378] | 433.6 [376.7, 507.6] | 0.1857 [0.1596, 0.2153] | + +Retention removed 16,000 transmissions and 256,000 observations. The empty pass +was slower on PostgreSQL, but small in absolute terms: 0.268 versus 1.732 ms. +Full startup and hot background completion were close; initial hot readiness +was slower on PostgreSQL. Both sustained the offered rate. + +Paired CPU time increased about **33%**, PSS about **11%**, and sampled database +plus WAL footprint about **2.12×**. Resource deltas cover 178.2–179.4 sampled +seconds of each 180-second window; sampled peaks can miss spikes. Cgroup memory +includes file cache and charges retained from preparation/import, so it is not +an application-heap estimate. PostgreSQL's run in pair 1 had all 174 measured samples +at or above 99% of the memory limit, reached within **16 KiB** of it, and recorded +444 `memory.events.max` increments. No OOM or process/group OOM kills occurred. +Every PostgreSQL full leg approached the cap during import/startup. This supplies +no spare-capacity assurance. + +The lower write counter is **cgroup device accounting**, not physical disk-write +volume; layered devices can be counted more than once. PostgreSQL retained +1,024 MiB of WAL segments from import throughout the measured windows. That is +part of the stored footprint, not WAL generated by timed ingestion. PSS accounts +for shared pages; summed RSS is reported separately in the details. + +The synthetic telemetry migration tool took **90.99 s [89.94, 94.13]**. Including +role grants and independent harness validation, the phase took **147.6 s +[144.7, 151.0]**. These timings exclude a real operator's shutdown, account +migration, backups, restore checks and rollback work; they are not an outage +estimate. Follow the [upgrade runbook](postgresql-upgrade.md) and rehearse on +copies of the deployment's own data. + +The earlier [100-event/s qualification](https://github.com/Kpa-clawbot/CoreScope/actions/runs/37897747657) +remains failed: SQLite dropped 82 of 24,000 schedules during an 11.745-second +retention burst. The bounded 1,024-event queue was unchanged for this separately +labeled 50-event/s comparison. Failed and successful experiments are not pooled. + +Production hashes matched before and after measurement; the full candidate, +baseline and harness hashes are in the details. Atomic serialized writes, native +SQL/indexes and maintenance behavior differ between implementations, so these +are conversion results rather than isolated engine effects. No L corpus, +OS-cold run, long soak, account workload or capacity qualification was performed. +Use the [harness instructions](../scripts/postgres-benchmark/README.md) to reproduce +the defined experiment; deployment hardware and workload can change the result. diff --git a/docs/postgresql-upgrade.md b/docs/postgresql-upgrade.md new file mode 100644 index 000000000..fc82c1a71 --- /dev/null +++ b/docs/postgresql-upgrade.md @@ -0,0 +1,213 @@ +> PostgreSQL is optional. Normal installs and updates keep SQLite by default. Use [storage operations](storage.md) for the current setup and verified bidirectional switch. This page retains the lower-level forward-import and PostgreSQL-native recovery details; adding a Compose override alone is not a conversion. + +# PostgreSQL installation and offline upgrade + +When selected, PostgreSQL stores telemetry and optional accounts. The tested database version is PostgreSQL 18.6. Keep the application checkout, image, shared Compose file, bootstrap scripts and configuration example from the same reviewed revision. + +Switching an existing SQLite instance to PostgreSQL needs a planned outage. Stop **both** the ingestor and server: the account janitor, sessions, proposals and notification evaluator also write data. MQTT messages arriving during this window may be lost unless the broker and publishers provide a separately verified replay mechanism. + +For an existing instance, the upgrade path is: + +1. [Stop the old deployment and save its recovery bundle](#preflight-and-recovery). +2. [Preserve configuration and select the correct source directories](#point-the-new-deployment-at-the-right-files). +3. [Run the offline import](#import-with-the-application-stopped). On interruption, use the [resume procedure](#interruption-and-resume). +4. Read the [rollback boundary](#acceptance-and-rollback), then [verify the sources, grant permissions and start](#check-the-retained-source-then-grant-and-start). +5. Check live ingestion and account access, then [verify native backup recovery](#native-backups-and-restores). + +## Databases and credentials + +The supplied deployment creates `corescope_telemetry` and `corescope_accounts` on one PostgreSQL service. They are separate logical databases, with `C` text collation. Runtime URLs must name one host and database; use the default `public` schema for native backups. + +| Role | Purpose | +|---|---| +| `corescope_owner` | Offline import, schema bootstrap and privilege grants. Keep it out of runtime processes. | +| `corescope_reader` | Server telemetry reads and native telemetry backups. | +| `corescope_writer` | Ingestor telemetry writes and retention; no schema ownership. | +| `corescope_accounts` | Account data writes; no schema or readiness-marker writes. | +| `corescope_channels` | Ingestor reads of the approved-channel view; no access to passwords or sessions. | + +Preserve an existing private `.env` and merge only missing PostgreSQL settings from `.env.postgres.example`; do not replace its paths, ports, broker settings or passwords. For a new PostgreSQL data directory, fill each missing password field with a different value from `openssl rand -hex 32` and keep `.env` permissions at `0600`. PostgreSQL creates roles only on first initialization: editing `.env` later does **not** rotate their passwords. If an interrupted installation has lost its matching `.env`, recover that file or perform a coordinated administrator password rotation with all application services stopped; do not delete the PostgreSQL volume to fix authentication. The Compose files supply runtime URLs; credentials do not belong in Git, screenshots, issue reports or command arguments. On native installations, use private environment configuration with `CORESCOPE_READER_DATABASE_URL`, `CORESCOPE_WRITER_DATABASE_URL`, `CORESCOPE_USERS_DATABASE_URL` and `CORESCOPE_APPROVED_CHANNELS_DATABASE_URL`. + +Before initial adoption, configure the intended PostgreSQL URLs explicitly; `CORESCOPE_DATABASE_URL` is the per-process fallback. Once recorded, the selected targets override stale `dbPath`, `userManagement.dbPath` and `DB_PATH` bootstrap hints, so removing those hints is unnecessary. `stateDir` / `CORESCOPE_STATE_DIR` is a separate, stable filesystem directory for the selection, queues, statistics and account backups. A database URL is never a directory name. + +Main runtime pool limits are four server telemetry connections, one ingestor connection and one account connection. Bootstrap, identity checks and backups open additional connections. Connection attempts default to a ten-second upper bound; a shorter URL `connect_timeout` is retained. Native application dumps also have a ten-minute deadline. Backend credentials and pool limits are not theme/customizer controls. + +## Supported SQLite sources + +| Source | Handling | +|---|---| +| Telemetry v3 with `observations.observer_idx` and the known optional columns | Supported. Missing supported columns/tables are added only to a working copy. | +| Telemetry v2 with `observer_id`, or an older Node-era layout | Refused. Upgrade a recovery copy using the last SQLite release, review the result, then import it. | +| Account `schema_version` 0–6 | Supported, including empty version 0. Each declared version must have its complete historical shape. | +| No account database has ever existed | Omit `-users-from-sqlite`; the configured account destination is initialized empty. | +| Accounts currently disabled, but a users database exists | Import it with `-users-from-sqlite` so accounts, audit records and tokens remain available if re-enabled. | +| Unknown/newer columns, tables, views, triggers or virtual tables | Refused, with no silent field or record loss. | +| Duplicate observation identity, empty required transmission fields or case-colliding node keys | Refused. Perform an explicit, reviewed repair on a disposable recovery copy with the last SQLite release. | + +Malformed JSON strings, empty strings, nulls, timestamp text, fractional scores, mixed-case values, ID zero and ID gaps are preserved when PostgreSQL can represent them. Invalid UTF-8, embedded NUL text, incompatible numeric values and records over the native 64 MiB SQLite length limit are refused. The importer never silently coerces incompatible records into zero or null. + +Identity sequences are reseeded above imported IDs, available `sqlite_sequence` high-water marks and historical references, including orphan observer references and deleted users mentioned by audit records. SQLite integer primary keys without `AUTOINCREMENT` do not retain a deleted high-water mark once all references disappear; that information cannot be recovered or invented. + +## Preflight and recovery + +1. While still using the **old checkout and its original `.env`**, record the Compose file, project name, application service/container, image tag **and image ID**, and the actual telemetry/account SQLite paths. Read the old `dbPath`, `userManagement.dbPath` and `DB_PATH` values before removing them. A custom `stateDir` does not prove where either database lives. +2. Stop the old deployment with its own command, before switching checkout or configuration. For example, from that checkout, use `docker compose -f "$OLD_COMPOSE" -p "$OLD_PROJECT" stop "$OLD_SERVICE"` with the recorded values; old `manage.sh` installations can use their existing `./manage.sh stop`. Verify the old application container is stopped, and stop any separately launched server, ingestor or maintenance writer. The new Compose project may have a different name and may require settings the old one never used. +3. Make a private, quiescent recovery bundle containing the old Compose files/scripts, `.env`, image identity, configuration/theme, keys and **whole state/source directories**, including SQLite `-wal` and `-shm` siblings. Keep an untouched original. Before first adoption, use the old deployment's backup procedure or those quiescent copies; the new `manage.sh backup` requires a validated selection and will not guess an unrecorded backend. Restore the preserved old configuration and image together if rolling back. +4. Select new, empty PostgreSQL storage and allocate a private migration-state directory. It will hold a WAL-aware recovery snapshot, normalized working snapshot and source/destination manifest for each store. Snapshot creation uses SQLite's backup API so hidden observer/mail-event row IDs do not change. +5. Measure free space against the actual source, both snapshot copies, PostgreSQL data/indexes, WAL and temporary work. The required space and duration depend on the data; no fixed multiplier or downtime estimate is guaranteed. + +For a host-side copy, after stopping every writer, choose a new private directory outside the active state tree and copy each complete source/state directory with `cp -a`. Include custom account directories too. Never omit a nonempty WAL or run `VACUUM` on the only recovery copy. If an uncheckpointed WAL cannot be read from a read-only mount, import a quiescent copy of the whole source directory from private writable storage, while retaining the original unchanged. + +## Compose cutover + +The common path below uses `docker-compose.example.yml`. Keep the complete pinned checkout and matching `CORESCOPE_IMAGE`; a downloaded YAML alone is insufficient because it extends `docker/postgres.compose.yml` and mounts the initialization scripts. For production/staging variants, retain their existing project name, Compose file and service names throughout. + +### Point the new deployment at the right files + +Preserve/merge `.env`; do not regenerate passwords after PostgreSQL has initialized. In the example variant, set `DATA_DIR` to the **existing absolute host state directory**, `POSTGRES_DATA_DIR` to the selected PostgreSQL storage and `CORESCOPE_IMAGE` to the matching reviewed image. `PROD_DATA_DIR` and `STAGING_DATA_DIR` do not control this variant. Set these before running bootstrap. Keep PostgreSQL storage separate from state/recovery directories. + +Inside the supplied container, the selected host `DATA_DIR` is always `/app/data` and the runtime state directory defaults to `/app/data`. Keep any configured `CORESCOPE_STATE_DIR` subdirectory stable and preserve config/theme/queues and the original database-path settings in the recovery bundle. Preserve unrelated configuration and broker/mail credentials. + +Confirm the exact source mounts before initializing schemas. This check does not start the application or PostgreSQL: + +```sh +set -eu +docker compose -f docker-compose.example.yml -f docker-compose.example.postgres.yml run --rm --no-deps \ + --entrypoint /bin/sh bootstrap -eu -c \ + 'test -f /app/data/config.json; test -f /app/data/meshcore.db; test -f /app/data/users.db' +``` + +Remove the `users.db` check and import flag only when no account source exists, even if account features are currently disabled. If the old installation intentionally had no config file, omit that one file check. For custom SQLite filenames or directories, add a read-only mount such as `-v "$SOURCE_DIR:/legacy:ro"` to **every import and source-check command** and use `/legacy/the-actual-name.db`. Verify those filenames with the same mount first. Keep both sources mounted as directories so their WAL siblings remain visible. + +### Import with the application stopped + +```sh +set -eu +docker compose -f docker-compose.example.yml -f docker-compose.example.postgres.yml up -d --wait postgres +install -d -m 700 migration-state + +docker compose -f docker-compose.example.yml -f docker-compose.example.postgres.yml run --rm \ + --entrypoint /bin/sh \ + -v "$PWD/migration-state:/migration" bootstrap -eu -c 'export CORESCOPE_USERS_DATABASE_URL="$CORESCOPE_USERS_OWNER_DATABASE_URL"; exec /app/corescope-migrate "$@"' sh \ + -offline -from-sqlite /app/data/meshcore.db \ + -users-from-sqlite /app/data/users.db -state-dir /migration +``` + +The current bootstrap service supplies the account owner as `CORESCOPE_USERS_OWNER_DATABASE_URL` and keeps `CORESCOPE_USERS_DATABASE_URL` for runtime. For the legacy commands on this page that omit `-storage-action`, run through `/bin/sh -eu -c 'export CORESCOPE_USERS_DATABASE_URL="$CORESCOPE_USERS_OWNER_DATABASE_URL"; exec /app/corescope-migrate "$@"' sh ...` in the bootstrap container, replacing `...` with the listed non-secret flags. Prefer the current storage-switch workflow above for routine operations. Its PostgreSQL health check only proves the server is accepting connections; successful importer authentication is still required. The importer copies in bounded batches, commits progress with each batch, verifies counts and logical digests in primary-key order, reseeds identities and analyzes the imported tables. Both requested imports must verify before readiness finalization begins. + +Success is exit status zero, a JSON report with `"verified":true` for each imported store, followed by: + +```text +telemetry import verified and finalized; keep the recovery snapshot for rollback +accounts import verified and finalized; keep the recovery snapshot for rollback +``` + +Without an account source, the account line is `accounts PostgreSQL schema initialized`. **On any error, keep all application services stopped.** A successful report for telemetry alone is not a successful two-store cutover. + +### Interruption and resume + +Keep the same source files, configured PostgreSQL host/port, database/schema and migration-state directory, then repeat the import command with `-resume` using the same pinned importer image. A container IP change behind the same configured DNS endpoint does not change import identity. Resume also handles a store not yet started and an interruption between the two readiness commits. An already-ready store is accepted only after its source/snapshot/target identity, saved verification report, completed progress, every table digest and identity sequence are rechecked without rewriting it. New PostgreSQL data or changed evidence refuses resume; it is not a way to re-import after cutover. + +If preparation stopped **before `/manifest.json` exists**, preserve that incomplete `` directory and the originals. Do not delete snapshots or guess a manifest. Confirm that store's PostgreSQL destination is still empty, move its incomplete per-store directory to a separate private recovery location, and retry with `-resume`; keep completed stores' state directories in place. This creates new preparation state only for the untouched store. If the destination has data but its manifest is missing, restore the matching state from recovery or investigate before proceeding. Never manually set readiness true. + +### Check the retained source, then grant and start + +If an old source needed an explicit repair/upgrade, the verified import belongs to the **repaired copy**, not the untouched original. Keep the original database and its WAL/SHM siblings together in the private rollback bundle outside `/app/data`. Put the exact repaired source used for import at the active default guard path, with its matching sidecars, or retain it at its custom path and run the explicit matching source check. Never leave the unrepaired original at `/app/data/meshcore.db` or `/app/data/users.db` and then bypass a fingerprint refusal. Do not repair or rewrite the accepted source after import. + +```sh +set -eu +docker compose -f docker-compose.example.yml -f docker-compose.example.postgres.yml run --rm \ + --entrypoint /bin/sh bootstrap -eu -c 'export CORESCOPE_USERS_DATABASE_URL="$CORESCOPE_USERS_OWNER_DATABASE_URL"; exec /app/corescope-migrate "$@"' sh \ + -check-import-kind=telemetry -from-sqlite /app/data/meshcore.db + +docker compose -f docker-compose.example.yml -f docker-compose.example.postgres.yml run --rm \ + --entrypoint /bin/sh bootstrap -eu -c 'export CORESCOPE_USERS_DATABASE_URL="$CORESCOPE_USERS_OWNER_DATABASE_URL"; exec /app/corescope-migrate "$@"' sh \ + -check-import-kind=accounts -users-from-sqlite /app/data/users.db + +docker compose -f docker-compose.example.yml -f docker-compose.example.postgres.yml run --rm \ + --entrypoint /bin/sh bootstrap -eu -c 'export CORESCOPE_USERS_DATABASE_URL="$CORESCOPE_USERS_OWNER_DATABASE_URL"; exec /app/corescope-migrate "$@"' sh -check-ready + +docker compose -f docker-compose.example.yml -f docker-compose.example.postgres.yml run --rm bootstrap +docker compose -f docker-compose.example.yml -f docker-compose.example.postgres.yml up -d corescope +``` + +Skip the account-source check only when there was no account source. The source checks print ` verified import matches the retained SQLite source`; readiness prints ` PostgreSQL schema is ready` for both configured destinations. These legacy source checks verify the retained files explicitly. Current packaged startup uses the authoritative installation selection and the new guarded setup action; publish a validated PostgreSQL selection with the current storage workflow before starting the application. Do not manually substitute an empty PostgreSQL target for a SQLite installation. Absent and zero-length WAL sidecars share a fingerprint; every nonempty WAL byte remains significant. + +The example publishes HTTP port 80 through bundled Caddy. Open the configured host port and verify `/api/healthz` plus actual packet ingestion. If you explicitly disable Caddy, your proxy or port mapping must instead reach container port 3000. + +For native installations, invoke `corescope-migrate` with owner URLs in `CORESCOPE_DATABASE_URL` and, when applicable, `CORESCOPE_USERS_DATABASE_URL`. The same `-offline`, source, state-directory and resume flags apply. `-check-ready` performs read-only schema checks. Run `docker/postgres-grants.sql` as the owner once in each database after bootstrap/import, then start the services with their restricted runtime URLs and a shared state directory. + +## Acceptance and rollback + +Verify packet and observation counts, representative raw/null values, observers and paths, live ingestion/WebSocket visibility, retention, account login, existing sessions/CSRF, proposals, notifications and account export. Test accounts enabled and disabled as applicable. Restore a native backup into fresh isolated databases and exercise the API/login there before declaring recovery proven. + +Rollback **before PostgreSQL accepts new writes**: stop the new application with its new Compose project, leave its PostgreSQL storage and migration evidence intact, then return to the preserved old checkout/Compose project, original `.env`, pinned SQLite image and untouched source/state directories. Restore the old database-path settings from that bundle; the PostgreSQL configuration cannot be passed to the SQLite release. Confirm the new writers are stopped before starting the old application. Once PostgreSQL has accepted new telemetry or account changes, the old SQLite copy is stale. Use the verified PostgreSQL-to-SQLite storage switch to retain new data; returning to an old pre-cutover copy would discard those writes. + +Keep at least one verified recovery copy. If you later archive the old SQLite files outside the active state directory, move their WAL siblings with them and preserve the import report and original fingerprint evidence. Do not remove the only rollback copy to reclaim space. + +## Native backups and restores + +`GET /api/backup` and `/api/admin/users-backup` download PostgreSQL custom-format `.dump` archives (`application/octet-stream`). Account snapshots remain subject to their configured schedule and rotation. `pg_dump` 18 must be available to the application; the supplied image includes it. + +For the production variant selected by `manage.sh` (not `docker-compose.example.yml`), `./manage.sh backup ` creates `telemetry.dump`, `accounts.dump` and available config/theme/Caddy files. Each database archive is transactionally consistent; two sequential dumps are not one cross-database snapshot. Stop both writers when you need a coordinated pair. Preserve private credentials, queues/state and TLS material separately as part of instance recovery. Never copy a live PostgreSQL data directory as a logical backup. + +`./manage.sh restore ` requires both databases to be empty and the application stopped. It refuses SQLite files and nonempty destinations, uses transactional `pg_restore`, reapplies runtime grants and retains previous configuration copies. Services remain stopped for validation. For a single downloaded archive, use `pg_restore --no-owner --no-privileges --exit-on-error --single-transaction --dbname=` with connection credentials supplied through private libpq environment settings; then apply the appropriate grants and validation. + +For the **example Compose variant**, use its PostgreSQL container directly. This helper reads the owner password already present inside that container; it does not put a URL or password in command arguments. Run it from the matching checkout with the matching private `.env`: + +```sh +pg_owner() { + docker compose -f docker-compose.example.yml -f docker-compose.example.postgres.yml exec -T postgres sh -eu -c ' + export PGHOST=127.0.0.1 PGUSER=corescope_owner PGSSLMODE=disable + export PGPASSWORD="$CORESCOPE_OWNER_PASSWORD" + exec "$@" + ' sh "$@" +} +``` + +For a coordinated backup pair, stop the application, create a new private backup directory, then dump both stores. Keep the application stopped until both commands succeed; a failed `.partial` file is not a backup. + +```sh +set -eu +umask 077 +docker compose -f docker-compose.example.yml -f docker-compose.example.postgres.yml stop corescope +BACKUP_DIR=$(mktemp -d "$PWD/corescope-backup.XXXXXX") +for store in telemetry accounts; do + pg_owner pg_dump --no-password --format=custom --no-owner --no-privileges \ + --dbname="corescope_$store" > "$BACKUP_DIR/$store.dump.partial" + mv "$BACKUP_DIR/$store.dump.partial" "$BACKUP_DIR/$store.dump" +done +# Preserve matching .env, config/theme, keys and remaining state privately too. +docker compose -f docker-compose.example.yml -f docker-compose.example.postgres.yml up -d corescope +``` + +Restore into **new empty PostgreSQL storage**, keeping the previous storage intact. With the original Compose file/project and `.env`, stop `corescope` and then `postgres` before changing any storage path. Record the old `POSTGRES_DATA_DIR`, select a new empty path in `.env`, use matching recovered credentials, and keep `corescope` stopped. Restore the matching config/theme and remaining state from the private bundle before bootstrap; preserve their previous copies. Set `BACKUP_DIR` to the verified backup bundle. Before either restore, check that both destinations are empty: + +```sh +set -eu +docker compose -f docker-compose.example.yml -f docker-compose.example.postgres.yml up -d --wait postgres +for store in telemetry accounts; do + test -f "$BACKUP_DIR/$store.dump" + objects=$(pg_owner psql -X -v ON_ERROR_STOP=1 -At --dbname="corescope_$store" \ + -c "SELECT count(*) FROM pg_class WHERE relnamespace='public'::regnamespace AND relkind IN ('r','p','v','m','S','f')") + test "$objects" = 0 || { echo 'Restore destination is not empty; stop here.' >&2; exit 1; } +done +for store in telemetry accounts; do + pg_owner pg_restore --no-password --no-owner --no-privileges \ + --exit-on-error --single-transaction --dbname="corescope_$store" \ + < "$BACKUP_DIR/$store.dump" +done +docker compose -f docker-compose.example.yml -f docker-compose.example.postgres.yml run --rm bootstrap +docker compose -f docker-compose.example.yml -f docker-compose.example.postgres.yml run --rm \ + --entrypoint /bin/sh bootstrap -eu -c 'export CORESCOPE_USERS_DATABASE_URL="$CORESCOPE_USERS_OWNER_DATABASE_URL"; exec /app/corescope-migrate "$@"' sh -check-ready +``` + +If one restore fails, the other database may already have restored successfully: keep services stopped and preserve both the failure evidence and old storage. Do not retry into an occupied database. After successful grants/readiness and the account checks below, validate the restored instance in isolation before starting it for normal traffic. + +Account restoration can revive deleted accounts, old passwords, revoked sessions and already-used links. Record relevant deletions from the current audit log before replacement. With the restored server still stopped, disable those accounts and invalidate sessions/tokens as required, then repeat account deletion through the application so mail addresses are anonymized and the deletion is audited. See [account recovery](user-guide/accounts.md#backups). + +## Test fixtures and benchmark evidence + +The committed historical fixture contains a duplicate observation identity group. CI explicitly repairs a disposable copy with `scripts/prepare-postgres-fixture.go`, reports before/after counts, freshens/seeds that copy, then runs `scripts/migrate-fixture-hashes.go` using the shared runtime hash implementation. These are test-fixture tools, not production repair flags. The strict importer still refuses the unreviewed original fixture. + +Run the PostgreSQL integration suites with `CORESCOPE_TEST_POSTGRES_URL` pointing to a disposable PostgreSQL 18 administrator connection and native 18.x dump/restore clients on `PATH`. Missing database configuration fails the tests. Performance reports must compare the pinned SQLite baseline and exact candidate with matched compiler, workload, durability and resource limits; migration time includes snapshotting, normalization, verification and `ANALYZE`. No speedup is assumed. diff --git a/docs/storage.md b/docs/storage.md new file mode 100644 index 000000000..3cbcd4638 --- /dev/null +++ b/docs/storage.md @@ -0,0 +1,179 @@ +# Choose, update and recover storage + +CoreScope uses **SQLite by default**. PostgreSQL 18.6 is optional. Both keep telemetry and accounts separate; the ingestor writes telemetry and the server reads it. Accounts remain at their recorded target while user management is disabled. If a legacy installation never initialized accounts, enabling them is an explicit setup operation: stop writers and run `./manage.sh setup` after editing the account configuration. Restart alone does not invent a missing account database. + +Use the [measured backend comparison](performance/optional-backend-b5-2026-10-09.md) to compare latency, resource use and conversion overhead before choosing PostgreSQL. + +Choose for the hardware and workload, not the age of the installation. SQLite +keeps the deployment small; PostgreSQL separates database resources and can help +with concurrent writes and maintenance, but adds a service and its own memory, +storage and backup costs. Neither backend is universally faster. The hot packet +cache now accounts for SQLite observation fan-out when admitting history: under +the same memory budget it may retain a shorter hot window. Omitted history stays +in SQL; a smaller cache does not change database retention. + +After startup loading finishes, the server emits one `[store] startup admission` +summary with the configured memory budget, admitted transmission and observation +counts, tracked memory estimate and retained `firstSeenRange`. `coverageFrom` is +the conservative complete-coverage boundary; an older first-seen timestamp can +belong to a long-lived hash and does not establish complete history coverage. +`stop=memory_budget` means the bounded cache is ready, `window_complete` means the +requested window finished, and `error` leaves startup unready. +`formerPolicyTransmissionEstimate` describes the previous admission formula +(ten assumed observations per transmission and a 1,000-transmission minimum); +it is not a measured previous memory footprint. Use `/api/healthz` for readiness +and SQL-backed views to inspect history outside the admitted cache. + +## Install + +Keep a complete checkout pinned to the same release as the image. For the usual single-host installation, preserve an existing private `.env` or copy `.env.example`, set `DATA_DIR` to the intended persistent directory, then run: + +```sh +docker compose -f docker-compose.example.yml up -d +``` + +No PostgreSQL variables or service are required. Before supervisor starts either process, packaged setup validates an existing supported SQLite installation or initializes only a provably fresh one. A missing selection alone is not permission to discard data. If setup reports an error, keep the application stopped and preserve the files. + +For a **new PostgreSQL installation**, merge `.env.postgres.example` into the private `.env`. Generate a distinct `openssl rand -hex 32` password for each blank field only before the first initialization of an empty PostgreSQL directory. Then add the matching override: + +```sh +docker compose -f docker-compose.example.yml -f docker-compose.example.postgres.yml up -d +``` + +The override starts a separate PostgreSQL service and a short-lived owner bootstrap. Runtime containers receive reader, writer, account and approved-channel roles, never owner credentials. There is no published database port. Existing SQLite data is not converted by adding this file. + +For source installations, `./manage.sh setup` offers SQLite (default) or PostgreSQL. It preserves existing private environment settings and keeps an installed backend. Its production files are `docker-compose.yml` / `docker-compose.no-mosquitto.yml`; PostgreSQL adds `docker-compose.postgres.yml`. Staging uses its own base and `docker-compose.staging.postgres.yml`, persistent directories and project. The example deployment is separate from the variants managed by `manage.sh`. + +Success means setup exits successfully, `/api/healthz` reports ready, and actual packets appear through the API/UI. A running container or PostgreSQL health check alone is not evidence of completed setup or authentication. + +## Keep an installed backend on update + +`storage-selection.json` in the shared state directory records the backend and both targets atomically. It contains no passwords. Old `db.backend`, `CORESCOPE_DB_BACKEND`, `dbPath` and account-path bootstrap hints cannot replace a recorded target. Credentials and TLS settings stay in private config/environment and can rotate without changing the endpoint identity. + +Use the same state mount and Compose files on restart/update. `./manage.sh update` keeps the recorded backend. For a pre-built image, pull the matching image and run `up -d` with the same files, including the PostgreSQL override when selected. Removing an override does not perform a reverse conversion. + +Before a PostgreSQL schema-version upgrade, take native backups of all recorded +stores, including disabled accounts, and preserve the matching state/configuration +and previous image. Stop both application writers +for the update. Managed bootstrap checks readiness and, when necessary, invokes +the offline owner upgrade for the existing recorded targets before runtime +starts. A supported older schema is reported as `upgrade required` with its +stored and target versions, followed by the verified upgrade result. The +provisional readiness error is not printed after a successful upgrade. +Missing, corrupt or future schemas still fail; their diagnostics remain visible +and runtime services stay stopped. Bootstrap rechecks readiness before granting +runtime access. It cannot replace missing initialized databases or add an absent account +store. Previously initialized accounts are upgraded even while their UI is +disabled. Native deployments use the same operation with owner credentials in +private environment configuration: + +```sh +corescope-migrate -storage-action=upgrade -offline \ + -selection-file /absolute/state/storage-selection.json -config-dir /absolute/config +``` + +Telemetry and accounts upgrade in separate database transactions; there is no +cross-database atomicity guarantee. Keep both writers stopped until upgrade and +readiness succeed. A failure keeps application startup blocked. +Preserve both databases and retry the supported upgrade after correcting the +reported problem. An older image may reject the newer schema, so reverting only +the image is not a complete rollback: restore both stores from their matching pre-upgrade backups +when required. Do not delete the selection or initialize replacement stores. + +Container state must stay at `/app/data` or a persistent subdirectory, selected with `CORESCOPE_STATE_DIR`. All supplied services share that value. The entrypoint refuses an ephemeral directory or a symlink escaping the data mount. Native relative SQLite paths resolve from the process working directory; container processes run in `/app`. A `config.json` that exists but cannot be read or parsed stops startup; it is not ignored in favour of defaults. + +Before the first upgrade from an older deployment, stop that old stack using its actual Compose project/service and checkout, before replacing configuration files. Preserve its checkout/Compose files, image identity, private `.env`, config/theme and a quiescent copy of the complete data directory including WAL/SHM siblings. Do not use a raw copy of a live SQLite main file as a backup. + +## Explicit offline backend change + +Plan a maintenance window and measure disk headroom on a representative copy. Keep source data, recovery snapshots and the new target together until validation is complete. Data/index/WAL and process memory costs differ between engines; one synthetic benchmark does not size every installation. + +The published B5 experiment observed a PostgreSQL DB/cluster plus WAL peak about +**2.12 times** the SQLite database plus sidecars (about **112% more**). This is an +observation, not a sizing multiplier or guarantee. Using that ratio only as an +illustration, source + one recovery copy + PostgreSQL alone is `1 + 1 + 2.12 = +4.12` times the source: a 16.8 GB source would total about **69.2 GB** before +additional working space. The current forward importer also retains a normalized +working snapshot. Allow separately for that copy, account data, index building, +temporary work and additional WAL. The measured PostgreSQL peak already included +1 GiB of WAL; it does not bound WAL on another installation. Rehearse the actual +conversion and keep its observed free-space margin. + +For the managed production deployment: + +```sh +./manage.sh storage status +./manage.sh backup ./backups/before-storage-change +./manage.sh storage switch postgres # or: sqlite +``` + +The switch asks for confirmation, stops the owned application processes, starts only the database service when needed, and invokes the verified converter. It does not bootstrap the destination schema before conversion. Existing accounts participate even when their UI is disabled. A PostgreSQL destination must be empty; preserve an old cluster and choose a new PostgreSQL directory instead of overwriting it. Switching to SQLite creates new destination paths under the existing persistent state directory, preserving the old source files. + +The converter snapshots/normalizes supported source data, copies bounded batches, verifies table counts, typed logical digests and reference/identity high-water marks, applies narrow destination grants, and checks runtime connections. Both requested stores must verify before the single selection replacement. The state directory itself does not move. + +The optional SQLite hop index is derived data, not a telemetry source. Conversion +accepts only the known v1 layout: `observation_hops`, +`observation_hops_coverage`, and all three `trg_observation_hops_ai`, `_ad`, `_au` +triggers with their expected definitions. Partial layouts, changed definitions +and extra indexes on these tables are refused. The converter removes these +objects only from its private normalized snapshot; the original source and +recovery copy retain them. Reports list both tables in `excluded_derived` with +their schema version and rebuild reason, outside the canonical row/digest +comparison. + +`retention.hopIndexDays` controls this cache on SQLite only. PostgreSQL uses its +native reach query and logs that the SQLite cache option is inactive; no +PostgreSQL hop-index schema is installed. After conversion back to SQLite, +the cache is absent until the ingestor normally installs and rebuilds it when +`hopIndexDays` is positive. + +On success, inspect the credential-free report and recorded targets. The management command leaves writers stopped: + +```sh +./manage.sh storage status +./manage.sh start +``` + +Validate packet/observation counts, raw bytes, paths, live ingestion, accounts/login, existing sessions, settings, proposals and notifications as applicable. Preserve source and recovery data after success. A pre-cutover copy becomes stale once the new backend accepts writes; it is not a lossless reverse migration. To return while retaining new writes, perform a verified switch in the other direction. + +### Interrupted conversion + +A pending journal blocks startup. Do not delete it, rewrite the selection by hand, or manually mark a target ready. Keep the same source, configured PostgreSQL host/port/schema, target paths, recovery directory and matching converter version. Status supplies the job ID: + +```sh +./manage.sh storage status +./manage.sh storage resume JOB_ID +# Or, only while the old selection is still authoritative: +./manage.sh storage abort JOB_ID +``` + +Resume rechecks saved source/target identity, snapshots, progress, data and sequences. Changed data or mismatched evidence is refused. Abort is allowed only before selection publication and preserves staged/recovery files. If preparation failed before a complete manifest existed, preserve the incomplete directory and original files; follow the reported recovery instructions instead of inventing progress or deleting the only snapshots. + +The native CLI has the same actions through `-storage-action`, always with an absolute `-selection-file`, and `-offline` for mutation. Its `-state-dir` is private recovery/job storage, not the stable application state directory. Use `-config` / `-config-dir` for the same fail-closed configuration rules as runtime. Owner connections belong in `CORESCOPE_DATABASE_URL` and, for new storage actions, `CORESCOPE_USERS_OWNER_DATABASE_URL`; `CORESCOPE_USERS_DATABASE_URL` remains the runtime account writer. Never put credential URLs in command arguments. Custom/external PostgreSQL endpoints require explicit native owner tooling; managed backup/restore refuses to substitute the local default database. + +## Native backups and restore + +`./manage.sh backup DIRECTORY` creates native SQLite `telemetry.db` / `accounts.db` snapshots or PostgreSQL `telemetry.dump` / `accounts.dump`, plus a private `state.tar`, matching `compose.env`, and available config/theme/Caddy files. SQLite uses a read-only native backup that includes committed WAL pages. The state archive excludes the selected live SQLite files/sidecars and PostgreSQL storage. An absent, never-created SQLite account store is recorded explicitly. Use a new empty backup directory outside live data. + +Each database snapshot is consistent; sequential snapshots plus copied state are not one cross-database transaction. Stop both writers when a coordinated pair/queue state is required. Keep the bundle and matching private `.env` encrypted off-host; do not commit or upload real databases, account state or credentials. + +For SQLite restore, stop the application, retain the current data directory, and select a **new empty** `PROD_DATA_DIR`. `./manage.sh restore DIRECTORY` stages the state archive and native files, validates schemas and integrity, then publishes the restored directory. It refuses occupied targets, archive links/escaping paths and targets outside the persistent mount. Failure leaves source backups and staged recovery material intact. The services remain stopped for review. Older standalone snapshots or custom layouts use the native recovery procedure instead of guessing a selection record. + +For PostgreSQL restore, keep the application stopped and preserve its matching state/selection and private settings. Select a new PostgreSQL storage directory, start only the database service, and restore into two empty databases with `./manage.sh restore DIRECTORY`. Do not run schema bootstrap first. The helper refuses occupied targets, restores transactionally per database, reapplies grants and analyzes data. If one restore fails, keep everything stopped; do not rerun into the already-restored database. For a lost host, restore and review the matching private state/selection before database restoration, using the same container paths. See [native PostgreSQL recovery](postgresql-upgrade.md#native-backups-and-restores). + +Restoring accounts can revive deleted users, old passwords, revoked sessions and used links. Preserve relevant current audit entries before replacing state; disable affected accounts and clear restored sessions/tokens before serving traffic, then repeat deletion through the application to anonymize mail history and audit it. See [account backups](user-guide/accounts.md#backups). + +## Supported source layouts + +| Store | Supported current/upgrade boundary | +|---|---| +| SQLite telemetry | Known v3 layout and supported append-only migrations; older/unknown layouts require the prior version's explicit upgrade/repair | +| SQLite accounts | Known versions0–6 and the exact fork identity-v6 layout upgrade to v7; includes companion links, non-reusing identities and explicit mail-event IDs | +| PostgreSQL telemetry/accounts | Telemetry readiness version2, with explicit offline owner upgrade from version1; account readiness version1 and logical schema version6 | + +Unknown/future columns, hidden generated columns, unsupported objects or conflicting data refuse conversion. Preserve a raw recovery snapshot before an explicit repair/upgrade of a disposable working copy. Deleted legacy integer IDs without allocator or retained-reference evidence have no recoverable historical high-water mark. Do not infer one or discard references to make validation pass. + +The large-instance preparation work tracked by [#2214](https://github.com/Kpa-clawbot/CoreScope/issues/2214) +is still pending. This update does not add a snapshot bypass or accept arbitrary +triggers. An unsupported schema remains a stop condition; retain the original +and recovery copies rather than deleting objects merely to pass validation. diff --git a/docs/user-guide/accounts.md b/docs/user-guide/accounts.md index 73bdc77c8..d6148bd69 100644 --- a/docs/user-guide/accounts.md +++ b/docs/user-guide/accounts.md @@ -72,7 +72,8 @@ incomplete, and the log says what is missing. |---|---| | `adminEmails` | Addresses that become admin on activation. They cannot be demoted, disabled or deleted from the UI. Remove an address here first. | | `publicBaseUrl` | The address visitors use. Every mail link is built from it, never from the request. It must match the browser origin, because state-changing requests from another origin are refused. | -| `dbPath` | Where accounts are stored. Default: `users.db` next to the analyzer database. | +| `dbPath` | SQLite account file, default `users.db` beside telemetry. The installed target remains recorded while accounts are disabled. | +| `databaseURL` | When PostgreSQL is selected, use a separate account database and restricted writer via `CORESCOPE_USERS_DATABASE_URL`. | | `sessionDays` | Login lifetime, extended while in use. Default 30, maximum 365. | | `trustedProxies` | CIDRs of your reverse proxy, so the per-IP login limits see real client IPs. Without it, behind a proxy every client shares the proxy's IP for the per-IP limits (they are switched off when that IP is loopback or private). Per-address and per-account limits apply either way. | | `mail.provider` | `brevo` (default) or `postal`. | @@ -182,11 +183,7 @@ on the *Proposals* tab. decision. - `maxApproved` bounds the decryption work: every approved key is tried on every group message. `perUserPerDay` and `maxPending` limit proposals (HTTP 429). -- The ingestor reads `users.db` read-only, from `userManagement.dbPath` or next to the - analyzer database. Set `dbPath` explicitly when the server and the ingestor are not - given the same analyzer database path (for example `DB_PATH` set for one of them). - Both log the absolute path they use at startup (server: `[users] user management - enabled: db=...`, ingestor: `[proposals] reading approved channels from ...`). +- The ingestor reads only the `approved_channels` view through `CORESCOPE_APPROVED_CHANNELS_DATABASE_URL`. This role can check account readiness and read approved channels, but cannot read users, password hashes, sessions or tokens. It connects to the same account database as the server with a separate restricted credential. - Names: at most 31 bytes including the `#` (MeshCore stores 32 with the terminator), no invisible or control characters (blank fillers such as U+3164 and spaces other than the plain space count as invisible), case-sensitive, not Public. Emoji work, except @@ -215,7 +212,7 @@ watched node changes state. Admins can also watch the instance. - A check runs every `intervalMinutes` (the first one an interval after startup, and only after the startup load). All changes for one user in one check go into one mail. The first check of a newly watched node stores its state without a mail; states - are kept in `users.db`, so a restart does not mail again. + are kept in the account database, so a restart does not mail again. - While ingest is stale (the newest packet in the packet store is older than 30 minutes, for example when the MQTT broker or the ingestor is down), the offline checks for nodes and observers pause: their states stay as they were and nothing is mailed for @@ -305,12 +302,12 @@ coverage counts as the owner's (*My coverage* on the coverage page). Links are p ### Backups -`users.db` holds password hashes and addresses. The server keeps its own snapshots of +The separate SQLite account file or PostgreSQL account database holds password hashes and addresses. The server keeps its own snapshots of it: at startup when the newest snapshot is older than 24 hours (or there is none), then -about every 24 hours (the check runs hourly). A snapshot is a complete copy named `users-.db` (UTC), -readable by the server's user only, in `backups/` next to `users.db`. After each new +about every 24 hours (the check runs hourly). A native snapshot is named `users-.db` for SQLite or `.dump` for PostgreSQL (UTC), +readable by the server's user only, in `backups/` under the configured `stateDir`. After each new snapshot the oldest ones beyond `keep` are deleted, never the one just written. -Temporary files of an interrupted snapshot (`users-.db.tmp`) are +Temporary files of an interrupted snapshot (`users-.db.tmp` or `.dump.tmp`) are deleted once they are older than 24 hours; other files in that directory are never touched. A failed snapshot is logged (`[users] backup failed: ...`) and the next run tries again. @@ -332,16 +329,16 @@ delete that copy. | Key | Meaning | |---|---| | `backup.enabled` | Default `true`, also when the block is absent. `false` turns the snapshots off. | -| `backup.dir` | Where snapshots go. Empty: `backups/` next to `users.db`. A relative path is relative to the server's working directory. | +| `backup.dir` | Where snapshots go. Empty: `backups/` under `stateDir`. A relative path is relative to the server's working directory. | | `backup.keep` | How many snapshots are kept. Default 7, also for 0 or less. | Snapshots on the same disk are lost with that disk. To keep a copy elsewhere, log in as an admin and open `/api/admin/users-backup` in the same browser: it downloads a fresh -snapshot (`corescope-users-.db`) and records it in the audit log +snapshot (`corescope-users-.db` or `.dump`, matching the backend) and records it in the audit log (`user.backup`). Store the download encrypted: it holds every password hash and address. The analyzer database has its own backup route, `GET /api/backup`. -**What a restore brings back.** `users.db` returns to the moment of the snapshot: +**What a restore brings back.** The account database returns to the moment of the snapshot: - accounts deleted after it, including accounts users deleted themselves; - old passwords of users who changed them since; @@ -351,36 +348,41 @@ The analyzer database has its own backup route, `GET /api/backup`. The audit log is replaced too, so note the deletions before you overwrite it. -**Restore** (not automated). The commands use the `sqlite3` command-line tool; replace -`2026-10-08 12:00:00` with the snapshot's time from its file name (UTC). +For SQLite, restore the native `.db` snapshot only while all account writers are stopped, into a new account target; keep the current file and WAL siblings for audit/recovery. Use the matching installation state and [native storage restore procedure](../storage.md#native-backups-and-restore). Do not replace a live main file or treat an old snapshot as a reverse migration after new writes. The deletion query below can use `datetime(at, 'unixepoch')` and an integer Unix cutoff on SQLite; the session/token cleanup applies to both engines. -1. Stop the server. -2. List the accounts deleted since the snapshot, from the current `users.db`: +**PostgreSQL restore:** use a fresh account database while the server and every account writer are stopped. Keep the old database intact for recovery and audit review. Supply PostgreSQL connection credentials through private libpq environment settings (`PGHOST`, `PGUSER`, `PGDATABASE` and a protected password source), never command arguments. - ```sh - sqlite3 users.db "SELECT target_user_id, action, datetime(at, 'unixepoch') FROM audit_log - WHERE action IN ('user.delete', 'user.delete.self') - AND at >= CAST(strftime('%s', '2026-10-08 12:00:00') AS INTEGER);" +1. On the current database, record accounts deleted since the snapshot. Replace the timestamp with the snapshot's UTC time: + + ```sql + SELECT target_user_id, action, to_timestamp(at) + FROM audit_log + WHERE action IN ('user.delete', 'user.delete.self') + AND at >= EXTRACT(EPOCH FROM TIMESTAMPTZ '2026-10-08 12:00:00+00'); ``` -3. Copy the snapshot over `users.db`, and delete `users.db-wal` and `users.db-shm` if - they exist. -4. Still with the server stopped, disable the accounts from step 2 (replace `12, 34` - with their ids) and clear all sessions and links, so everyone logs in again: +2. Point the owner connection at an empty account database and restore: ```sh - sqlite3 users.db "UPDATE users SET status = 'disabled' WHERE id IN (12, 34); - DELETE FROM sessions; DELETE FROM tokens;" + pg_restore --no-owner --no-privileges --exit-on-error --single-transaction --dbname="$PGDATABASE" account-backup.dump + psql -X -v ON_ERROR_STOP=1 -f docker/postgres-grants.sql + psql -X -v ON_ERROR_STOP=1 -c 'ANALYZE' + ``` + +3. Still with the server stopped, disable the previously deleted accounts and clear sessions and one-time links. Replace the example IDs with the recorded IDs: + + ```sql + BEGIN; + UPDATE users SET status = 'disabled' WHERE id IN (12, 34); + DELETE FROM sessions; + DELETE FROM tokens; + COMMIT; ``` -5. Start the server. -6. Delete the accounts from step 2 again in the admin area. Deleting there, not in - `sqlite3`, also replaces the addresses in the mail history with a hash and records - the deletion in the audit log. +4. Check schema readiness with the matching `corescope-migrate -check-ready` binary and the owner account URL. Keep the matching selection and restricted runtime target identities, validate the restored stores, then start the server. A different target must be explicitly adopted/selected through the offline operator workflow, not silently substituted by editing a URL. +5. Delete those accounts again through the admin area. Application deletion also replaces addresses in mail history with a hash and records the deletion in the audit log. -A snapshot made by a newer CoreScope version is refused at startup ("database schema -version N is newer than this binary supports"): run that version or newer. Deleting -`users.db` removes all accounts and nothing else. +Keep notification delivery paused until its restored state has been reviewed; old notification preferences and unsubscribe links also return. A newer schema is refused by older binaries. Never delete or overwrite the current database just to make restoration succeed. See [native backups and restores](../postgresql-upgrade.md#native-backups-and-restores) for the two-database recovery procedure. ## For users diff --git a/internal/dbconfig/dbconfig.go b/internal/dbconfig/dbconfig.go index 21ac79416..8072a202d 100644 --- a/internal/dbconfig/dbconfig.go +++ b/internal/dbconfig/dbconfig.go @@ -1,12 +1,14 @@ // Package dbconfig provides the shared DBConfig struct used by both the server -// and ingestor binaries for SQLite vacuum and maintenance settings (#919, #921). +// and ingestor binaries for storage selection, startup and maintenance settings. package dbconfig -// DBConfig controls SQLite vacuum and maintenance behavior (#919). +// DBConfig controls storage and startup loading. SQLite maintenance options +// remain active for SQLite and are ignored by PostgreSQL. type DBConfig struct { - VacuumOnStartup bool `json:"vacuumOnStartup"` // one-time full VACUUM on startup if auto_vacuum is not INCREMENTAL - IncrementalVacuumPages int `json:"incrementalVacuumPages"` // pages returned to OS per reaper cycle (default 1024) - AnalysisLimit int `json:"analysisLimit"` // index rows ANALYZE visits per index (default 10000); negative disables the planner stats refresh (#2058) + Backend Backend `json:"backend,omitempty"` + VacuumOnStartup bool `json:"vacuumOnStartup"` + IncrementalVacuumPages int `json:"incrementalVacuumPages"` + AnalysisLimit int `json:"analysisLimit"` // Load controls chunked startup loading (#1009). Load *LoadConfig `json:"load,omitempty"` diff --git a/internal/dbconfig/selection.go b/internal/dbconfig/selection.go new file mode 100644 index 000000000..e58130c69 --- /dev/null +++ b/internal/dbconfig/selection.go @@ -0,0 +1,331 @@ +package dbconfig + +import ( + "crypto/rand" + "encoding/hex" + "errors" + "net" + "net/url" + "os" + "path/filepath" + "reflect" + "regexp" + "strconv" + "strings" +) + +const selectionVersion = 1 + +var ( + ErrSelectionMissing = errors.New("installation selection is absent; setup or the writer must validate and adopt existing storage before reader startup") + ErrSelectionBusy = errors.New("installation is in use; stop its services before backend adoption, switching or recovery") + ErrSelectionInProgress = errors.New("an incomplete backend switch blocks startup; keep writers stopped and explicitly resume or abort that job") + ErrSelectionChanged = errors.New("recorded storage target or generation changed; use a verified offline selection update") +) + +type PostgresTarget struct { + Host string `json:"host"` + Port uint16 `json:"port"` + Database string `json:"database"` + Schema string `json:"schema"` +} + +type Target struct { + SQLitePath string `json:"sqlite_path,omitempty"` + Postgres *PostgresTarget `json:"postgres,omitempty"` +} + +// Selection records endpoints, never authentication or TLS settings. Accounts +// describe the configured store independently of whether its feature is enabled. +type Selection struct { + Version int `json:"version"` + Generation string `json:"generation"` + Backend Backend `json:"backend"` + StateDir string `json:"state_dir"` + Telemetry Target `json:"telemetry"` + Accounts *Target `json:"accounts"` +} + +func SelectionPath(stateDir string) string { return filepath.Join(stateDir, "storage-selection.json") } + +func newSelectionID() (string, error) { + var value [16]byte + if _, err := rand.Read(value[:]); err != nil { + return "", err + } + return hex.EncodeToString(value[:]), nil +} + +func validSelectionID(value string) bool { + decoded, err := hex.DecodeString(value) + return err == nil && len(decoded) == 16 && value == strings.ToLower(value) +} + +// NewSelection constructs a proposal. Callers validate the same native targets +// before adopting/committing it. It never connects or infers freshness. +func NewSelection(storage Storage) (Selection, error) { + s := Selection{Version: selectionVersion, Backend: storage.Backend, StateDir: storage.StateDir} + var err error + if s.Generation, err = newSelectionID(); err != nil { + return Selection{}, err + } + switch s.Backend { + case SQLite: + s.Telemetry = Target{SQLitePath: storage.DBPath} + if storage.UsersDBPath != "" { + s.Accounts = &Target{SQLitePath: storage.UsersDBPath} + } + case Postgres: + if s.Telemetry, err = postgresRoleTarget(storage.ReaderDatabaseURL, storage.WriterDatabaseURL); err != nil { + return Selection{}, err + } + if storage.UsersDatabaseURL != "" || storage.ApprovedChannelsDatabaseURL != "" { + target, err := postgresRoleTarget(storage.UsersDatabaseURL, storage.ApprovedChannelsDatabaseURL) + if err != nil { + return Selection{}, err + } + s.Accounts = &target + } + default: + return Selection{}, errors.New("selection requires a validated backend") + } + return s, s.validate() +} + +func (s Selection) validate() error { + if s.Version != selectionVersion || !validSelectionID(s.Generation) || !filepath.IsAbs(s.StateDir) || filepath.Clean(s.StateDir) != s.StateDir { + return errors.New("invalid or unsupported installation selection") + } + check := func(target Target) bool { + if s.Backend == SQLite { + return target.Postgres == nil && filepath.IsAbs(target.SQLitePath) && filepath.Clean(target.SQLitePath) == target.SQLitePath && !strings.ContainsRune(target.SQLitePath, 0) + } + p := target.Postgres + return s.Backend == Postgres && target.SQLitePath == "" && p != nil && p.Host != "" && p.Port != 0 && p.Database != "" && p.Schema != "" && !strings.ContainsRune(p.Host+p.Database+p.Schema, 0) + } + if !check(s.Telemetry) || (s.Accounts != nil && !check(*s.Accounts)) { + return errors.New("installation selection has invalid storage targets") + } + if s.Accounts != nil { + if s.Backend == SQLite && s.Telemetry.SQLitePath == s.Accounts.SQLitePath { + return errors.New("telemetry and account targets must remain separate") + } + if s.Backend == Postgres { + a, b := s.Telemetry.Postgres, s.Accounts.Postgres + if a.Host == b.Host && a.Port == b.Port && a.Database == b.Database { + return errors.New("telemetry and account databases must remain separate") + } + } + } + return nil +} + +func postgresRoleTarget(first, second string) (Target, error) { + if first == "" { + first = second + } + if first == "" { + return Target{}, errors.New("PostgreSQL target requires a role URL") + } + value, err := postgresTarget(first) + if err != nil { + return Target{}, err + } + if err := bootstrapPostgresEnvironment(first); err != nil { + return Target{}, err + } + if second != "" { + other, err := postgresTarget(second) + if err != nil { + return Target{}, err + } + if value != other { + return Target{}, ErrSelectionChanged + } + if err := bootstrapPostgresEnvironment(second); err != nil { + return Target{}, err + } + } + return Target{Postgres: &value}, nil +} + +func bootstrapPostgresEnvironment(value string) error { + if os.Getenv("PGOPTIONS") != "" || os.Getenv("PGSERVICE") != "" { + return errors.New("initial PostgreSQL selection requires explicit URL settings instead of PGSERVICE or PGOPTIONS") + } + u, err := url.Parse(value) + if err != nil { + return errors.New("invalid PostgreSQL target URL") + } + if u.Port() == "" && os.Getenv("PGPORT") != "" { + port, err := strconv.ParseUint(os.Getenv("PGPORT"), 10, 16) + if err != nil || port != 5432 { + return errors.New("initial PostgreSQL selection has an ambiguous ambient PGPORT; supply an explicit URL port") + } + } + return nil +} + +var unquotedSchema = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_$]*$`) + +func selectionSchema(value string) (string, error) { + value = strings.TrimSpace(value) + if value == "" { + return "public", nil + } + if strings.HasPrefix(value, `"`) && strings.HasSuffix(value, `"`) && len(value) > 2 { + inner := value[1 : len(value)-1] + if strings.Contains(strings.ReplaceAll(inner, `""`, ""), `"`) || strings.ContainsRune(inner, 0) { + return "", errors.New("selection requires one explicit PostgreSQL schema") + } + return strings.ReplaceAll(inner, `""`, `"`), nil + } + if !unquotedSchema.MatchString(value) { + return "", errors.New("selection requires one explicit PostgreSQL schema") + } + return strings.ToLower(value), nil +} + +func postgresTarget(value string) (PostgresTarget, error) { + u, err := url.Parse(value) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") || u.Hostname() == "" || strings.Contains(u.Host, ",") || strings.TrimPrefix(u.Path, "/") == "" { + return PostgresTarget{}, errors.New("invalid PostgreSQL target URL") + } + port := uint64(5432) + if u.Port() != "" { + port, err = strconv.ParseUint(u.Port(), 10, 16) + if err != nil || port == 0 { + return PostgresTarget{}, errors.New("invalid PostgreSQL target port") + } + } + q, err := url.ParseQuery(u.RawQuery) + if err != nil || len(q["search_path"]) > 1 { + return PostgresTarget{}, errors.New("invalid PostgreSQL target schema setting") + } + // Drivers accept connection fields in the query as overrides of the URL. + // Keep the recorded endpoint unambiguous, including service-file settings + // and startup options that can replace search_path outside this parser. + for _, key := range []string{"host", "hostaddr", "port", "dbname", "database", "service", "servicefile", "options"} { + if _, present := q[key]; present { + return PostgresTarget{}, errors.New("PostgreSQL target overrides must be expressed in the URL authority, database path and search_path") + } + } + schema, err := selectionSchema(q.Get("search_path")) + if err != nil { + return PostgresTarget{}, err + } + host := strings.ToLower(u.Hostname()) + if ip := net.ParseIP(host); ip != nil { + host = ip.String() + } + return PostgresTarget{Host: host, Port: uint16(port), Database: strings.TrimPrefix(u.Path, "/"), Schema: schema}, nil +} + +func bindSelectedURL(value string, target *PostgresTarget) (string, error) { + if value == "" { + return "", nil + } + if target == nil { + return "", ErrSelectionChanged + } + u, err := url.Parse(value) + if err != nil { + return "", errors.New("invalid PostgreSQL target URL") + } + // Missing port/schema are bootstrap defaults, not permission to change an + // installed target through PGPORT or a rotated role's default search_path. + if u.Port() == "" { + u.Host = net.JoinHostPort(u.Hostname(), strconv.Itoa(int(target.Port))) + } + q, err := url.ParseQuery(u.RawQuery) + if err != nil { + return "", errors.New("invalid PostgreSQL target URL query") + } + if !q.Has("search_path") { + q.Set("search_path", `"`+strings.ReplaceAll(target.Schema, `"`, `""`)+`"`) + u.RawQuery = q.Encode() + } + actual, err := postgresTarget(u.String()) + if err != nil { + return "", err + } + if actual != *target { + return "", ErrSelectionChanged + } + u.Host = net.JoinHostPort(target.Host, strconv.Itoa(int(target.Port))) + u.Path = "/" + target.Database + u.RawPath = "" + // Pin the selected schema even when rotated role defaults differ. Host and + // database are never rebound; credentials and TLS options remain untouched. + q.Set("search_path", `"`+strings.ReplaceAll(target.Schema, `"`, `""`)+`"`) + u.RawQuery = q.Encode() + return u.String(), nil +} + +// ApplyTo makes the installed record authoritative over bootstrap selectors +// and paths. Only active PostgreSQL role credentials/TLS remain caller inputs. +func (s Selection) ApplyTo(raw StorageInputs) (StorageInputs, error) { + if err := s.validate(); err != nil { + return StorageInputs{}, err + } + raw.Backend, raw.EnvBackend, raw.ExistingBackend = s.Backend, s.Backend, s.Backend + raw.FreshInstall = false + raw.StateDir = s.StateDir + raw.DBPath, raw.UsersDBPath = "", "" + if s.Backend == SQLite { + raw.DBPath = s.Telemetry.SQLitePath + if s.Accounts != nil { + raw.UsersDBPath = s.Accounts.SQLitePath + } + return raw, nil + } + reader, writer := raw.ReaderDatabaseURL, raw.WriterDatabaseURL + if reader == "" { + reader = raw.DatabaseURL + } + if writer == "" { + writer = raw.DatabaseURL + } + var err error + if raw.ReaderDatabaseURL, err = bindSelectedURL(reader, s.Telemetry.Postgres); err != nil { + return StorageInputs{}, err + } + if raw.WriterDatabaseURL, err = bindSelectedURL(writer, s.Telemetry.Postgres); err != nil { + return StorageInputs{}, err + } + var account *PostgresTarget + if s.Accounts != nil { + account = s.Accounts.Postgres + } else { + raw.UsersDatabaseURL, raw.ApprovedChannelsDatabaseURL = "", "" + } + if raw.UsersDatabaseURL, err = bindSelectedURL(raw.UsersDatabaseURL, account); err != nil { + return StorageInputs{}, err + } + if raw.ApprovedChannelsDatabaseURL, err = bindSelectedURL(raw.ApprovedChannelsDatabaseURL, account); err != nil { + return StorageInputs{}, err + } + raw.DatabaseURL = "" // Role URLs are now bound; an obsolete fallback is inactive. + return raw, nil +} + +func sameSelectionTargets(a, b Selection) bool { + a.Generation, b.Generation = "", "" + return reflect.DeepEqual(a, b) +} + +func cloneSelection(s Selection) Selection { + if s.Telemetry.Postgres != nil { + target := *s.Telemetry.Postgres + s.Telemetry.Postgres = &target + } + if s.Accounts != nil { + target := *s.Accounts + s.Accounts = &target + if target.Postgres != nil { + postgres := *target.Postgres + target.Postgres = &postgres + } + } + return s +} diff --git a/internal/dbconfig/selection_file.go b/internal/dbconfig/selection_file.go new file mode 100644 index 000000000..49c935413 --- /dev/null +++ b/internal/dbconfig/selection_file.go @@ -0,0 +1,510 @@ +package dbconfig + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "reflect" + "sync" +) + +const maxSelectionBytes = 64 * 1024 + +type selectionLease struct { + file *os.File + once sync.Once + err error +} + +func (l *selectionLease) Close() error { + l.once.Do(func() { l.err = errors.Join(unlockSelectionFile(l.file), l.file.Close()) }) + return l.err +} + +func acquireSelectionLease(path string, exclusive bool) (*selectionLease, error) { + if !filepath.IsAbs(path) || filepath.Clean(path) != path { + return nil, errors.New("selection path must be absolute and normalized") + } + f, err := os.OpenFile(path+".lock", os.O_CREATE|os.O_RDWR, 0600) + if errors.Is(err, os.ErrNotExist) { + return nil, ErrSelectionMissing + } + if err != nil { + return nil, err + } + if err = lockSelectionFile(f, exclusive); err != nil { + f.Close() + return nil, err + } + return &selectionLease{file: f}, nil +} + +func readSelectionJSON(path string, out any) error { + info, err := os.Lstat(path) + if err != nil { + return err + } + if !info.Mode().IsRegular() || info.Size() > maxSelectionBytes { + return errors.New("invalid selection metadata file") + } + f, err := openSelectionRead(path) + if err != nil { + return err + } + data, err := io.ReadAll(io.LimitReader(f, maxSelectionBytes+1)) + closeErr := f.Close() + if err != nil { + return err + } + if closeErr != nil { + return closeErr + } + if len(data) > maxSelectionBytes { + return errors.New("selection metadata exceeds its bound") + } + d := json.NewDecoder(bytes.NewReader(data)) + d.DisallowUnknownFields() + if err := d.Decode(out); err != nil { + return errors.New("invalid selection metadata JSON") + } + if err := d.Decode(new(any)); err != io.EOF { + return errors.New("unexpected trailing selection metadata") + } + return nil +} + +func readSelected(path string) (Selection, error) { + var s Selection + if err := readSelectionJSON(path, &s); err != nil { + if errors.Is(err, os.ErrNotExist) { + return s, ErrSelectionMissing + } + return s, err + } + if err := s.validate(); err != nil { + return Selection{}, err + } + if SelectionPath(s.StateDir) != path { + return Selection{}, errors.New("selection state directory does not match its discovery path") + } + return s, nil +} + +func selectionJournalError(path string) error { + _, err := os.Lstat(path + ".switch.json") + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return err + } + var journal selectionJournal + if readSelectionJSON(path+".switch.json", &journal) == nil && validSelectionID(journal.ID) { + return fmt.Errorf("%w (job %s)", ErrSelectionInProgress, journal.ID) + } + return ErrSelectionInProgress +} + +// ReadSelection is for inspection. A runtime must retain OpenSelection's lease. +func ReadSelection(path string) (Selection, error) { + if err := selectionJournalError(path); err != nil { + return Selection{}, err + } + s, err := readSelected(path) + if err != nil { + return Selection{}, err + } + if err := selectionJournalError(path); err != nil { + return Selection{}, err + } + return s, nil +} + +// OpenSelection returns a shared local lease held until service shutdown. On +// any error, including an absent record, no lease remains to deadlock adoption. +func OpenSelection(path string) (Selection, io.Closer, error) { + return openSelectionLease(path, false) +} + +// OpenSelectionForMaintenance excludes running services and selection changes +// during an offline schema upgrade. It never creates or rewrites the record. +func OpenSelectionForMaintenance(path string) (Selection, io.Closer, error) { + return openSelectionLease(path, true) +} + +func openSelectionLease(path string, exclusive bool) (Selection, io.Closer, error) { + lease, err := acquireSelectionLease(path, exclusive) + if err != nil { + return Selection{}, nil, err + } + s, err := ReadSelection(path) + if err != nil { + lease.Close() + return Selection{}, nil, err + } + return s, lease, nil +} + +// AdoptSelection follows caller validation of an existing or newly initialized +// installation. It never silently changes an existing selection's targets. +func AdoptSelection(path string, validated Selection) (Selection, error) { + if err := validated.validate(); err != nil { + return Selection{}, err + } + if SelectionPath(validated.StateDir) != path { + return Selection{}, errors.New("adoption state directory differs from discovery path") + } + if current, lease, err := OpenSelection(path); err == nil { + defer lease.Close() + if !sameSelectionTargets(current, validated) { + return Selection{}, ErrSelectionChanged + } + return current, nil + } else if !errors.Is(err, ErrSelectionMissing) { + return Selection{}, err + } + if err := os.MkdirAll(validated.StateDir, 0700); err != nil { + return Selection{}, err + } + u, err := BeginSelectionUpdate(path, "") + if err != nil { + return Selection{}, err + } + defer u.Close() + if _, err := u.Commit(validated); err != nil { + return Selection{}, err + } + return validated, nil +} + +type selectionJournal struct { + Version int `json:"version"` + ID string `json:"id"` + Source *Selection `json:"source"` + Next *Selection `json:"next,omitempty"` +} + +// SelectionStatus is safe to inspect before choosing a database service or +// loading credentials. Unrecorded means setup validation is still required. +type SelectionStatus struct { + Version int `json:"version"` + State string `json:"state"` + Backend Backend `json:"backend"` + Generation string `json:"generation"` + JobID string `json:"job_id"` + SourceBackend Backend `json:"source_backend"` + TargetBackend Backend `json:"target_backend"` + StateDir string `json:"state_dir"` + Telemetry *Target `json:"telemetry"` + Accounts *Target `json:"accounts"` +} + +func (s *SelectionStatus) selectedTargets(selected Selection) { + s.Backend, s.Generation, s.StateDir = selected.Backend, selected.Generation, selected.StateDir + s.Telemetry, s.Accounts = &selected.Telemetry, selected.Accounts +} + +func InspectSelection(path string) (SelectionStatus, error) { + var status SelectionStatus + var err error + for attempt := 0; attempt < 3; attempt++ { + status, err = inspectSelection(path) + if !errors.Is(err, ErrSelectionInProgress) && !errors.Is(err, ErrSelectionChanged) { + return status, err + } + } + if errors.Is(err, ErrSelectionInProgress) { + status.State = "pending" + return status, nil + } + return status, err +} + +func inspectSelection(path string) (SelectionStatus, error) { + status := SelectionStatus{Version: selectionVersion, State: "corrupt"} + if !filepath.IsAbs(path) || filepath.Clean(path) != path { + return status, errors.New("selection path must be absolute and normalized") + } + status.StateDir = filepath.Dir(path) + var journal selectionJournal + err := readSelectionJSON(path+".switch.json", &journal) + if err == nil { + if err = journal.validate(path); err != nil { + return status, err + } + u := SelectionUpdate{path: path, journal: journal} + replaced, err := u.currentState() + if err != nil { + return status, err + } + status.State, status.JobID = "pending", journal.ID + if journal.Source != nil { + status.SourceBackend = journal.Source.Backend + status.selectedTargets(*journal.Source) + } + if journal.Next != nil { + status.TargetBackend = journal.Next.Backend + if replaced { + status.selectedTargets(*journal.Next) + } + } + return status, nil + } + if !errors.Is(err, os.ErrNotExist) { + return status, err + } + s, err := ReadSelection(path) + if errors.Is(err, ErrSelectionMissing) { + status.State = "unrecorded" + return status, nil + } + if err != nil { + return status, err + } + status.State = "ready" + status.selectedTargets(s) + return status, nil +} + +func (j selectionJournal) validate(path string) error { + if j.Version != 1 || !validSelectionID(j.ID) { + return errors.New("invalid selection recovery journal") + } + for _, s := range []*Selection{j.Source, j.Next} { + if s == nil { + continue + } + if err := s.validate(); err != nil { + return err + } + if SelectionPath(s.StateDir) != path { + return errors.New("journal state directory differs from discovery path") + } + } + if j.Source != nil && j.Next != nil && j.Source.Generation == j.Next.Generation { + return ErrSelectionChanged + } + return nil +} + +// SelectionUpdate is a single local selection-file transaction, not a database +// converter. Its caller verifies both stores and holds native DB writer fences. +type SelectionUpdate struct { + path string + lease *selectionLease + journal selectionJournal + afterReplace func() error // fault injection in package tests +} + +func (u *SelectionUpdate) ID() string { return u.journal.ID } +func (u *SelectionUpdate) Current() Selection { + if u.journal.Source == nil { + return Selection{} + } + return cloneSelection(*u.journal.Source) +} + +func (u *SelectionUpdate) Target() Selection { + if u.journal.Next == nil { + return Selection{} + } + return cloneSelection(*u.journal.Next) +} + +// Stage binds the exact unselected destination before any copy begins. It does +// not authorize startup or claim data verification; only Commit selects it. +func (u *SelectionUpdate) Stage(next Selection) error { + if u.lease == nil { + return errors.New("selection update is closed") + } + if _, err := u.currentState(); err != nil { + return err + } + if err := next.validate(); err != nil { + return err + } + if SelectionPath(next.StateDir) != u.path || (u.journal.Source != nil && next.StateDir != u.journal.Source.StateDir) { + return errors.New("backend switching cannot relocate the installation state directory") + } + if u.journal.Source != nil && next.Generation == u.journal.Source.Generation { + return errors.New("selection update requires a new generation") + } + if u.journal.Next != nil { + if !reflect.DeepEqual(next, *u.journal.Next) { + return ErrSelectionChanged + } + return nil + } + staged := u.journal + next = cloneSelection(next) + staged.Next = &next + if _, err := writeSelectionJSON(u.path+".switch.json", staged); err != nil { + return err + } + u.journal = staged + return nil +} +func (u *SelectionUpdate) Close() error { + if u.lease == nil { + return nil + } + err := u.lease.Close() + u.lease = nil + return err +} + +func BeginSelectionUpdate(path, expectedGeneration string) (*SelectionUpdate, error) { + lease, err := acquireSelectionLease(path, true) + if err != nil { + return nil, err + } + fail := func(err error) (*SelectionUpdate, error) { lease.Close(); return nil, err } + if err := selectionJournalError(path); err != nil { + return fail(err) + } + current, err := readSelected(path) + if err != nil && !errors.Is(err, ErrSelectionMissing) { + return fail(err) + } + if (err == nil && current.Generation != expectedGeneration) || (err != nil && expectedGeneration != "") { + return fail(ErrSelectionChanged) + } + id, err := newSelectionID() + if err != nil { + return fail(err) + } + u := &SelectionUpdate{path: path, lease: lease, journal: selectionJournal{Version: 1, ID: id}} + if current.Generation != "" { + u.journal.Source = ¤t + } + if _, err = writeSelectionJSON(path+".switch.json", u.journal); err != nil { + return fail(err) + } + return u, nil +} + +func ResumeSelectionUpdate(path, jobID string) (*SelectionUpdate, error) { + lease, err := acquireSelectionLease(path, true) + if err != nil { + return nil, err + } + fail := func(err error) (*SelectionUpdate, error) { lease.Close(); return nil, err } + var journal selectionJournal + if err := readSelectionJSON(path+".switch.json", &journal); err != nil { + return fail(err) + } + if journal.ID != jobID { + return fail(errors.New("selection recovery requires the exact recorded job")) + } + if err := journal.validate(path); err != nil { + return fail(err) + } + u := &SelectionUpdate{path: path, lease: lease, journal: journal} + if _, err := u.currentState(); err != nil { + return fail(err) + } + return u, nil +} + +func (u *SelectionUpdate) currentState() (bool, error) { + current, err := readSelected(u.path) + if errors.Is(err, ErrSelectionMissing) && u.journal.Source == nil { + return false, nil + } + if err != nil { + return false, err + } + if u.journal.Next != nil && reflect.DeepEqual(current, *u.journal.Next) { + return true, nil + } + if u.journal.Source != nil && reflect.DeepEqual(current, *u.journal.Source) { + return false, nil + } + return false, ErrSelectionChanged +} + +// Commit is called after complete source/target verification. A true result +// with an error means replacement occurred but recovery/cleanup is unfinished. +// Inspect status after cleanup errors; Abort never restores the old record. +func (u *SelectionUpdate) Commit(next Selection) (bool, error) { + if u.lease == nil { + return false, errors.New("selection update is closed") + } + replaced, err := u.currentState() + if err != nil { + return false, err + } + if err := u.Stage(next); err != nil { + return replaced, err + } + if !replaced { + replaced, err = writeSelectionJSON(u.path, next) + if err != nil { + // Native replacement errors can be ambiguous. Classify from the actual + // record, keeping the journal regardless; never claim an old selection. + if actual, readErr := readSelected(u.path); readErr == nil && reflect.DeepEqual(actual, next) { + replaced = true + } + return replaced, err + } + if u.afterReplace != nil { + if err := u.afterReplace(); err != nil { + return true, err + } + } + } + if err := os.Remove(u.path + ".switch.json"); err != nil { + return true, err + } + return true, syncSelectionDirectory(filepath.Dir(u.path)) +} + +func (u *SelectionUpdate) Abort() error { + if u.lease == nil { + return errors.New("selection update is closed") + } + replaced, err := u.currentState() + if err != nil { + return err + } + if replaced { + return errors.New("selected generation already changed; resume verified completion instead of aborting") + } + if err := os.Remove(u.path + ".switch.json"); err != nil { + return err + } + return syncSelectionDirectory(filepath.Dir(u.path)) +} + +func writeSelectionJSON(path string, value any) (replaced bool, err error) { + data, err := json.MarshalIndent(value, "", " ") + if err != nil { + return false, err + } + if len(data) > maxSelectionBytes { + return false, errors.New("selection metadata exceeds its bound") + } + f, err := os.CreateTemp(filepath.Dir(path), ".storage-selection-*") + if err != nil { + return false, err + } + temp := f.Name() + defer func() { f.Close(); os.Remove(temp) }() + if _, err = f.Write(append(data, '\n')); err != nil { + return false, err + } + if err = f.Sync(); err != nil { + return false, err + } + if err = f.Close(); err != nil { + return false, err + } + if err = replaceSelectionFile(temp, path); err != nil { + return false, err + } + return true, syncSelectionDirectory(filepath.Dir(path)) +} diff --git a/internal/dbconfig/selection_maintenance_test.go b/internal/dbconfig/selection_maintenance_test.go new file mode 100644 index 000000000..a35107dca --- /dev/null +++ b/internal/dbconfig/selection_maintenance_test.go @@ -0,0 +1,48 @@ +package dbconfig + +import ( + "bytes" + "errors" + "os" + "testing" +) + +func TestSelectionMaintenanceLeaseExcludesServicesWithoutChangingRecord(t *testing.T) { + selected := sqliteSelection(t) + path := SelectionPath(selected.StateDir) + if _, err := AdoptSelection(path, selected); err != nil { + t.Fatal(err) + } + before, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + _, runtime, err := OpenSelection(path) + if err != nil { + t.Fatal(err) + } + if _, lease, err := OpenSelectionForMaintenance(path); !errors.Is(err, ErrSelectionBusy) { + if lease != nil { + lease.Close() + } + runtime.Close() + t.Fatal("maintenance accepted running service", err) + } + runtime.Close() + got, lease, err := OpenSelectionForMaintenance(path) + if err != nil || got.Generation != selected.Generation { + t.Fatal("maintenance lease", err) + } + defer lease.Close() + if _, runtime, err := OpenSelection(path); !errors.Is(err, ErrSelectionBusy) { + if runtime != nil { + runtime.Close() + } + t.Fatal("service started during maintenance", err) + } + lease.Close() + after, err := os.ReadFile(path) + if err != nil || !bytes.Equal(before, after) { + t.Fatal("maintenance changed selection", err) + } +} diff --git a/internal/dbconfig/selection_test.go b/internal/dbconfig/selection_test.go new file mode 100644 index 000000000..3238f43ee --- /dev/null +++ b/internal/dbconfig/selection_test.go @@ -0,0 +1,464 @@ +package dbconfig + +import ( + "encoding/json" + "errors" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" +) + +func sqliteSelection(t *testing.T) Selection { + t.Helper() + s, err := ResolveStorage(StorageInputs{FreshInstall: true, BaseDir: t.TempDir()}) + if err != nil { + t.Fatal(err) + } + selection, err := NewSelection(s) + if err != nil { + t.Fatal(err) + } + return selection +} + +func TestSelectionMissingAndAuthoritativeAdoption(t *testing.T) { + want := sqliteSelection(t) + path := SelectionPath(want.StateDir) + if _, _, err := OpenSelection(path); !errors.Is(err, ErrSelectionMissing) { + t.Fatalf("missing record=%v", err) + } + got, err := AdoptSelection(path, want) + if err != nil { + t.Fatal(err) + } + if got.Generation != want.Generation { + t.Fatal("initial generation changed") + } + raw := StorageInputs{BaseDir: t.TempDir(), Backend: Postgres, EnvBackend: Postgres, FreshInstall: true, DBPath: "obsolete.db", UsersDBPath: "obsolete-users.db", StateDir: "old-state", DatabaseURL: "inactive-private-url"} + raw, err = got.ApplyTo(raw) + if err != nil { + t.Fatal(err) + } + resolved, err := ResolveStorage(raw) + if err != nil { + t.Fatal(err) + } + if raw.FreshInstall || raw.ExistingBackend != SQLite || resolved.Backend != SQLite || resolved.DBPath != want.Telemetry.SQLitePath || resolved.UsersDBPath != want.Accounts.SQLitePath || resolved.StateDir != want.StateDir { + t.Fatal("bootstrap settings overrode installed selection") + } + duplicate := want + duplicate.Generation = strings.Repeat("a", 32) + got, err = AdoptSelection(path, duplicate) + if err != nil || got.Generation != want.Generation { + t.Fatalf("idempotent adoption: %v", err) + } + duplicate.Telemetry.SQLitePath = filepath.Join(want.StateDir, "other.db") + if _, err := AdoptSelection(path, duplicate); !errors.Is(err, ErrSelectionChanged) { + t.Fatalf("adoption changed targets: %v", err) + } +} + +func TestSelectionPostgresCredentialsAreNotPersisted(t *testing.T) { + t.Setenv("PGPORT", "") + t.Setenv("PGOPTIONS", "") + t.Setenv("PGSERVICE", "") + base := t.TempDir() + storage, err := ResolveStorage(StorageInputs{BaseDir: base, Backend: Postgres, DatabaseURL: "postgres://first:never-record-password@localhost/telemetry?sslmode=require&search_path=public", UsersDatabaseURL: "postgres://accounts:never-record-password@localhost/accounts"}) + if err != nil { + t.Fatal(err) + } + selected, err := NewSelection(storage) + if err != nil { + t.Fatal(err) + } + data, err := json.Marshal(selected) + if err != nil { + t.Fatal(err) + } + for _, private := range []string{"never-record-password", "sslmode", "postgres://", "first:"} { + if strings.Contains(string(data), private) { + t.Fatal("credentials or TLS settings persisted") + } + } + raw := StorageInputs{BaseDir: base, Backend: SQLite, DBPath: "inactive.db", DatabaseURL: "postgres://rotated:replacement-secret@localhost:5432/telemetry?sslmode=verify-full", UsersDatabaseURL: "postgres://rotated:replacement-secret@localhost/accounts?sslmode=require"} + raw, err = selected.ApplyTo(raw) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(raw.ReaderDatabaseURL, "replacement-secret") || !strings.Contains(raw.ReaderDatabaseURL, "sslmode=verify-full") || !strings.Contains(raw.ReaderDatabaseURL, "search_path=") { + t.Fatal("rotation lost credentials/TLS or left schema unbound") + } + raw.ReaderDatabaseURL = "postgres://reader:secret@localhost/another" + if _, err := selected.ApplyTo(raw); !errors.Is(err, ErrSelectionChanged) { + t.Fatalf("active PG retarget accepted: %v", err) + } +} + +func TestSelectionPinsRecordedPortAndRefusesAmbiguousBootstrap(t *testing.T) { + t.Setenv("PGPORT", "") + t.Setenv("PGOPTIONS", "") + t.Setenv("PGSERVICE", "") + base := t.TempDir() + selected, err := NewSelection(Storage{Backend: Postgres, StateDir: base, WriterDatabaseURL: "postgres://owner:secret@localhost/data"}) + if err != nil { + t.Fatal(err) + } + t.Setenv("PGPORT", "6543") + bound, err := selected.ApplyTo(StorageInputs{BaseDir: base, DatabaseURL: "postgres://rotated:secret@localhost/data"}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(bound.WriterDatabaseURL, "localhost:5432/") { + t.Fatal("recorded port was left to ambient PGPORT") + } + if _, err := NewSelection(Storage{Backend: Postgres, StateDir: base, WriterDatabaseURL: "postgres://owner:secret@localhost/data"}); err == nil { + t.Fatal("ambiguous implicit bootstrap port accepted") + } + if _, err := NewSelection(Storage{Backend: Postgres, StateDir: base, WriterDatabaseURL: "postgres://owner:secret@localhost:5432/data"}); err != nil { + t.Fatal("explicit port refused", err) + } + for _, key := range []string{"PGOPTIONS", "PGSERVICE"} { + t.Setenv(key, "ambiguous") + if _, err := NewSelection(Storage{Backend: Postgres, StateDir: base, WriterDatabaseURL: "postgres://owner:secret@localhost:5432/data"}); err == nil { + t.Fatal("ambient target options accepted", key) + } + t.Setenv(key, "") + } +} + +func TestRecordedAccountAbsenceDoesNotDefaultOrActivateRawSettings(t *testing.T) { + base := t.TempDir() + selected, err := NewSelection(Storage{Backend: SQLite, StateDir: base, DBPath: filepath.Join(base, "telemetry.sqlite")}) + if err != nil { + t.Fatal(err) + } + if selected.Accounts != nil { + t.Fatal("absent accounts recorded as an initialized target") + } + raw, err := selected.ApplyTo(StorageInputs{BaseDir: base, UsersDBPath: "must-not-create.db", UsersDatabaseURL: "inactive"}) + if err != nil { + t.Fatal(err) + } + resolved, err := ResolveStorage(raw) + if err != nil { + t.Fatal(err) + } + if resolved.UsersDBPath != "" { + t.Fatal("recorded account absence defaulted to a new file") + } + bootstrap, err := ResolveStorage(StorageInputs{BaseDir: base, Backend: SQLite, DBPath: filepath.Join(base, "telemetry.sqlite")}) + if err != nil || bootstrap.UsersDBPath == "" { + t.Fatal("bootstrap account default lost", err) + } + pg, err := NewSelection(Storage{Backend: Postgres, StateDir: base, WriterDatabaseURL: "postgres://owner:secret@localhost:5432/data"}) + if err != nil { + t.Fatal(err) + } + raw, err = pg.ApplyTo(StorageInputs{BaseDir: base, DatabaseURL: "postgres://writer:secret@localhost:5432/data", UsersDatabaseURL: "inactive-not-selected", ApprovedChannelsDatabaseURL: "inactive-not-selected"}) + if err != nil { + t.Fatal(err) + } + resolved, err = ResolveStorage(raw) + if err != nil || resolved.UsersDatabaseURL != "" || resolved.ApprovedChannelsDatabaseURL != "" { + t.Fatal("raw credentials activated uninitialized accounts", err) + } +} + +func TestSelectionRuntimeLeasesAndCrashRecovery(t *testing.T) { + current := sqliteSelection(t) + path := SelectionPath(current.StateDir) + if _, err := AdoptSelection(path, current); err != nil { + t.Fatal(err) + } + _, a, err := OpenSelection(path) + if err != nil { + t.Fatal(err) + } + _, b, err := OpenSelection(path) + if err != nil { + t.Fatal(err) + } + if _, err := BeginSelectionUpdate(path, current.Generation); !errors.Is(err, ErrSelectionBusy) { + t.Fatalf("runtime lease did not fence switch: %v", err) + } + a.Close() + b.Close() + u, err := BeginSelectionUpdate(path, current.Generation) + if err != nil { + t.Fatal(err) + } + id := u.ID() + if _, _, err := OpenSelection(path); err == nil { + t.Fatal("runtime entered active switch") + } + if _, err := ResumeSelectionUpdate(path, id); !errors.Is(err, ErrSelectionBusy) { + t.Fatalf("concurrent resume=%v", err) + } + if err := u.Close(); err != nil { + t.Fatal(err) + } // process-exit behavior: journal remains + if _, _, err := OpenSelection(path); !errors.Is(err, ErrSelectionInProgress) { + t.Fatalf("crash journal ignored: %v", err) + } + if _, err := ResumeSelectionUpdate(path, "wrong-job"); err == nil { + t.Fatal("wrong resume accepted") + } + u, err = ResumeSelectionUpdate(path, id) + if err != nil { + t.Fatal(err) + } + if u.Current().Generation != current.Generation { + t.Fatal("resume changed source generation") + } + if err := u.Abort(); err != nil { + t.Fatal(err) + } + u.Close() + got, lease, err := OpenSelection(path) + if err != nil { + t.Fatal(err) + } + defer lease.Close() + if got.Generation != current.Generation { + t.Fatal("abort changed selection") + } +} + +func TestSelectionCommitAndFailedCommitRemainRecoverable(t *testing.T) { + old := sqliteSelection(t) + path := SelectionPath(old.StateDir) + if _, err := AdoptSelection(path, old); err != nil { + t.Fatal(err) + } + u, err := BeginSelectionUpdate(path, old.Generation) + if err != nil { + t.Fatal(err) + } + next := old + next.Generation = strings.Repeat("b", 32) + next.Telemetry.SQLitePath = filepath.Join(old.StateDir, "staged.db") + bad := next + bad.StateDir = t.TempDir() + if replaced, err := u.Commit(bad); err == nil || replaced { + t.Fatal("switch moved state directory") + } + if err := u.Stage(next); err != nil { + t.Fatal(err) + } + if u.Target().Generation != next.Generation { + t.Fatal("staged target absent") + } + view := u.Target() + view.Accounts.SQLitePath = filepath.Join(old.StateDir, "mutated-view.db") + if u.Target().Accounts.SQLitePath != next.Accounts.SQLitePath { + t.Fatal("caller mutated staged target through a shared pointer") + } + changed := next + changed.Telemetry.SQLitePath = filepath.Join(old.StateDir, "substituted.db") + if err := u.Stage(changed); !errors.Is(err, ErrSelectionChanged) { + t.Fatal("staged target changed", err) + } + u.afterReplace = func() error { return errors.New("injected crash after replacement") } + if replaced, err := u.Commit(next); err == nil || !replaced { + t.Fatalf("post-replace result=%v %v", replaced, err) + } + id := u.ID() + u.Close() + if _, _, err := OpenSelection(path); !errors.Is(err, ErrSelectionInProgress) { + t.Fatal("post-replace failure allowed startup") + } + u, err = ResumeSelectionUpdate(path, id) + if err != nil { + t.Fatal(err) + } + if err := u.Abort(); err == nil { + t.Fatal("abort rolled back a changed selection") + } + if replaced, err := u.Commit(Selection{}); err == nil || !replaced { + t.Fatal("failed retry concealed an already replaced selection") + } + if replaced, err := u.Commit(next); err != nil || !replaced { + t.Fatalf("verified retry failed: %v %v", replaced, err) + } + u.Close() + got, lease, err := OpenSelection(path) + if err != nil { + t.Fatal(err) + } + defer lease.Close() + if got.Generation != next.Generation || got.Telemetry.SQLitePath != next.Telemetry.SQLitePath { + t.Fatal("retry did not publish exact verified target") + } +} + +func TestSelectionStatusAndNativeProcessCrash(t *testing.T) { + s := sqliteSelection(t) + path := SelectionPath(s.StateDir) + status, err := InspectSelection(path) + if err != nil || status.State != "unrecorded" || status.Backend != "" { + t.Fatalf("missing status: %+v %v", status, err) + } + if _, err := AdoptSelection(path, s); err != nil { + t.Fatal(err) + } + status, err = InspectSelection(path) + if err != nil || status.State != "ready" || status.Generation != s.Generation || status.Backend != SQLite { + t.Fatalf("ready status: %+v %v", status, err) + } + if status.StateDir != s.StateDir || status.Telemetry == nil || status.Telemetry.SQLitePath != s.Telemetry.SQLitePath || status.Accounts == nil || status.Accounts.SQLitePath != s.Accounts.SQLitePath { + t.Fatal("status omitted selected targets needed for native backups") + } + child := exec.Command(os.Args[0], "-test.run=^TestSelectionCrashChild$") + ready := filepath.Join(t.TempDir(), "ready") + child.Env = append(os.Environ(), "CORESCOPE_SELECTION_CRASH_TEST="+path, "CORESCOPE_SELECTION_CRASH_READY="+ready) + if err := child.Start(); err != nil { + t.Fatal(err) + } + defer func() { child.Process.Kill(); child.Wait() }() + deadline := time.Now().Add(10 * time.Second) + for { + if _, err := os.Stat(ready); err == nil { + break + } + if time.Now().After(deadline) { + t.Fatal("child did not acquire native lease") + } + time.Sleep(10 * time.Millisecond) + } + if _, _, err := OpenSelection(path); !errors.Is(err, ErrSelectionBusy) { + t.Fatalf("another process entered switch lease: %v", err) + } + if err := child.Process.Kill(); err != nil { + t.Fatal(err) + } + child.Wait() + status, err = InspectSelection(path) + if err != nil || status.State != "pending" || status.JobID == "" || status.SourceBackend != SQLite { + t.Fatalf("crashed process status: %+v %v", status, err) + } + if _, _, err := OpenSelection(path); !errors.Is(err, ErrSelectionInProgress) { + t.Fatalf("crash did not retain startup gate: %v", err) + } + u, err := ResumeSelectionUpdate(path, status.JobID) + if err != nil { + t.Fatal(err) + } + if err := u.Abort(); err != nil { + t.Fatal(err) + } + u.Close() + if err := os.WriteFile(path+".switch.json", []byte(`{"version":99}`), 0600); err != nil { + t.Fatal(err) + } + status, err = InspectSelection(path) + if err == nil || status.State != "corrupt" { + t.Fatalf("corrupt status: %+v %v", status, err) + } +} + +func TestSelectionCrashChild(t *testing.T) { + path := os.Getenv("CORESCOPE_SELECTION_CRASH_TEST") + if path == "" { + return + } + s, err := ReadSelection(path) + if err != nil { + t.Fatal(err) + } + u, err := BeginSelectionUpdate(path, s.Generation) + if err != nil { + t.Fatal(err) + } + defer u.Close() + if err := os.WriteFile(os.Getenv("CORESCOPE_SELECTION_CRASH_READY"), []byte("ready"), 0600); err != nil { + t.Fatal(err) + } + time.Sleep(time.Minute) + t.Fatal("parent did not terminate child") +} + +func TestSelectionRefusesHiddenPostgresRetargeting(t *testing.T) { + for _, query := range []string{"host=other", "hostaddr=127.0.0.2", "port=5000", "dbname=other", "database=other", "service=other", "servicefile=other", "options=-csearch_path%3Dother"} { + if _, err := postgresTarget("postgres://role:secret@localhost/telemetry?" + query); err == nil { + t.Errorf("accepted target override %s", query) + } + } +} + +func TestSelectionStatusDoesNotBlockAtomicReplacement(t *testing.T) { + selected := sqliteSelection(t) + path := SelectionPath(selected.StateDir) + if _, err := AdoptSelection(path, selected); err != nil { + t.Fatal(err) + } + stop := make(chan struct{}) + fail := make(chan error, 1) + var reader sync.WaitGroup + reader.Add(1) + go func() { + defer reader.Done() + for { + select { + case <-stop: + return + default: + } + status, err := InspectSelection(path) + if err != nil || (status.State != "ready" && status.State != "pending") { + if err == nil { + err = errors.New("unexpected status during update") + } + select { + case fail <- err: + default: + } + return + } + } + }() + defer func() { + close(stop) + reader.Wait() + select { + case err := <-fail: + t.Error(err) + default: + } + }() + for i := 0; i < 30; i++ { + update, err := BeginSelectionUpdate(path, selected.Generation) + if err != nil { + t.Fatal(err) + } + next := cloneSelection(selected) + next.Generation, err = newSelectionID() + if err != nil { + update.Close() + t.Fatal(err) + } + if _, err := update.Commit(next); err != nil { + update.Close() + t.Fatal(err) + } + update.Close() + selected = next + } +} + +func TestSelectionCorruptionDoesNotDefault(t *testing.T) { + s := sqliteSelection(t) + path := SelectionPath(s.StateDir) + if _, err := AdoptSelection(path, s); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(`{"version":99,"backend":"sqlite"}`), 0600); err != nil { + t.Fatal(err) + } + if _, _, err := OpenSelection(path); err == nil || errors.Is(err, ErrSelectionMissing) { + t.Fatal("invalid record treated as fresh") + } +} diff --git a/internal/dbconfig/selection_unix.go b/internal/dbconfig/selection_unix.go new file mode 100644 index 000000000..c7ed64c73 --- /dev/null +++ b/internal/dbconfig/selection_unix.go @@ -0,0 +1,33 @@ +//go:build linux || darwin || freebsd || openbsd || netbsd || dragonfly + +package dbconfig + +import ( + "errors" + "os" + "syscall" +) + +func openSelectionRead(path string) (*os.File, error) { return os.Open(path) } + +func lockSelectionFile(f *os.File, exclusive bool) error { + flag := syscall.LOCK_SH | syscall.LOCK_NB + if exclusive { + flag = syscall.LOCK_EX | syscall.LOCK_NB + } + err := syscall.Flock(int(f.Fd()), flag) + if errors.Is(err, syscall.EWOULDBLOCK) || errors.Is(err, syscall.EAGAIN) { + return ErrSelectionBusy + } + return err +} +func unlockSelectionFile(f *os.File) error { return syscall.Flock(int(f.Fd()), syscall.LOCK_UN) } +func replaceSelectionFile(source, target string) error { return os.Rename(source, target) } +func syncSelectionDirectory(path string) error { + f, err := os.Open(path) + if err != nil { + return err + } + defer f.Close() + return f.Sync() +} diff --git a/internal/dbconfig/selection_windows.go b/internal/dbconfig/selection_windows.go new file mode 100644 index 000000000..0626c2975 --- /dev/null +++ b/internal/dbconfig/selection_windows.go @@ -0,0 +1,102 @@ +//go:build windows + +package dbconfig + +import ( + "os" + "strings" + "syscall" + "time" + "unsafe" +) + +var selectionKernel = syscall.NewLazyDLL("kernel32.dll") +var selectionLock = selectionKernel.NewProc("LockFileEx") +var selectionUnlock = selectionKernel.NewProc("UnlockFileEx") +var selectionMove = selectionKernel.NewProc("MoveFileExW") + +func openSelectionRead(path string) (*os.File, error) { + name, err := syscall.UTF16PtrFromString(selectionLongPath(path)) + if err != nil { + return nil, err + } + // Inspection must not prevent another process replacing the complete + // metadata file. The open handle still reads the old complete generation. + var handle syscall.Handle + for attempt := 0; attempt < 32; attempt++ { + handle, err = syscall.CreateFile(name, syscall.GENERIC_READ, syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE|syscall.FILE_SHARE_DELETE, nil, syscall.OPEN_EXISTING, syscall.FILE_ATTRIBUTE_NORMAL, 0) + if err != syscall.Errno(5) && err != syscall.Errno(32) { + break + } + time.Sleep(2 * time.Millisecond) + } + if err != nil { + return nil, &os.PathError{Op: "open", Path: path, Err: err} + } + return os.NewFile(uintptr(handle), path), nil +} + +func lockSelectionFile(f *os.File, exclusive bool) error { + flags := uintptr(1) // LOCKFILE_FAIL_IMMEDIATELY + if exclusive { + flags |= 2 + } + var overlap syscall.Overlapped + ok, _, err := selectionLock.Call(f.Fd(), flags, 0, 1, 0, uintptr(unsafe.Pointer(&overlap))) + if ok != 0 { + return nil + } + if err == syscall.Errno(33) || err == syscall.Errno(997) { + return ErrSelectionBusy + } + return err +} + +func unlockSelectionFile(f *os.File) error { + var overlap syscall.Overlapped + ok, _, err := selectionUnlock.Call(f.Fd(), 0, 1, 0, uintptr(unsafe.Pointer(&overlap))) + if ok != 0 { + return nil + } + return err +} + +func selectionLongPath(path string) string { + if strings.HasPrefix(path, `\\?\`) { + return path + } + if strings.HasPrefix(path, `\\`) { + return `\\?\UNC\` + strings.TrimPrefix(path, `\\`) + } + return `\\?\` + path +} + +func replaceSelectionFile(source, target string) error { + from, err := syscall.UTF16PtrFromString(selectionLongPath(source)) + if err != nil { + return err + } + to, err := syscall.UTF16PtrFromString(selectionLongPath(target)) + if err != nil { + return err + } + // Same-directory rename only: replacement + write-through, never copy/delete. + // A just-deleted journal name can remain delete-pending until an inspection + // handle closes. Retry only Windows' transient sharing/access responses; + // never replace via a copy/delete fallback or wait on a long-lived reader. + var moveErr error + for attempt := 0; attempt < 32; attempt++ { + ok, _, err := selectionMove.Call(uintptr(unsafe.Pointer(from)), uintptr(unsafe.Pointer(to)), 9) + if ok != 0 { + return nil + } + moveErr = err + if err != syscall.Errno(5) && err != syscall.Errno(32) { + break + } + time.Sleep(2 * time.Millisecond) + } + return moveErr +} + +func syncSelectionDirectory(string) error { return nil } // MoveFileExW uses WRITE_THROUGH. diff --git a/internal/dbconfig/sqlite_import.go b/internal/dbconfig/sqlite_import.go new file mode 100644 index 000000000..500a97bf8 --- /dev/null +++ b/internal/dbconfig/sqlite_import.go @@ -0,0 +1,23 @@ +package dbconfig + +import ( + "database/sql" + "errors" +) + +// The native resume ledger also fences adoption through a different state +// directory. Conversion removes it only after complete data verification. +const SQLiteImportPendingTable = "corescope_reverse_progress" + +var ErrSQLiteImportIncomplete = errors.New("SQLite target is a staged incomplete conversion; keep it unselected and resume or abort its original offline job") + +func AssertSQLiteImportComplete(db *sql.DB) error { + var count int + if err := db.QueryRow(`SELECT count(*) FROM sqlite_master WHERE name=?1`, SQLiteImportPendingTable).Scan(&count); err != nil { + return err + } + if count != 0 { + return ErrSQLiteImportIncomplete + } + return nil +} diff --git a/internal/dbconfig/sqlite_uri.go b/internal/dbconfig/sqlite_uri.go new file mode 100644 index 000000000..25ba8a63c --- /dev/null +++ b/internal/dbconfig/sqlite_uri.go @@ -0,0 +1,29 @@ +package dbconfig + +import ( + "errors" + "fmt" + "net/url" + "path/filepath" + "runtime" + "strings" +) + +// SQLiteURI encodes a filesystem path as a native file: URI. It does not open +// the file or choose access/durability options; callers supply those explicitly. +// A path is never interpreted as an existing URI or an in-memory database name. +func SQLiteURI(path string, options url.Values) (string, error) { + if path == "" || strings.ContainsRune(path, 0) { + return "", errors.New("SQLite requires a nonempty filesystem path without NUL") + } + abs, err := filepath.Abs(path) + if err != nil { + return "", fmt.Errorf("resolve SQLite filesystem path: %w", err) + } + abs = filepath.ToSlash(abs) + if runtime.GOOS == "windows" && !strings.HasPrefix(abs, "//") { + abs = "/" + abs + } + u := url.URL{Scheme: "file", Path: abs, RawQuery: options.Encode()} + return u.String(), nil +} diff --git a/internal/dbconfig/sqlite_uri_test.go b/internal/dbconfig/sqlite_uri_test.go new file mode 100644 index 000000000..cbd2a6d10 --- /dev/null +++ b/internal/dbconfig/sqlite_uri_test.go @@ -0,0 +1,66 @@ +package dbconfig + +import ( + "net/url" + "path/filepath" + "reflect" + "runtime" + "strings" + "testing" +) + +func TestSQLiteURILiteralPathsAndOptions(t *testing.T) { + for _, name := range []string{"plain.db", "literal # ? % space µ.db"} { + path := filepath.Join(t.TempDir(), name) + opts := url.Values{"mode": {"ro"}, "_pragma": {"busy_timeout(5000)", "foreign_keys(1)"}} + before := opts.Encode() + text, err := SQLiteURI(path, opts) + if err != nil { + t.Fatal(err) + } + u, err := url.Parse(text) + if err != nil { + t.Fatal(err) + } + if u.Scheme != "file" || u.Host != "" || u.Fragment != "" || !reflect.DeepEqual(u.Query(), opts) || opts.Encode() != before { + t.Fatal("URI changed path boundaries/options") + } + want := filepath.ToSlash(path) + if runtime.GOOS == "windows" && !strings.HasPrefix(want, "//") { + want = "/" + want + } + if u.Path != want { + t.Fatalf("path=%q want=%q", u.Path, want) + } + if strings.Contains(text, "#") || strings.Contains(text, " µ") { + t.Fatal("literal path was not escaped") + } + } +} + +func TestSQLiteURIInvalidAndRelativePaths(t *testing.T) { + for _, path := range []string{"", "bad\x00name"} { + if _, err := SQLiteURI(path, nil); err == nil { + t.Fatal("invalid path accepted") + } + } + text, err := SQLiteURI("relative.sqlite", nil) + if err != nil { + t.Fatal(err) + } + u, err := url.Parse(text) + if err != nil { + t.Fatal(err) + } + want, err := filepath.Abs("relative.sqlite") + if err != nil { + t.Fatal(err) + } + want = filepath.ToSlash(want) + if runtime.GOOS == "windows" && !strings.HasPrefix(want, "//") { + want = "/" + want + } + if u.Path != want { + t.Fatalf("relative path=%q want=%q", u.Path, want) + } +} diff --git a/internal/dbconfig/storage.go b/internal/dbconfig/storage.go new file mode 100644 index 000000000..092cbb3d0 --- /dev/null +++ b/internal/dbconfig/storage.go @@ -0,0 +1,174 @@ +package dbconfig + +import ( + "errors" + "net/url" + "path/filepath" + "runtime" + "strconv" + "strings" +) + +type Backend string + +const ( + SQLite Backend = "sqlite" + Postgres Backend = "postgres" +) + +// Parameter emits a numbered positional placeholder. SQLite drivers differ in +// how they bind dollar names; question-mark numbers work positionally in both. +// Construct queries once when preparing; this is not a SQL-text rewriting API. +// Invalid backend/position values are programming errors. +func (b Backend) Parameter(position int) string { + if position < 1 { + panic("database parameter position must be positive") + } + switch b { + case SQLite: + return "?" + strconv.Itoa(position) + case Postgres: + return "$" + strconv.Itoa(position) + default: + panic("database parameter requires a validated backend") + } +} + +// StorageInputs contains raw configured values, before defaults are inserted. +// The caller supplies installation facts; resolution does no filesystem, +// environment or database reads. ExistingBackend is the recorded choice, +// and FreshInstall must not be inferred from a failed database connection. +type StorageInputs struct { + Backend, EnvBackend, ExistingBackend Backend + FreshInstall bool + BaseDir, StateDir string + DBPath, UsersDBPath string + DatabaseURL, ReaderDatabaseURL, WriterDatabaseURL string + UsersDatabaseURL, ApprovedChannelsDatabaseURL string +} + +// Storage contains only the selected engine's targets. Role URLs can be empty +// when that service/optional feature is not configured; the caller opening a +// connection must require its role's URL. Never log this value: URLs may carry +// credentials. Runtime openers retain their engine-specific validation and +// read-only/role checks, including actual telemetry/account identity checks. +type Storage struct { + Backend Backend + StateDir string + DBPath, UsersDBPath string + ReaderDatabaseURL, WriterDatabaseURL string + UsersDatabaseURL, ApprovedChannelsDatabaseURL string +} + +func ResolveStorage(in StorageInputs) (Storage, error) { + var out Storage + for _, backend := range []Backend{in.Backend, in.EnvBackend, in.ExistingBackend} { + if backend != "" && backend != SQLite && backend != Postgres { + return out, errors.New("unsupported database backend; choose sqlite or postgres") + } + } + selected := in.Backend + if in.EnvBackend != "" { + if selected != "" && selected != in.EnvBackend { + return out, errors.New("database backend conflicts between configuration and environment") + } + selected = in.EnvBackend + } + if in.ExistingBackend != "" { + if selected != "" && selected != in.ExistingBackend { + return out, errors.New("changing an existing database backend requires verified offline conversion") + } + selected = in.ExistingBackend + } + if selected == "" { + hasSQLite := in.DBPath != "" || in.UsersDBPath != "" + hasPostgres := in.DatabaseURL != "" || in.ReaderDatabaseURL != "" || in.WriterDatabaseURL != "" || in.UsersDatabaseURL != "" || in.ApprovedChannelsDatabaseURL != "" + switch { + case hasSQLite && hasPostgres: + return out, errors.New("ambiguous legacy database settings; record the existing backend explicitly") + case hasPostgres: + selected = Postgres + case hasSQLite || in.FreshInstall: + selected = SQLite + default: + return out, errors.New("existing database backend is unknown; supply the recorded choice or unambiguous legacy settings") + } + } + if !filepath.IsAbs(in.BaseDir) { + return out, errors.New("storage base directory must be absolute") + } + out.Backend = selected + stateDir := in.StateDir + if selected == SQLite { + path := in.DBPath + if path == "" { + path = filepath.Join("data", "meshcore.db") + } + var err error + if out.DBPath, err = storagePath(in.BaseDir, path); err != nil { + return Storage{}, err + } + path = in.UsersDBPath + if path == "" && in.ExistingBackend == "" { + path = filepath.Join(filepath.Dir(out.DBPath), "users.db") + } + if path != "" { + if out.UsersDBPath, err = storagePath(in.BaseDir, path); err != nil { + return Storage{}, err + } + } + same := out.DBPath == out.UsersDBPath + if runtime.GOOS == "windows" { + same = strings.EqualFold(out.DBPath, out.UsersDBPath) + } + if same { + return Storage{}, errors.New("telemetry and accounts require separate database paths") + } + if stateDir == "" { + stateDir = filepath.Dir(out.DBPath) + } + } else { + out.ReaderDatabaseURL = in.ReaderDatabaseURL + if out.ReaderDatabaseURL == "" { + out.ReaderDatabaseURL = in.DatabaseURL + } + out.WriterDatabaseURL = in.WriterDatabaseURL + if out.WriterDatabaseURL == "" { + out.WriterDatabaseURL = in.DatabaseURL + } + if out.ReaderDatabaseURL == "" && out.WriterDatabaseURL == "" { + return Storage{}, errors.New("PostgreSQL is selected but its telemetry connection settings are missing") + } + out.UsersDatabaseURL, out.ApprovedChannelsDatabaseURL = in.UsersDatabaseURL, in.ApprovedChannelsDatabaseURL + for _, value := range []string{out.ReaderDatabaseURL, out.WriterDatabaseURL, out.UsersDatabaseURL, out.ApprovedChannelsDatabaseURL} { + if value == "" { + continue + } + u, err := url.Parse(value) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") || u.Host == "" || strings.Trim(u.Path, "/") == "" { + return Storage{}, errors.New("PostgreSQL targets require explicit PostgreSQL database URLs") + } + } + if stateDir == "" { + stateDir = "data" + } + } + var err error + if out.StateDir, err = storagePath(in.BaseDir, stateDir); err != nil { + return Storage{}, err + } + return out, nil +} + +func storagePath(base, path string) (string, error) { + if strings.Contains(path, "://") || strings.HasPrefix(path, "postgres:") || strings.HasPrefix(path, "postgresql:") { + return "", errors.New("a database URL cannot be used as a storage path") + } + if filepath.IsAbs(path) { + return filepath.Clean(path), nil + } + if filepath.VolumeName(path) != "" { + return "", errors.New("drive-relative storage paths are ambiguous; use an absolute path") + } + return filepath.Join(base, path), nil +} diff --git a/internal/dbconfig/storage_config.go b/internal/dbconfig/storage_config.go new file mode 100644 index 000000000..22620eba0 --- /dev/null +++ b/internal/dbconfig/storage_config.go @@ -0,0 +1,40 @@ +package dbconfig + +import ( + "encoding/json" + "errors" + "io" + "os" +) + +// ReadStorageConfig reads just the shared storage fields, including disabled +// account stores. It preserves raw paths; callers apply their explicit config +// discovery, environment precedence and ResolveStorage exactly once. +func ReadStorageConfig(path string) (StorageInputs, error) { + f, err := os.Open(path) + if err != nil { + return StorageInputs{}, err + } + defer f.Close() + var config *struct { + DB DBConfig `json:"db"` + DBPath string `json:"dbPath"` + DatabaseURL string `json:"databaseURL"` + StateDir string `json:"stateDir"` + Users struct { + DBPath string `json:"dbPath"` + DatabaseURL string `json:"databaseURL"` + ApprovedChannelsDatabaseURL string `json:"approvedChannelsDatabaseURL"` + } `json:"userManagement"` + } + const maxConfigBytes = 4 * 1024 * 1024 + limit := &io.LimitedReader{R: f, N: maxConfigBytes + 1} + decoder := json.NewDecoder(limit) + if err := decoder.Decode(&config); err != nil || config == nil { + return StorageInputs{}, errors.New("invalid storage configuration JSON") + } + if err := decoder.Decode(new(any)); err != io.EOF || limit.N <= 0 { + return StorageInputs{}, errors.New("storage configuration exceeds its bound or has trailing JSON") + } + return StorageInputs{Backend: config.DB.Backend, DBPath: config.DBPath, DatabaseURL: config.DatabaseURL, StateDir: config.StateDir, UsersDBPath: config.Users.DBPath, UsersDatabaseURL: config.Users.DatabaseURL, ApprovedChannelsDatabaseURL: config.Users.ApprovedChannelsDatabaseURL}, nil +} diff --git a/internal/dbconfig/storage_config_test.go b/internal/dbconfig/storage_config_test.go new file mode 100644 index 000000000..5d53869a2 --- /dev/null +++ b/internal/dbconfig/storage_config_test.go @@ -0,0 +1,32 @@ +package dbconfig + +import ( + "os" + "path/filepath" + "testing" +) + +func TestReadStorageConfigKeepsRawPathsAndDisabledAccounts(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.json") + if err := os.WriteFile(path, []byte(`{"db":{"backend":"sqlite"},"dbPath":"relative.db","databaseURL":"inactive","stateDir":"stable","userManagement":{"enabled":false,"dbPath":"accounts.db","databaseURL":"runtime","approvedChannelsDatabaseURL":"reader"},"port":3000}`), 0600); err != nil { + t.Fatal(err) + } + got, err := ReadStorageConfig(path) + if err != nil { + t.Fatal(err) + } + if got.Backend != SQLite || got.DBPath != "relative.db" || got.UsersDBPath != "accounts.db" || got.StateDir != "stable" || got.UsersDatabaseURL != "runtime" || got.ApprovedChannelsDatabaseURL != "reader" || got.DatabaseURL != "inactive" { + t.Fatal("storage inputs were defaulted or dropped") + } + for _, invalid := range []string{`{"dbPath":8}`, `{} {}`, `null`, `{"db":{"backend":42}}`} { + if err := os.WriteFile(path, []byte(invalid), 0600); err != nil { + t.Fatal(err) + } + if _, err := ReadStorageConfig(path); err == nil { + t.Fatal("invalid explicit config accepted", invalid) + } + } + if _, err := ReadStorageConfig(filepath.Join(t.TempDir(), "missing")); err == nil { + t.Fatal("missing explicit file accepted") + } +} diff --git a/internal/dbconfig/storage_test.go b/internal/dbconfig/storage_test.go new file mode 100644 index 000000000..b322585b4 --- /dev/null +++ b/internal/dbconfig/storage_test.go @@ -0,0 +1,107 @@ +package dbconfig + +import ( + "path/filepath" + "strings" + "testing" +) + +func TestStorageSelection(t *testing.T) { + base := t.TempDir() + pg := "postgres://reader:private-value@localhost/telemetry" + cases := []struct { + name string + in StorageInputs + want Backend + err bool + }{ + {"fresh", StorageInputs{FreshInstall: true}, SQLite, false}, + {"unknown existing", StorageInputs{}, "", true}, + {"legacy sqlite", StorageInputs{DBPath: "old/mesh.db"}, SQLite, false}, + {"legacy postgres", StorageInputs{DatabaseURL: pg}, Postgres, false}, + {"legacy conflict", StorageInputs{DBPath: "old.db", DatabaseURL: pg}, "", true}, + {"recorded postgres missing url", StorageInputs{ExistingBackend: Postgres, FreshInstall: true}, "", true}, + {"recorded postgres", StorageInputs{ExistingBackend: Postgres, ReaderDatabaseURL: pg}, Postgres, false}, + {"ordinary backend change", StorageInputs{Backend: SQLite, ExistingBackend: Postgres}, "", true}, + {"selector conflict", StorageInputs{Backend: SQLite, EnvBackend: Postgres, DatabaseURL: pg}, "", true}, + {"unknown selector", StorageInputs{Backend: "mysql", FreshInstall: true}, "", true}, + {"explicit sqlite ignores inactive pg", StorageInputs{Backend: SQLite, DatabaseURL: "private-invalid-pg"}, SQLite, false}, + {"explicit postgres ignores old path", StorageInputs{Backend: Postgres, DBPath: "old.db", DatabaseURL: pg}, Postgres, false}, + {"postgres path is not url", StorageInputs{Backend: Postgres, DatabaseURL: "old.db"}, "", true}, + {"sqlite url is not path", StorageInputs{Backend: SQLite, DBPath: pg}, "", true}, + {"accounts alone cannot configure telemetry", StorageInputs{UsersDatabaseURL: pg}, "", true}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + c.in.BaseDir = base + got, err := ResolveStorage(c.in) + if (err != nil) != c.err { + t.Fatalf("error = %v, want error %v", err, c.err) + } + if err != nil { + if strings.Contains(err.Error(), "private-") { + t.Fatal("error leaked connection value") + } + return + } + if got.Backend != c.want { + t.Fatalf("backend=%q, want %q", got.Backend, c.want) + } + }) + } +} + +func TestStorageTargets(t *testing.T) { + base := t.TempDir() + got, err := ResolveStorage(StorageInputs{BaseDir: base, Backend: SQLite, DBPath: "legacy/mesh.db"}) + if err != nil { + t.Fatal(err) + } + if got.DBPath != filepath.Join(base, "legacy", "mesh.db") || got.UsersDBPath != filepath.Join(base, "legacy", "users.db") || got.StateDir != filepath.Join(base, "legacy") { + t.Fatalf("SQLite paths: %+v", got) + } + if _, err := ResolveStorage(StorageInputs{BaseDir: base, Backend: SQLite, DBPath: "same.db", UsersDBPath: "same.db"}); err == nil { + t.Fatal("accepted same telemetry/account target") + } + if _, err := ResolveStorage(StorageInputs{BaseDir: "relative", Backend: SQLite}); err == nil { + t.Fatal("accepted non-absolute base instead of caller-resolved location") + } + common := "postgres://common@localhost/telemetry" + reader := "postgres://reader@localhost/telemetry" + got, err = ResolveStorage(StorageInputs{BaseDir: base, Backend: Postgres, DatabaseURL: common, ReaderDatabaseURL: reader, StateDir: "state"}) + if err != nil { + t.Fatal(err) + } + if got.ReaderDatabaseURL != reader || got.WriterDatabaseURL != common || got.StateDir != filepath.Join(base, "state") || got.DBPath != "" || got.UsersDBPath != "" { + t.Fatal("PostgreSQL roles/state were not resolved independently of file paths") + } + got, err = ResolveStorage(StorageInputs{BaseDir: base, FreshInstall: true}) + if err != nil { + t.Fatal(err) + } + if got.DBPath != filepath.Join(base, "data", "meshcore.db") || got.UsersDBPath != filepath.Join(base, "data", "users.db") { + t.Fatal("fresh SQLite defaults changed") + } +} + +func TestBackendParameters(t *testing.T) { + for _, backend := range []Backend{SQLite, Postgres} { + prefix := "$" + if backend == SQLite { + prefix = "?" + } + if got := backend.Parameter(12); got != prefix+"12" { + t.Fatalf("parameter=%q", got) + } + } + for _, call := range []func(){func() { SQLite.Parameter(0) }, func() { Backend("unknown").Parameter(1) }} { + func() { + defer func() { + if recover() == nil { + t.Error("invalid parameter construction did not fail") + } + }() + call() + }() + } +} diff --git a/internal/dbschema/backend_test.go b/internal/dbschema/backend_test.go new file mode 100644 index 000000000..f05d72737 --- /dev/null +++ b/internal/dbschema/backend_test.go @@ -0,0 +1,26 @@ +package dbschema + +import ( + "os" + "testing" + + "github.com/meshcore-analyzer/pgutil/pgtest" +) + +// PostgreSQL-only assertions belong to the explicitly selected PG matrix. +// A missing URL in that matrix is an error, never a skipped integration test. +func postgresSchema(t testing.TB) string { + t.Helper() + switch os.Getenv("CORESCOPE_TEST_BACKEND") { + case "", "sqlite": + t.Skip("PostgreSQL-specific test; select CORESCOPE_TEST_BACKEND=postgres") + case "postgres": + if os.Getenv("CORESCOPE_TEST_POSTGRES_URL") == "" { + t.Fatal("PostgreSQL test matrix requires CORESCOPE_TEST_POSTGRES_URL") + } + return pgtest.NewSchema(t) + default: + t.Fatal("CORESCOPE_TEST_BACKEND must be sqlite or postgres") + } + return "" +} diff --git a/internal/dbschema/collation_test.go b/internal/dbschema/collation_test.go new file mode 100644 index 000000000..23239ee55 --- /dev/null +++ b/internal/dbschema/collation_test.go @@ -0,0 +1,30 @@ +package dbschema + +import ( + "testing" + + "github.com/meshcore-analyzer/pgutil" +) + +func TestTelemetryTextUsesBinaryCollation(t *testing.T) { + db, err := pgutil.Open(postgresSchema(t), false) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { db.Close() }) + if err := Apply(db, t.Logf); err != nil { + t.Fatal(err) + } + var inherited int + err = db.QueryRow(`SELECT COUNT(*) FROM pg_attribute a + JOIN pg_class c ON c.oid=a.attrelid JOIN pg_namespace n ON n.oid=c.relnamespace + WHERE n.nspname=current_schema() AND c.relkind='r' AND c.relname<>'corescope_schema' + AND a.attnum>0 AND NOT a.attisdropped AND a.atttypid='text'::regtype + AND a.attcollation<>'"C"'::regcollation`).Scan(&inherited) + if err != nil { + t.Fatal(err) + } + if inherited != 0 { + t.Fatalf("%d telemetry text columns inherit host-specific collation instead of SQLite byte order", inherited) + } +} diff --git a/internal/dbschema/dbschema.go b/internal/dbschema/dbschema.go index c6885b02a..e7463e6aa 100644 --- a/internal/dbschema/dbschema.go +++ b/internal/dbschema/dbschema.go @@ -1,759 +1,275 @@ -// Package dbschema centralizes schema migrations and read-side schema -// assertions for the CoreScope SQLite DB. Per issue #1287 the writer -// (cmd/ingestor) owns ALL CREATE/ALTER/INSERT/UPDATE/DELETE on schema -// objects; the server (cmd/server) only ASSERTS that the schema is in -// the expected shape and refuses to start otherwise. -// -// Apply(rw, log) runs from the ingestor at startup BEFORE subscribing to -// MQTT. AssertReady(ro) runs from the server at startup and returns an -// error listing every missing column/index/table. -// -// INVARIANT (#1321): Any optional column the SERVER detects via PRAGMA -// (e.g. cmd/server/db.go detectSchema → hasScopeName, hasDefaultScope, -// hasObsRawHex, hasResolvedPath) MUST be added here, in Apply, AND -// asserted in AssertReady. Adding it only to cmd/ingestor/db.go -// applySchema reintroduces the startup-race bug from #1321: the -// server's PRAGMA can run before the ingestor finishes applySchema, -// the boolean caches `false`, and feature endpoints permanently 500 -// until the server restarts. Source of truth lives here — full stop. +// Package dbschema owns telemetry schema entrypoints for SQLite and PostgreSQL. +// SQLite's writer applies migrations; PostgreSQL uses offline bootstrap. Readers +// only assert readiness. Apply and AssertReady retain PostgreSQL compatibility. package dbschema import ( + "context" "database/sql" - "errors" + _ "embed" "fmt" "strings" ) -// Logger is the minimal logging surface used by Apply. Both cmd/server -// and cmd/ingestor satisfy this with the stdlib `log` package's Printf -// (passed as a closure to avoid an indirect log dependency here). -type Logger func(format string, args ...interface{}) +type Logger func(string, ...interface{}) -// Apply runs every server-side ensure_* migration against the given -// read-write SQLite connection. Each operation is idempotent -// (IF NOT EXISTS / column-probe-before-ALTER). Safe to call repeatedly. +// SchemaSQL is inspectable native PostgreSQL DDL, shared with the importer. // -// Called by the ingestor at startup. The server MUST NOT call this — -// it only calls AssertReady. -func Apply(rw *sql.DB, logf Logger) error { - if logf == nil { - logf = func(string, ...interface{}) {} - } - if err := ensureServerIndexes(rw); err != nil { - return fmt.Errorf("ensure server indexes: %w", err) - } - if err := ensureObservationsDedupIndex(rw, logf); err != nil { - return fmt.Errorf("ensure observations dedup index: %w", err) - } - if err := ensureNeighborEdgesTable(rw); err != nil { - return fmt.Errorf("ensure neighbor_edges: %w", err) - } - if err := ensureInactiveNodesTable(rw); err != nil { - return fmt.Errorf("ensure inactive_nodes: %w", err) - } - if err := ensureResolvedPathColumn(rw, logf); err != nil { - return fmt.Errorf("ensure resolved_path: %w", err) - } - if err := ensureObserverInactiveColumn(rw, logf); err != nil { - return fmt.Errorf("ensure observers.inactive: %w", err) - } - if err := ensureLastPacketAtColumn(rw, logf); err != nil { - return fmt.Errorf("ensure observers.last_packet_at: %w", err) - } - if err := ensureObserverIATAColumn(rw, logf); err != nil { - return fmt.Errorf("ensure observers.iata: %w", err) - } - // Must run after ensureObserverIATAColumn: the expression index below - // is on observers.iata, which that step guarantees exists. - if err := ensureChannelIndexes(rw); err != nil { - return fmt.Errorf("ensure channel indexes: %w", err) - } - if err := ensureForeignAdvertColumn(rw, logf); err != nil { - return fmt.Errorf("ensure foreign_advert: %w", err) - } - if err := ensureFromPubkeyColumn(rw, logf); err != nil { - return fmt.Errorf("ensure from_pubkey: %w", err) - } - if err := ensureScopeNameColumn(rw, logf); err != nil { - return fmt.Errorf("ensure scope_name: %w", err) - } - if err := ensureDefaultScopeColumns(rw, logf); err != nil { - return fmt.Errorf("ensure default_scope: %w", err) - } - if err := ensureConfiguredScopeColumns(rw, logf); err != nil { - return fmt.Errorf("ensure configured_scope: %w", err) - } - if err := ensureObservationsRawHexColumn(rw, logf); err != nil { - return fmt.Errorf("ensure observations.raw_hex: %w", err) - } - if err := ensureMultibyteCapColumns(rw, logf); err != nil { - return fmt.Errorf("ensure multibyte_cap columns: %w", err) - } - if err := ensureObserverNaiveClockColumns(rw, logf); err != nil { - return fmt.Errorf("ensure observers naive-clock columns: %w", err) - } - if err := ensureObserverCanRelayColumn(rw, logf); err != nil { - return fmt.Errorf("ensure observers.can_relay: %w", err) - } - if err := ensureObserverCanRelaySeenColumn(rw, logf); err != nil { - return fmt.Errorf("ensure observers.can_relay_seen: %w", err) - } - // #1690: denormalized last_seen on transmissions so cold-load filters - // on effective recency rather than first-ever first_seen. The column - // add + index creation are cheap (single ALTER, indexed INTEGER - // column); the *backfill* from observations is potentially expensive - // and runs via RunAsyncMigration from the ingestor (see - // cmd/ingestor/db.go OpenStore). - if err := ensureTransmissionsLastSeenColumn(rw, logf); err != nil { - return fmt.Errorf("ensure transmissions.last_seen: %w", err) - } - if err := ensureAdvertEvidence(rw); err != nil { - return fmt.Errorf("ensure advert route evidence: %w", err) - } - if err := ensureKeyBackfill(rw); err != nil { - return fmt.Errorf("ensure key backfill: %w", err) - } - // Advert evidence is intentionally not required by AssertReady: a legacy - // read-only server reports unknown until this table/backfill is available, - // and re-probes absence instead of caching the startup race permanently. - // tx_rewrite_feed (#2107) follows the same rule for the same reason: a - // server without it simply never sees in-place rewrites until restart, - // which is exactly the behaviour before the feed existed. - return nil -} +//go:embed schema.sql +var SchemaSQL string -// AssertReady verifies the schema is in the expected shape. The server -// calls this at startup against a read-only connection; if it returns -// non-nil, the server MUST fatal-log and exit so the operator restarts -// the ingestor (which owns migrations). -func AssertReady(ro *sql.DB) error { - var missing []string +const Version = 2 - mustCol := func(table, col string) { - has, err := TableHasColumn(ro, table, col) - if err != nil { - missing = append(missing, fmt.Sprintf("%s.%s (probe error: %v)", table, col, err)) - return - } - if !has { - missing = append(missing, fmt.Sprintf("%s.%s", table, col)) - } - } - mustTable := func(name string) { - var n int - err := ro.QueryRow(`SELECT 1 FROM sqlite_master WHERE type='table' AND name=?`, name).Scan(&n) - if errors.Is(err, sql.ErrNoRows) { - missing = append(missing, "table:"+name) - } else if err != nil { - missing = append(missing, fmt.Sprintf("table:%s (probe error: %v)", name, err)) - } - } - - mustTable("neighbor_edges") - mustCol("observations", "resolved_path") - mustCol("observers", "inactive") - mustCol("observers", "last_packet_at") - mustCol("observers", "iata") - mustCol("nodes", "foreign_advert") - mustCol("inactive_nodes", "foreign_advert") - mustCol("transmissions", "from_pubkey") - // #1690: denormalized recency axis for cold-load. Owned by ingestor - // (cmd/ingestor/db.go OpenStore: ALTER + index + async backfill); - // server reads it to filter the hot-window query. - mustCol("transmissions", "last_seen") - // #1321: server's detectSchema PRAGMA-detects these and caches a - // boolean. To kill the startup race they're owned + asserted here. - mustCol("transmissions", "scope_name") - mustCol("nodes", "default_scope") - mustCol("inactive_nodes", "default_scope") - // #1865: confirmed region scopes from the observer /neighbors report. - mustCol("nodes", "configured_scope") - mustCol("inactive_nodes", "configured_scope") - mustCol("observations", "raw_hex") - // Multi-byte capability cache (#1324 follow-up; PR #903 surface). - // Owned by ingestor — server reads these for O(1) /api/nodes - // enrichment, ingestor's RunMultibyteCapPersist is the only writer. - mustCol("nodes", "multibyte_sup") - mustCol("nodes", "multibyte_evidence") - mustCol("inactive_nodes", "multibyte_sup") - mustCol("inactive_nodes", "multibyte_evidence") - // Issue #1478: per-observer naive-clock skew tracking. Server reads - // all three to populate ObserverResp.ClockNaive / ClockSkew* fields. - mustCol("observers", "clock_skew_seconds") - mustCol("observers", "clock_skew_count_24h") - mustCol("observers", "clock_last_naive_at") - // Issue #1290: firmware 1.16 publishes a `repeat: on|off` flag in - // the MQTT /status JSON. Ingestor persists it as can_relay; server - // reads it to filter listener-only observers out of the path-hop - // disambiguator candidate set. Default 1 preserves prior behavior - // for legacy observers that never sent the field. - mustCol("observers", "can_relay") - // Issue #1290 follow-up (PR #1624 MAJOR-2): tri-state badge. The - // can_relay column defaults to 1 at INSERT, so we cannot distinguish - // "firmware confirmed repeater" from "legacy observer that never - // sent a repeat field". can_relay_seen=1 means the ingestor wrote - // an explicit value; can_relay_seen=0 means leave the UI badge - // unset (unknown state). - mustCol("observers", "can_relay_seen") +// WriterLockKey serializes ID allocation through commit, including backfills, +// across ingestor processes. The server's ID cursor relies on this ordering. +const WriterLockKey int64 = 18950410699801933 - if len(missing) > 0 { - return fmt.Errorf("schema not migrated by ingestor; restart ingestor first. missing: %s", - strings.Join(missing, ", ")) - } - return nil -} - -// Querier is the read surface shared by *sql.DB and *sql.Tx. -// -// Not *sql.Conn: it exposes only QueryContext/QueryRowContext, so it does not -// satisfy this. Widen the interface to the Context variants if a caller ever -// needs one. -// -// It exists so schema probes can run on whichever handle the caller already -// holds. Taking *sql.DB unconditionally is a deadlock waiting to happen: a -// caller inside a transaction has the connection checked out, and on a pool -// capped at one connection — which is what cmd/ingestor runs — a probe against -// the pool waits forever for the connection its own transaction is holding. type Querier interface { - Query(query string, args ...any) (*sql.Rows, error) - QueryRow(query string, args ...any) *sql.Row + Query(string, ...any) (*sql.Rows, error) + QueryRow(string, ...any) *sql.Row } -// TableHasColumn reports whether the given table has the given column. -// Exported because tests and the read-side need it without re-implementing. -// -// Pass the transaction, not the pool, when you are inside one. See Querier. -func TableHasColumn(db Querier, table, column string) (bool, error) { - rows, err := db.Query(fmt.Sprintf("PRAGMA table_info(%s)", table)) - if err != nil { - return false, err - } - defer rows.Close() - for rows.Next() { - var cid int - var name string - var ctype sql.NullString - var notnull, pk int - var dflt sql.NullString - if err := rows.Scan(&cid, &name, &ctype, ¬null, &dflt, &pk); err != nil { - return false, err - } - if name == column { - return true, nil - } - } - return false, rows.Err() -} +func Apply(db *sql.DB, logf Logger) error { return apply(db, logf, true) } -// ─── ensure_* helpers (writer side) ──────────────────────────────────────── +// ApplyForImport atomically creates an unready target; runtime cannot see a +// temporary ready-empty schema while the importer starts its first batch. +func ApplyForImport(db *sql.DB, logf Logger) error { return apply(db, logf, false) } -func ensureServerIndexes(rw *sql.DB) error { - stmts := []string{ - `CREATE INDEX IF NOT EXISTS idx_transmissions_first_seen ON transmissions(first_seen)`, - `CREATE INDEX IF NOT EXISTS idx_transmissions_hash ON transmissions(hash)`, - `CREATE INDEX IF NOT EXISTS idx_transmissions_payload_type ON transmissions(payload_type)`, - `CREATE INDEX IF NOT EXISTS idx_observations_timestamp ON observations(timestamp)`, - `CREATE INDEX IF NOT EXISTS idx_observations_transmission_id ON observations(transmission_id)`, - // Composite covers GetChannelMessages' grouped MAX(timestamp) per - // transmission_id (issue #1366 / PR #1368). With this index sqlite can - // satisfy the aggregate index-only without touching the heap. - `CREATE INDEX IF NOT EXISTS idx_observations_tx_ts ON observations(transmission_id, timestamp)`, - } - for _, s := range stmts { - if _, err := rw.Exec(s); err != nil { - return fmt.Errorf("ensure index %q: %w", s, err) - } - } - // observer_idx (v3) vs observer_id (v2) — probe + index the matching one. - hasIdx, err := TableHasColumn(rw, "observations", "observer_idx") +// ApplyForImportTx lets the importer commit schema and source/progress metadata +// in one transaction. The caller owns commit/rollback; no ready window exists. +func ApplyForImportTx(tx *sql.Tx, _ Logger) error { return applyTx(tx, false) } +func apply(db *sql.DB, logf Logger, initialReady bool) error { + tx, err := db.BeginTx(context.Background(), nil) if err != nil { return err } - if hasIdx { - if _, err := rw.Exec(`CREATE INDEX IF NOT EXISTS idx_observations_observer_idx ON observations(observer_idx)`); err != nil { - return err - } - } - hasID, err := TableHasColumn(rw, "observations", "observer_id") - if err != nil { + defer tx.Rollback() + if err := applyTx(tx, initialReady); err != nil { return err } - if hasID { - if _, err := rw.Exec(`CREATE INDEX IF NOT EXISTS idx_observations_observer_id ON observations(observer_id)`); err != nil { - return err - } - } - return nil -} - -// ensureChannelIndexes speeds up /api/channels and -// /api/channels/{hash}/messages. -// -// - idx_transmissions_channel_hash_payload adds first_seen to the -// existing channel_hash index (cmd/ingestor/db.go migration -// 'channel_hash_v1', idx_tx_channel_hash) so GetChannels' "latest -// message" correlated subquery (channel_hash + payload_type, ORDER BY -// first_seen DESC LIMIT 1) and GetChannelMessages' count/page queries -// resolve as index-order scans instead of sorting per channel. -// - idx_observers_iata_norm indexes the UPPER(TRIM(iata)) expression -// that GetChannels/GetEncryptedChannels/GetChannelMessages use for -// region filtering; a plain index on iata can't be used under that -// expression, so the expression itself is indexed. -// -// transmissions.channel_hash itself is added by the ingestor's legacy -// 'channel_hash_v1' migration (cmd/ingestor/db.go), not by this package, -// so it isn't guaranteed to exist yet on every DB Apply runs against — -// probe before indexing, same pattern as the observer_idx/observer_id -// probe in ensureServerIndexes above. -func ensureChannelIndexes(rw *sql.DB) error { - hasChannelHash, err := TableHasColumn(rw, "transmissions", "channel_hash") - if err != nil { + if err := tx.Commit(); err != nil { return err } - if hasChannelHash { - if _, err := rw.Exec(`CREATE INDEX IF NOT EXISTS idx_transmissions_channel_hash_payload ON transmissions(channel_hash, payload_type, first_seen)`); err != nil { - return fmt.Errorf("ensure idx_transmissions_channel_hash_payload: %w", err) + if initialReady { + if err := AssertReady(db); err != nil { + return err } } - if _, err := rw.Exec(`CREATE INDEX IF NOT EXISTS idx_observers_iata_norm ON observers(UPPER(TRIM(iata)))`); err != nil { - return fmt.Errorf("ensure idx_observers_iata_norm: %w", err) + if logf != nil { + logf("[dbschema] PostgreSQL telemetry schema %d initialized", Version) } return nil } - -func ensureNeighborEdgesTable(rw *sql.DB) error { - _, err := rw.Exec(`CREATE TABLE IF NOT EXISTS neighbor_edges ( - node_a TEXT NOT NULL, - node_b TEXT NOT NULL, - count INTEGER DEFAULT 1, - last_seen TEXT, - PRIMARY KEY (node_a, node_b) - )`) - return err -} - -// ensureInactiveNodesTable creates the inactive_nodes table if missing. -// The ingestor's applySchema also creates this table — duplicating it -// here makes dbschema.Apply self-sufficient when called against a -// fixture DB that pre-dates the soft-delete feature (e.g. CI's -// test-fixtures/e2e-fixture.db, which never had any inactive rows). -// Schema kept in sync with cmd/ingestor/db.go:applySchema. -func ensureInactiveNodesTable(rw *sql.DB) error { - _, err := rw.Exec(`CREATE TABLE IF NOT EXISTS inactive_nodes ( - public_key TEXT PRIMARY KEY, - name TEXT, - role TEXT, - lat REAL, - lon REAL, - last_seen TEXT, - first_seen TEXT, - advert_count INTEGER DEFAULT 0, - battery_mv INTEGER, - temperature_c REAL, - foreign_advert INTEGER DEFAULT 0 - )`) - if err != nil { +func applyTx(tx *sql.Tx, initialReady bool) error { + var err error + if _, err = tx.Exec(`SELECT pg_advisory_xact_lock(hashtextextended(current_schema(),$1))`, WriterLockKey); err != nil { return err } - _, err = rw.Exec(`CREATE INDEX IF NOT EXISTS idx_inactive_nodes_last_seen ON inactive_nodes(last_seen)`) - return err -} - -func ensureResolvedPathColumn(rw *sql.DB, logf Logger) error { - has, err := TableHasColumn(rw, "observations", "resolved_path") - if err != nil { - return err + if _, err = tx.Exec(`CREATE TABLE IF NOT EXISTS corescope_schema (version INTEGER NOT NULL, kind TEXT PRIMARY KEY, ready BOOLEAN NOT NULL)`); err != nil { + return fmt.Errorf("create readiness marker: %w", err) } - if has { + var version int + var ready bool + err = tx.QueryRow(`SELECT version,ready FROM corescope_schema WHERE kind='telemetry'`).Scan(&version, &ready) + if err == nil { + if version != 1 && version != Version { + return fmt.Errorf("unsupported telemetry schema version %d (binary supports %d)", version, Version) + } + if !ready { + return fmt.Errorf("telemetry import is incomplete; finish or restore the offline import") + } + if !initialReady { + return fmt.Errorf("import target is already initialized") + } + if version == 1 { + if _, err := tx.Exec(keyBackfillSchemaSQL); err != nil { + return fmt.Errorf("upgrade telemetry key backfill schema: %w", err) + } + _, err := tx.Exec(`UPDATE corescope_schema SET version=$1 WHERE kind='telemetry'`, Version) + return err + } return nil } - if _, err := rw.Exec("ALTER TABLE observations ADD COLUMN resolved_path TEXT"); err != nil { + if err != sql.ErrNoRows { return err } - logf("[dbschema] added resolved_path column to observations") - return nil -} - -func ensureObserverInactiveColumn(rw *sql.DB, logf Logger) error { - has, err := TableHasColumn(rw, "observers", "inactive") - if err != nil { + var count int + if err = tx.QueryRow(`SELECT count(*) FROM information_schema.tables WHERE table_schema=current_schema() AND table_name <> 'corescope_schema'`).Scan(&count); err != nil { return err } - if has { - return nil + if count != 0 { + return fmt.Errorf("refusing to initialize a non-empty unversioned schema") } - if _, err := rw.Exec("ALTER TABLE observers ADD COLUMN inactive INTEGER DEFAULT 0"); err != nil { + if _, err = tx.Exec(SchemaSQL); err != nil { + return fmt.Errorf("create telemetry schema: %w", err) + } + if _, err = tx.Exec(`INSERT INTO corescope_schema(version,kind,ready) VALUES($1,'telemetry',$2)`, Version, initialReady); err != nil { return err } - logf("[dbschema] added inactive column to observers") + return nil } -func ensureLastPacketAtColumn(rw *sql.DB, logf Logger) error { - has, err := TableHasColumn(rw, "observers", "last_packet_at") - if err != nil { - return err +func AssertReady(db *sql.DB) error { + var version int + var ready bool + if err := db.QueryRow(`SELECT version,ready FROM corescope_schema WHERE kind='telemetry'`).Scan(&version, &ready); err != nil { + return fmt.Errorf("telemetry schema is not initialized; run the offline migration command: %w", err) } - if has { - return nil + if version != Version { + return fmt.Errorf("unsupported telemetry schema version %d (binary supports %d)", version, Version) } - if _, err := rw.Exec("ALTER TABLE observers ADD COLUMN last_packet_at TEXT"); err != nil { - return err + if !ready { + return fmt.Errorf("telemetry import is incomplete; finish or restore the offline import") } - logf("[dbschema] added last_packet_at column to observers") - return nil -} - -func ensureObserverIATAColumn(rw *sql.DB, logf Logger) error { - has, err := TableHasColumn(rw, "observers", "iata") + rows, err := db.Query(`SELECT table_name,column_name,data_type,is_identity FROM information_schema.columns WHERE table_schema=current_schema()`) if err != nil { return err } - if has { - return nil - } - if _, err := rw.Exec("ALTER TABLE observers ADD COLUMN iata TEXT"); err != nil { - return err - } - logf("[dbschema] added iata column to observers") - return nil -} - -func ensureForeignAdvertColumn(rw *sql.DB, logf Logger) error { - for _, table := range []string{"nodes", "inactive_nodes"} { - has, err := TableHasColumn(rw, table, "foreign_advert") - if err != nil { - return fmt.Errorf("inspect %s: %w", table, err) - } - if has { - continue - } - if _, err := rw.Exec(fmt.Sprintf("ALTER TABLE %s ADD COLUMN foreign_advert INTEGER DEFAULT 0", table)); err != nil { + defer rows.Close() + have := make(map[string]bool) + types := make(map[string]string) + identities := make(map[string]bool) + for rows.Next() { + var table, col, dataType, identity string + if err := rows.Scan(&table, &col, &dataType, &identity); err != nil { return err } - logf("[dbschema] added foreign_advert column to %s", table) + have[table+"."+col] = true + types[table+"."+col] = strings.ToUpper(dataType) + identities[table+"."+col] = identity == "YES" } - return nil -} - -func ensureFromPubkeyColumn(rw *sql.DB, logf Logger) error { - has, err := TableHasColumn(rw, "transmissions", "from_pubkey") - if err != nil { + if err := rows.Err(); err != nil { return err } - if !has { - if _, err := rw.Exec("ALTER TABLE transmissions ADD COLUMN from_pubkey TEXT"); err != nil { - return err + var missing []string + for _, table := range Tables { + for _, col := range table.Columns { + key := table.Name + "." + col.Name + if !have[key] { + missing = append(missing, key) + } else if types[key] != col.Type || identities[key] != col.Identity { + missing = append(missing, key+" (unexpected type or identity)") + } } - logf("[dbschema] added from_pubkey column to transmissions (#1143)") - } - if _, err := rw.Exec("CREATE INDEX IF NOT EXISTS idx_transmissions_from_pubkey ON transmissions(from_pubkey)"); err != nil { - return err } - return nil -} - -// ensureScopeNameColumn adds transmissions.scope_name (+ partial index) and -// records the legacy `_migrations` marker so the ingestor's old gated path -// stays idempotent on existing DBs. Moved here from cmd/ingestor/db.go in -// #1321 to be the canonical source of truth for the optional column the -// server PRAGMA-detects as hasScopeName. -func ensureScopeNameColumn(rw *sql.DB, logf Logger) error { - if err := ensureMigrationsTable(rw); err != nil { - return err + if !have["packets_v.id"] { + missing = append(missing, "packets_v") } - has, err := TableHasColumn(rw, "transmissions", "scope_name") + indexRows, err := db.Query(`SELECT indexname FROM pg_indexes WHERE schemaname=current_schema()`) if err != nil { return err } - if !has { - if _, err := rw.Exec(`ALTER TABLE transmissions ADD COLUMN scope_name TEXT DEFAULT NULL`); err != nil { - return fmt.Errorf("alter transmissions add scope_name: %w", err) + indexes := make(map[string]bool) + for indexRows.Next() { + var name string + if err := indexRows.Scan(&name); err != nil { + indexRows.Close() + return err } - logf("[dbschema] added scope_name column to transmissions (#899)") - } - if _, err := rw.Exec(`CREATE INDEX IF NOT EXISTS idx_tx_scope_name ON transmissions(scope_name) WHERE scope_name IS NOT NULL`); err != nil { - return fmt.Errorf("create idx_tx_scope_name: %w", err) - } - if _, err := rw.Exec(`INSERT OR IGNORE INTO _migrations (name) VALUES ('scope_name_v1')`); err != nil { - return fmt.Errorf("record scope_name_v1: %w", err) + indexes[name] = true } - return nil -} - -// ensureDefaultScopeColumns adds default_scope to nodes + inactive_nodes -// and records the `_migrations` marker. Source of truth lives here per -// #1321 (was previously cmd/ingestor/db.go only). -func ensureDefaultScopeColumns(rw *sql.DB, logf Logger) error { - if err := ensureMigrationsTable(rw); err != nil { + if err := indexRows.Err(); err != nil { + indexRows.Close() return err } - for _, table := range []string{"nodes", "inactive_nodes"} { - has, err := TableHasColumn(rw, table, "default_scope") - if err != nil { - return fmt.Errorf("inspect %s.default_scope: %w", table, err) - } - if has { - continue + indexRows.Close() + for _, name := range Indexes { + if !indexes[name] { + missing = append(missing, "index:"+name) } - if _, err := rw.Exec(fmt.Sprintf(`ALTER TABLE %s ADD COLUMN default_scope TEXT DEFAULT NULL`, table)); err != nil { - return fmt.Errorf("alter %s add default_scope: %w", table, err) - } - logf("[dbschema] added default_scope column to %s (#899)", table) } - if _, err := rw.Exec(`INSERT OR IGNORE INTO _migrations (name) VALUES ('nodes_default_scope_v1')`); err != nil { - return fmt.Errorf("record nodes_default_scope_v1: %w", err) + if len(missing) > 0 { + return fmt.Errorf("incomplete telemetry schema: %s", strings.Join(missing, ", ")) } return nil } -// ensureConfiguredScopeColumns adds nodes.configured_scope + -// nodes.configured_scope_at (and mirrors on inactive_nodes) for #1865. -// Unlike default_scope (inferred from observed advert transport scope, and -// overwritten on every observation), configured_scope holds the region scopes -// a node has CONFIGURED, taken as concrete evidence from the observer -// /neighbors report — written only for the observer's own `self` scopes and -// for neighbors whose OTA scope query returned status="responded". The -// server PRAGMA-detects configured_scope as hasConfiguredScope. -func ensureConfiguredScopeColumns(rw *sql.DB, logf Logger) error { - if err := ensureMigrationsTable(rw); err != nil { - return err - } - for _, table := range []string{"nodes", "inactive_nodes"} { - for _, col := range []string{"configured_scope", "configured_scope_at"} { - has, err := TableHasColumn(rw, table, col) - if err != nil { - return fmt.Errorf("inspect %s.%s: %w", table, col, err) - } - if has { - continue - } - if _, err := rw.Exec(fmt.Sprintf(`ALTER TABLE %s ADD COLUMN %s TEXT DEFAULT NULL`, table, col)); err != nil { - return fmt.Errorf("alter %s add %s: %w", table, col, err) - } - logf("[dbschema] added %s column to %s (#1865)", col, table) - } - } - if _, err := rw.Exec(`INSERT OR IGNORE INTO _migrations (name) VALUES ('nodes_configured_scope_v1')`); err != nil { - return fmt.Errorf("record nodes_configured_scope_v1: %w", err) - } - return nil +func TableHasColumn(db Querier, table, column string) (bool, error) { + var found bool + err := db.QueryRow(`SELECT EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema=current_schema() AND table_name=$1 AND column_name=$2)`, table, column).Scan(&found) + return found, err } -// ensureObservationsRawHexColumn adds observations.raw_hex (#881). -// Source of truth lives here per #1321 (was previously cmd/ingestor/db.go only): -// the server PRAGMA-detects this column as hasObsRawHex. -func ensureObservationsRawHexColumn(rw *sql.DB, logf Logger) error { - if err := ensureMigrationsTable(rw); err != nil { - return err - } - has, err := TableHasColumn(rw, "observations", "raw_hex") +// AssertWriter refuses migration-owner credentials in the runtime process. +func AssertWriter(db *sql.DB) error { + var elevated, createSchema, ownsSchema bool + err := db.QueryRow(`SELECT r.rolsuper OR r.rolcreatedb OR r.rolcreaterole, has_schema_privilege(current_schema(),'CREATE') OR has_database_privilege(current_database(),'CREATE'), pg_has_role(current_user,n.nspowner,'MEMBER') OR EXISTS(SELECT 1 FROM pg_class c WHERE c.relnamespace=n.oid AND pg_has_role(current_user,c.relowner,'MEMBER')) OR has_table_privilege('corescope_schema','INSERT,UPDATE,DELETE,TRUNCATE,TRIGGER') FROM pg_roles r JOIN pg_namespace n ON n.nspname=current_schema() WHERE r.rolname=current_user`).Scan(&elevated, &createSchema, &ownsSchema) if err != nil { return err } - if !has { - if _, err := rw.Exec(`ALTER TABLE observations ADD COLUMN raw_hex TEXT`); err != nil { - return fmt.Errorf("alter observations add raw_hex: %w", err) - } - logf("[dbschema] added raw_hex column to observations (#881)") - } - if _, err := rw.Exec(`INSERT OR IGNORE INTO _migrations (name) VALUES ('observations_raw_hex_v1')`); err != nil { - return fmt.Errorf("record observations_raw_hex_v1: %w", err) + if elevated || createSchema || ownsSchema { + return fmt.Errorf("telemetry runtime requires a restricted writer role without schema ownership, CREATE, or administrative privileges") } - return nil -} - -// ensureMigrationsTable is idempotent — the ingestor's applySchema also -// creates this table on legacy paths. Keeping it here lets the gated -// `INSERT OR IGNORE` markers above stay self-sufficient even when Apply -// runs against a brand-new fixture DB (e.g. dbschema_test.go). -func ensureMigrationsTable(rw *sql.DB) error { - _, err := rw.Exec(`CREATE TABLE IF NOT EXISTS _migrations (name TEXT PRIMARY KEY)`) - return err -} - -// SoftDeleteBlacklistedObservers marks the given observer IDs as -// inactive=1 (case-insensitive match). Returns count affected. -// Writer-side helper; ingestor calls it at startup with the operator -// blacklist (read from config). -func SoftDeleteBlacklistedObservers(rw *sql.DB, blacklist []string) (int64, error) { - placeholders := make([]string, 0, len(blacklist)) - args := make([]interface{}, 0, len(blacklist)) - for _, pk := range blacklist { - t := strings.TrimSpace(pk) - if t == "" { - continue + for _, table := range Tables { + var allowed bool + if err := db.QueryRow(`SELECT has_table_privilege($1,'SELECT') AND has_table_privilege($1,'INSERT') AND has_table_privilege($1,'UPDATE') AND has_table_privilege($1,'DELETE')`, table.Name).Scan(&allowed); err != nil { + return err } - placeholders = append(placeholders, "LOWER(?)") - args = append(args, t) - } - if len(placeholders) == 0 { - return 0, nil - } - q := "UPDATE observers SET inactive = 1 WHERE LOWER(id) IN (" + - strings.Join(placeholders, ",") + ") AND (inactive IS NULL OR inactive = 0)" - res, err := rw.Exec(q, args...) - if err != nil { - return 0, err - } - n, _ := res.RowsAffected() - return n, nil -} - -// ensureMultibyteCapColumns adds the multi-byte capability cache columns -// to nodes / inactive_nodes (PR #903, canonical owner per #1324 -// follow-up). These columns are populated by the ingestor's -// RunMultibyteCapPersist from snapshot files written by the server's -// analytics cycle; the server is read-only since #1289 and MUST NOT -// write here. The schema itself lives here in dbschema (the writer -// owns migrations, the read-only server merely AssertReady's them). -func ensureMultibyteCapColumns(rw *sql.DB, logf Logger) error { - for _, table := range []string{"nodes", "inactive_nodes"} { - hasSup, err := TableHasColumn(rw, table, "multibyte_sup") - if err != nil { - return fmt.Errorf("inspect %s.multibyte_sup: %w", table, err) + if !allowed { + return fmt.Errorf("telemetry writer lacks data privileges on %s", table.Name) } - if !hasSup { - if _, err := rw.Exec(fmt.Sprintf( - "ALTER TABLE %s ADD COLUMN multibyte_sup INTEGER NOT NULL DEFAULT 0", table)); err != nil { - return fmt.Errorf("add %s.multibyte_sup: %w", table, err) + for _, column := range table.Columns { + if !column.Identity { + continue } - logf("[dbschema] added multibyte_sup column to %s", table) - } - hasEvid, err := TableHasColumn(rw, table, "multibyte_evidence") - if err != nil { - return fmt.Errorf("inspect %s.multibyte_evidence: %w", table, err) - } - if !hasEvid { - if _, err := rw.Exec(fmt.Sprintf( - "ALTER TABLE %s ADD COLUMN multibyte_evidence TEXT", table)); err != nil { - return fmt.Errorf("add %s.multibyte_evidence: %w", table, err) + if err := db.QueryRow(`SELECT has_sequence_privilege(pg_get_serial_sequence($1,$2),'USAGE')`, table.Name, column.Name).Scan(&allowed); err != nil { + return err + } + if !allowed { + return fmt.Errorf("telemetry writer lacks sequence USAGE on %s.%s", table.Name, column.Name) } - logf("[dbschema] added multibyte_evidence column to %s", table) } } return nil } -// ensureObserverNaiveClockColumns adds the three per-observer naive-clock -// skew tracking columns (#1478). Server reads them to populate the -// clock_naive / clock_skew_seconds / clock_skew_count_24h / -// clock_last_naive_at fields in /api/observers responses; ingestor writes -// them from resolveRxTime via Store.RecordNaiveSkew on each clamp event. -// Owned here per the #1321 source-of-truth invariant — the cmd/ingestor -// copy migrates real prod DBs, dbschema migrates fixture/staging DBs via -// cmd/migrate. -func ensureObserverNaiveClockColumns(rw *sql.DB, logf Logger) error { - type col struct{ name, ddl string } - cols := []col{ - {"clock_skew_seconds", "ALTER TABLE observers ADD COLUMN clock_skew_seconds INTEGER DEFAULT NULL"}, - {"clock_skew_count_24h", "ALTER TABLE observers ADD COLUMN clock_skew_count_24h INTEGER DEFAULT 0"}, - {"clock_last_naive_at", "ALTER TABLE observers ADD COLUMN clock_last_naive_at TEXT DEFAULT NULL"}, - } - for _, c := range cols { - has, err := TableHasColumn(rw, "observers", c.name) - if err != nil { - return fmt.Errorf("inspect observers.%s: %w", c.name, err) - } - if has { - continue - } - if _, err := rw.Exec(c.ddl); err != nil { - return fmt.Errorf("add observers.%s: %w", c.name, err) +// ReseedIdentities preserves imported IDs and legacy deleted high-water marks. +// The caller runs this in its import transaction before marking ready. +func ReseedIdentities(tx *sql.Tx, highWater map[string]int64) error { + for _, table := range Tables { + for _, column := range table.Columns { + if !column.Identity { + continue + } + key := table.Name + "." + column.Name + var high int64 + if err := tx.QueryRow(`SELECT COALESCE(MAX("` + column.Name + `"),0) FROM "` + table.Name + `"`).Scan(&high); err != nil { + return err + } + if highWater[key] > high { + high = highWater[key] + } + if high < 1 { + high = 1 + if _, err := tx.Exec(`SELECT setval(pg_get_serial_sequence($1,$2),$3,false)`, table.Name, column.Name, high); err != nil { + return err + } + } else { + if _, err := tx.Exec(`SELECT setval(pg_get_serial_sequence($1,$2),$3,true)`, table.Name, column.Name, high); err != nil { + return err + } + } } - logf("[dbschema] added %s column to observers", c.name) } return nil } -// ensureObserverCanRelayColumn adds the can_relay column to observers. -// Firmware 1.16 publishes a `repeat: on|off` flag in the MQTT /status -// JSON (#1290); the ingestor parses it and writes 0/1 here. The server's -// path-hop disambiguator (cmd/server/store.go pm.resolveWithContext) -// excludes observers with can_relay=0 from the candidate set. Default 1 -// preserves prior behavior for legacy observers (no repeat field). -func ensureObserverCanRelayColumn(rw *sql.DB, logf Logger) error { - has, err := TableHasColumn(rw, "observers", "can_relay") - if err != nil { - return err - } - if has { - return nil - } - // PREFLIGHT: async=true reason="single-column ALTER on observers (low-cardinality, ~1k rows in prod); DEFAULT 1 is a constant so SQLite does the rewrite as a metadata-only schema update, no row scan" - if _, err := rw.Exec("ALTER TABLE observers ADD COLUMN can_relay INTEGER DEFAULT 1"); err != nil { - return err - } - logf("[dbschema] added can_relay column to observers") - return nil -} - -// ensureObserverCanRelaySeenColumn adds the can_relay_seen tracking column. -// Issue #1290 follow-up (PR #1624 MAJOR-2): can_relay defaults to 1 at -// INSERT, which conflates "confirmed repeater" with "legacy observer -// that never sent the repeat field". can_relay_seen=1 is written by -// the ingestor whenever the firmware actually provided the field; the -// server's read layer returns CanRelay=nil whenever seen=0 so the UI -// can render the tri-state badge (no badge for unknown). -func ensureObserverCanRelaySeenColumn(rw *sql.DB, logf Logger) error { - has, err := TableHasColumn(rw, "observers", "can_relay_seen") - if err != nil { - return err - } - if has { - return nil - } - // PREFLIGHT: async=true reason="single-column ALTER on observers (low-cardinality, ~1k rows in prod); DEFAULT 0 is a constant so SQLite does the rewrite as a metadata-only schema update, no row scan" - if _, err := rw.Exec("ALTER TABLE observers ADD COLUMN can_relay_seen INTEGER DEFAULT 0"); err != nil { - return err +func SoftDeleteBlacklistedObservers(db *sql.DB, blacklist []string) (int64, error) { + var slots []string + var args []any + for _, id := range blacklist { + if id = strings.TrimSpace(id); id != "" { + args = append(args, id) + slots = append(slots, fmt.Sprintf("LOWER($%d)", len(args))) + } } - logf("[dbschema] added can_relay_seen column to observers") - return nil -} - -// ensureTransmissionsLastSeenColumn adds transmissions.last_seen (#1690), -// the denormalized "most recent observation timestamp" column the cold-load -// path filters on. Without it, hot-window queries that match -// `first_seen >= cutoff` exclude long-lived hashes whose hashes were first -// inserted weeks ago but are still active — see the issue for the full -// 0.3%-of-DB-loaded post-mortem. -// -// The backfill from `MAX(observations.timestamp) GROUP BY transmission_id` -// is potentially expensive (1.9M+ obs rows in prod) and runs as a separate -// async migration scheduled by cmd/ingestor/db.go::OpenStore via -// Store.RunAsyncMigration. -func ensureTransmissionsLastSeenColumn(rw *sql.DB, logf Logger) error { - if err := ensureMigrationsTable(rw); err != nil { - return err + if len(args) == 0 { + return 0, nil } - has, err := TableHasColumn(rw, "transmissions", "last_seen") + res, err := db.Exec(`UPDATE observers SET inactive=1 WHERE LOWER(id) IN (`+strings.Join(slots, ",")+`) AND (inactive IS NULL OR inactive=0)`, args...) if err != nil { - return err - } - if !has { - // PREFLIGHT: async=true reason="single-column ALTER TABLE on transmissions; DEFAULT 0 is a constant so SQLite does a metadata-only schema rewrite (no row scan) — cheap at any scale" - if _, err := rw.Exec(`ALTER TABLE transmissions ADD COLUMN last_seen INTEGER NOT NULL DEFAULT 0`); err != nil { - return fmt.Errorf("alter transmissions add last_seen: %w", err) - } - logf("[dbschema] added last_seen column to transmissions (#1690)") - } - // #1740 step (a): create the PARTIAL index first. Scans only rows with - // last_seen=0 — i.e. the un-backfilled hot subset the chunked backfill - // MAX(id) lookup walks. Degenerates to near-empty once the backfill - // converges, so it stops competing for page cache. - // - // PREFLIGHT: async=false reason="partial index on `WHERE last_seen=0`; subset is bounded by un-backfilled rows (≤0 in steady state, ≤ingest-rate during ops) — cheap at any scale even on a 1.9M-row prod table" - if _, err := rw.Exec(`CREATE INDEX IF NOT EXISTS idx_tx_last_seen_zero ON transmissions(id) WHERE last_seen=0`); err != nil { - return fmt.Errorf("create idx_tx_last_seen_zero: %w", err) - } - // #1740 step (b): gated DROP of the legacy full index. Only runs AFTER - // step (a) succeeded (sequential — we're past the CREATE above). The - // DROP is a metadata-only schema rewrite in SQLite; no row scan. - // - // PREFLIGHT: async=false reason="DROP INDEX is metadata-only in SQLite (no row scan, no rewrite); safe inline at any DB size" - if _, err := rw.Exec(`DROP INDEX IF EXISTS idx_tx_last_seen`); err != nil { - return fmt.Errorf("drop legacy idx_tx_last_seen: %w", err) + return 0, err } - return nil + return res.RowsAffected() } diff --git a/internal/dbschema/go.mod b/internal/dbschema/go.mod index 3dca3dc9d..da81f1bb0 100644 --- a/internal/dbschema/go.mod +++ b/internal/dbschema/go.mod @@ -1,5 +1,23 @@ module github.com/meshcore-analyzer/dbschema -go 1.22 +go 1.25.0 -require github.com/mattn/go-sqlite3 v1.14.52 +require ( + github.com/mattn/go-sqlite3 v1.14.52 + github.com/meshcore-analyzer/dbconfig v0.0.0 + github.com/meshcore-analyzer/pgutil v0.0.0 +) + +require ( + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/pgx/v5 v5.7.6 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + golang.org/x/crypto v0.37.0 // indirect + golang.org/x/sync v0.13.0 // indirect + golang.org/x/text v0.24.0 // indirect +) + +replace github.com/meshcore-analyzer/pgutil => ../pgutil + +replace github.com/meshcore-analyzer/dbconfig => ../dbconfig diff --git a/internal/dbschema/go.sum b/internal/dbschema/go.sum index 75bf6eb53..c83dde8cc 100644 --- a/internal/dbschema/go.sum +++ b/internal/dbschema/go.sum @@ -1,2 +1,30 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.7.6 h1:rWQc5FwZSPX58r1OQmkuaNicxdmExaEz5A2DO2hUuTk= +github.com/jackc/pgx/v5 v5.7.6/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/mattn/go-sqlite3 v1.14.52 h1:wVbm2Qnf4OXkqhBTSPuCRZDRnxfbVrrmiCEroVdog8U= github.com/mattn/go-sqlite3 v1.14.52/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= +golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= +golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= +golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= +golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/dbschema/key_backfill.go b/internal/dbschema/key_backfill.go index 0934cc7ab..490967184 100644 --- a/internal/dbschema/key_backfill.go +++ b/internal/dbschema/key_backfill.go @@ -1,52 +1,26 @@ package dbschema -import "database/sql" - -// ensureKeyBackfill creates the tables behind the key-set backfill (#2107). -// -// tx_rewrite_feed is the change feed: the ingestor appends one row per -// transmission it rewrote in place (a GRP_TXT it can now decrypt, a transport -// scope it can now name), in the same transaction as the rewrite itself. The -// read-only server tails it with its own cursor, exactly like -// advert_route_evidence, so a rewritten row reaches the in-memory store without -// a restart. AUTOINCREMENT keeps feed IDs monotonic after pruning; the -// created_at index serves the age-based prune, and the tx_id index serves the -// ON DELETE CASCADE, so retention deleting a transmission does not scan the -// whole feed. -// -// key_backfill_state holds at most one in-flight run per kind. run_fp is the -// fingerprint of the key set the run is working with: a run whose fingerprint -// no longer matches the live key set is restarted, which is what makes a key -// added mid-run safe. cursor advances in the same transaction as the rewrites -// it covers, so a cancelled run resumes exactly where it committed. -// -// key_backfill_done records which keys have had a complete pass over history, -// so a restart with an unchanged key set does no work at all. -// -// PREFLIGHT: async=true reason="creates three empty tables and two indexes on the empty feed; the backfill itself runs batched in the background after live ingest is ready" -func ensureKeyBackfill(rw *sql.DB) error { - _, err := rw.Exec(` - CREATE TABLE IF NOT EXISTS tx_rewrite_feed ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - tx_id INTEGER NOT NULL REFERENCES transmissions(id) ON DELETE CASCADE, - created_at INTEGER NOT NULL DEFAULT (CAST(strftime('%s','now') AS INTEGER)) - ); - CREATE INDEX IF NOT EXISTS idx_tx_rewrite_feed_created ON tx_rewrite_feed(created_at); - CREATE INDEX IF NOT EXISTS idx_tx_rewrite_feed_tx ON tx_rewrite_feed(tx_id); - CREATE TABLE IF NOT EXISTS key_backfill_state ( - kind TEXT PRIMARY KEY CHECK(kind IN ('channel','region')), - run_fp TEXT NOT NULL, - cursor INTEGER NOT NULL DEFAULT 0, - upper INTEGER NOT NULL DEFAULT 0, - rewritten INTEGER NOT NULL DEFAULT 0, - started_at INTEGER NOT NULL DEFAULT (CAST(strftime('%s','now') AS INTEGER)) - ); - CREATE TABLE IF NOT EXISTS key_backfill_done ( - kind TEXT NOT NULL, - key_fp TEXT NOT NULL, - done_at INTEGER NOT NULL DEFAULT (CAST(strftime('%s','now') AS INTEGER)), - PRIMARY KEY(kind, key_fp) - ) WITHOUT ROWID; - `) - return err -} +// Version 2 adds the bounded rewrite feed and restart-safe key backfill ledger. +// Only the offline migration owner applies this additive upgrade. +const keyBackfillSchemaSQL = `CREATE TABLE tx_rewrite_feed ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + tx_id BIGINT NOT NULL REFERENCES transmissions(id) ON DELETE CASCADE, + created_at BIGINT NOT NULL DEFAULT FLOOR(EXTRACT(EPOCH FROM CURRENT_TIMESTAMP))::BIGINT +); +CREATE INDEX idx_tx_rewrite_feed_created ON tx_rewrite_feed(created_at); +CREATE INDEX idx_tx_rewrite_feed_tx ON tx_rewrite_feed(tx_id); +CREATE TABLE key_backfill_state ( + kind TEXT COLLATE "C" PRIMARY KEY CHECK(kind IN ('channel','region')), + run_fp TEXT COLLATE "C" NOT NULL, + cursor BIGINT NOT NULL DEFAULT 0, + upper BIGINT NOT NULL DEFAULT 0, + rewritten BIGINT NOT NULL DEFAULT 0, + started_at BIGINT NOT NULL DEFAULT FLOOR(EXTRACT(EPOCH FROM CURRENT_TIMESTAMP))::BIGINT +); +CREATE TABLE key_backfill_done ( + kind TEXT COLLATE "C" NOT NULL, + key_fp TEXT COLLATE "C" NOT NULL, + done_at BIGINT NOT NULL DEFAULT FLOOR(EXTRACT(EPOCH FROM CURRENT_TIMESTAMP))::BIGINT, + PRIMARY KEY(kind,key_fp) +); +` diff --git a/internal/dbschema/key_backfill_upgrade_test.go b/internal/dbschema/key_backfill_upgrade_test.go new file mode 100644 index 000000000..ee1c49826 --- /dev/null +++ b/internal/dbschema/key_backfill_upgrade_test.go @@ -0,0 +1,71 @@ +package dbschema + +import ( + "strings" + "testing" + + "github.com/meshcore-analyzer/pgutil" +) + +func TestPostgresV1KeyBackfillUpgradeIsAtomicAndPreservesRows(t *testing.T) { + for _, conflict := range []bool{false, true} { + name := "normal" + if conflict { + name = "unknown conflicting table" + } + t.Run(name, func(t *testing.T) { + db, err := pgutil.Open(postgresSchema(t), false) + if err != nil { + t.Fatal(err) + } + defer db.Close() + oldSQL := strings.Replace(SchemaSQL, keyBackfillSchemaSQL, "", 1) + if oldSQL == SchemaSQL { + t.Fatal("v1 fixture did not remove v2 additions") + } + if _, err := db.Exec(oldSQL); err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`CREATE TABLE corescope_schema(version INTEGER NOT NULL,kind TEXT PRIMARY KEY,ready BOOLEAN NOT NULL); + INSERT INTO corescope_schema VALUES(1,'telemetry',true); + INSERT INTO transmissions(id,raw_hex,hash,first_seen) VALUES(7,'00','retained','raw-date')`); err != nil { + t.Fatal(err) + } + if err := AssertReady(db); err == nil { + t.Fatal("v1 accepted before owner upgrade") + } + if conflict { + if _, err := db.Exec(`CREATE TABLE key_backfill_state(private_extension TEXT)`); err != nil { + t.Fatal(err) + } + } + err = ApplyPostgres(db, nil) + if conflict { + if err == nil { + t.Fatal("unknown conflicting table accepted") + } + var version, added int + if err := db.QueryRow(`SELECT version FROM corescope_schema WHERE kind='telemetry'`).Scan(&version); err != nil || version != 1 { + t.Fatal("failed upgrade changed marker", err) + } + if err := db.QueryRow(`SELECT count(*) FROM information_schema.tables WHERE table_schema=current_schema() AND table_name='tx_rewrite_feed'`).Scan(&added); err != nil || added != 0 { + t.Fatal("failed upgrade left partial table", err) + } + return + } + if err != nil { + t.Fatal(err) + } + if err := AssertReady(db); err != nil { + t.Fatal(err) + } + if err := ApplyPostgres(db, nil); err != nil { + t.Fatal("repeat upgrade", err) + } + var hash, date string + if err := db.QueryRow(`SELECT hash,first_seen FROM transmissions WHERE id=7`).Scan(&hash, &date); err != nil || hash != "retained" || date != "raw-date" { + t.Fatal("old row changed", err) + } + }) + } +} diff --git a/internal/dbschema/advert_evidence.go b/internal/dbschema/legacy/advert_evidence.go similarity index 98% rename from internal/dbschema/advert_evidence.go rename to internal/dbschema/legacy/advert_evidence.go index dc6e2bacc..62648f0b4 100644 --- a/internal/dbschema/advert_evidence.go +++ b/internal/dbschema/legacy/advert_evidence.go @@ -1,4 +1,4 @@ -package dbschema +package legacy import "database/sql" diff --git a/internal/dbschema/legacy/base.go b/internal/dbschema/legacy/base.go new file mode 100644 index 000000000..e26ab9a3c --- /dev/null +++ b/internal/dbschema/legacy/base.go @@ -0,0 +1,707 @@ +package legacy + +import ( + "database/sql" + "fmt" + "log" + "strings" +) + +func ApplyBase(db *sql.DB) error { + // auto_vacuum=INCREMENTAL is set via DSN pragma (must be before journal_mode). + // Logging of current mode is handled by CheckAutoVacuum — no duplicate log here. + + schema := ` + CREATE TABLE IF NOT EXISTS nodes ( + public_key TEXT PRIMARY KEY, + name TEXT, + role TEXT, + lat REAL, + lon REAL, + last_seen TEXT, + first_seen TEXT, + advert_count INTEGER DEFAULT 0, + battery_mv INTEGER, + temperature_c REAL, + foreign_advert INTEGER DEFAULT 0 + ); + + CREATE TABLE IF NOT EXISTS scope_match_totals ( + id INTEGER PRIMARY KEY CHECK (id = 1), + since_unix INTEGER NOT NULL, + unique_matches INTEGER NOT NULL, + explicit_over_derived INTEGER NOT NULL, + ambiguous INTEGER NOT NULL, + none_matches INTEGER NOT NULL, + updated_unix INTEGER NOT NULL + ); + + CREATE TABLE IF NOT EXISTS observers ( + id TEXT PRIMARY KEY, + name TEXT, + iata TEXT, + last_seen TEXT, + first_seen TEXT, + packet_count INTEGER DEFAULT 0, + model TEXT, + firmware TEXT, + client_version TEXT, + radio TEXT, + battery_mv INTEGER, + uptime_secs INTEGER, + noise_floor REAL, + inactive INTEGER DEFAULT 0, + last_packet_at TEXT DEFAULT NULL, + clock_skew_seconds INTEGER DEFAULT NULL, + clock_skew_count_24h INTEGER DEFAULT 0, + clock_last_naive_at TEXT DEFAULT NULL, + can_relay INTEGER DEFAULT 1, + can_relay_seen INTEGER DEFAULT 0 + ); + + CREATE INDEX IF NOT EXISTS idx_nodes_last_seen ON nodes(last_seen); + CREATE INDEX IF NOT EXISTS idx_observers_last_seen ON observers(last_seen); + + CREATE TABLE IF NOT EXISTS inactive_nodes ( + public_key TEXT PRIMARY KEY, + name TEXT, + role TEXT, + lat REAL, + lon REAL, + last_seen TEXT, + first_seen TEXT, + advert_count INTEGER DEFAULT 0, + battery_mv INTEGER, + temperature_c REAL, + foreign_advert INTEGER DEFAULT 0 + ); + + CREATE INDEX IF NOT EXISTS idx_inactive_nodes_last_seen ON inactive_nodes(last_seen); + + CREATE TABLE IF NOT EXISTS transmissions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + raw_hex TEXT NOT NULL, + hash TEXT NOT NULL UNIQUE, + first_seen TEXT NOT NULL, + route_type INTEGER, + payload_type INTEGER, + payload_version INTEGER, + decoded_json TEXT, + from_pubkey TEXT, + last_seen INTEGER NOT NULL DEFAULT 0, + created_at TEXT DEFAULT (datetime('now')), + code1 TEXT, + code2 TEXT + ); + + CREATE INDEX IF NOT EXISTS idx_transmissions_hash ON transmissions(hash); + CREATE INDEX IF NOT EXISTS idx_transmissions_first_seen ON transmissions(first_seen); + CREATE INDEX IF NOT EXISTS idx_transmissions_payload_type ON transmissions(payload_type); + -- idx_tx_code1 is created by the transport_codes_v1 migration below, + -- after the ALTER runs — same reasoning as idx_transmissions_from_pubkey + -- and idx_tx_last_seen_zero above: a legacy DB (table-exists, + -- column-missing) would trip on this CREATE INDEX referencing code1 + -- before the column exists, since CREATE TABLE IF NOT EXISTS is a + -- no-op against a pre-existing table. + -- idx_transmissions_from_pubkey is created by the from_pubkey_v1 + -- migration after the column is added on legacy DBs (#1143). + -- idx_tx_last_seen_zero (partial, WHERE last_seen=0) is created by + -- dbschema.Apply after ensuring the last_seen column exists (#1690, + -- partial-index swap #1740) — keep it OUT of this base schema block + -- so legacy DBs (table-exists, column-missing) don't trip on the + -- CREATE INDEX before the ALTER runs. + + -- Mobile client RX coverage: a roaming companion = a mobile observer + -- with a moving GPS position, so it gets its own table rather than + -- observations (which assumes a fixed observer/location). + CREATE TABLE IF NOT EXISTS client_receptions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + rx_pubkey TEXT NOT NULL, + heard_key TEXT NOT NULL, + heard_keylen INTEGER NOT NULL, + rssi INTEGER, + snr REAL, + lat REAL NOT NULL, + lon REAL NOT NULL, + pos_acc_m REAL, + rx_at TEXT NOT NULL, + ingested_at TEXT NOT NULL, + src TEXT NOT NULL, + UNIQUE(rx_pubkey, heard_key, rx_at) + ); + -- Coverage queries filter by bbox AND match the heard node either by full + -- key (heard_keylen=32 AND heard_key=?) or by 2-3 byte prefix. The composite + -- (heard_key, heard_keylen, lat, lon) serves the heard_key-equality seek and + -- carries lat/lon so the bbox range is satisfied from the index; it also + -- supersedes the old single-column heard_key index. idx_client_recept_latlon + -- lets the planner instead drive from a selective bbox. (#5, #18) + CREATE INDEX IF NOT EXISTS idx_client_recept_heard_geo ON client_receptions(heard_key, heard_keylen, lat, lon); + CREATE INDEX IF NOT EXISTS idx_client_recept_latlon ON client_receptions(lat, lon); + -- rx_at backs both the retention reaper (DELETE WHERE rx_at < ?) and the + -- leaderboard, which range-scans WHERE rx_at >= ? and aggregates per + -- rx_pubkey in Go (see rxLeaderboard's frontier-weighted scoring). Without + -- this index either would full-scan the table under the writer lock + -- (verified by an EXPLAIN test). A dedicated rx_pubkey index stays + -- redundant — the leaderboard no longer groups by rx_pubkey in SQL. + CREATE INDEX IF NOT EXISTS idx_client_recept_rxat ON client_receptions(rx_at); + DROP INDEX IF EXISTS idx_client_recept_rxpk; + + -- Self-reported name of each mobile client (companion), from the SELF_INFO + -- name the app sends as "origin". Lets the leaderboard show a name even + -- when the companion never advertised (so it isn't in the nodes table). + CREATE TABLE IF NOT EXISTS client_observers ( + pubkey TEXT PRIMARY KEY, + name TEXT, + last_seen TEXT + ); + + -- Diagnostic RF observations from mobile clients. Unlike client_receptions + -- this holds EVERY decodable packet, attributable or not, so it must never + -- be used for coverage. pkt_hash is ComputeContentHash() — identical to + -- transmissions.hash — so dark-traffic queries are a plain equality join. + CREATE TABLE IF NOT EXISTS client_rx_observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + rx_pubkey TEXT NOT NULL, + rx_at TEXT NOT NULL, + ingested_at TEXT NOT NULL, + pkt_hash TEXT NOT NULL, + route_type INTEGER NOT NULL, + payload_type INTEGER NOT NULL, + code1 TEXT, + code2 TEXT, + scope_name TEXT, + hash_size INTEGER NOT NULL, + hop_count INTEGER NOT NULL, + path_json TEXT, + forwarder TEXT, + snr REAL, + rssi INTEGER, + lat REAL NOT NULL, + lon REAL NOT NULL, + pos_acc_m REAL, + UNIQUE(rx_pubkey, pkt_hash, rx_at) + ); + CREATE INDEX IF NOT EXISTS idx_cro_prune ON client_rx_observations(rx_at); + CREATE INDEX IF NOT EXISTS idx_cro_hash ON client_rx_observations(pkt_hash, rx_at); + CREATE INDEX IF NOT EXISTS idx_cro_forwarder ON client_rx_observations(forwarder, rx_at); + CREATE INDEX IF NOT EXISTS idx_cro_scope ON client_rx_observations(scope_name, rx_at); + + -- RF environment samples from mobile clients: radio counters paired with + -- a GPS point. Absolutes only — deltas are computed at query time, and a + -- decrease in uptime_secs (reboot) or any counter (wrap) breaks the chain. + CREATE TABLE IF NOT EXISTS client_rf_samples ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + rx_pubkey TEXT NOT NULL, + sampled_at TEXT NOT NULL, + ingested_at TEXT NOT NULL, + lat REAL NOT NULL, + lon REAL NOT NULL, + pos_acc_m REAL, + stationary INTEGER NOT NULL DEFAULT 0, + uptime_secs INTEGER NOT NULL, + battery_mv INTEGER, + queue_len INTEGER, + errors INTEGER, + noise_floor INTEGER, + last_rssi INTEGER, + last_snr REAL, + tx_air_secs INTEGER, + rx_air_secs INTEGER, + recv INTEGER, + sent INTEGER, + flood_rx INTEGER, + direct_rx INTEGER, + flood_tx INTEGER, + direct_tx INTEGER, + recv_errors INTEGER, + UNIQUE(rx_pubkey, sampled_at) + ); + CREATE INDEX IF NOT EXISTS idx_crf_prune ON client_rf_samples(sampled_at); + CREATE INDEX IF NOT EXISTS idx_crf_track ON client_rf_samples(rx_pubkey, sampled_at); + + -- Declared region lists reported by repeaters via ANON_REQ_TYPE_REGIONS. + -- Observations, never state: "current" is the greatest observed_at for a + -- target, NOT the greatest ingested_at — a drive buffered offline can + -- arrive days late and must not overwrite a fresher reading. + CREATE TABLE IF NOT EXISTS node_declared_regions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + target TEXT NOT NULL, + rx_pubkey TEXT NOT NULL, + observed_at TEXT NOT NULL, + ingested_at TEXT NOT NULL, + regions_csv TEXT NOT NULL, + truncated INTEGER NOT NULL DEFAULT 0, + lat REAL, + lon REAL, + pos_acc_m REAL, + repeater_clock INTEGER, + UNIQUE(target, rx_pubkey, observed_at) + ); + CREATE INDEX IF NOT EXISTS idx_ndr_target ON node_declared_regions(target, observed_at); + CREATE INDEX IF NOT EXISTS idx_ndr_prune ON node_declared_regions(observed_at); + ` + if _, err := db.Exec(schema); err != nil { + return fmt.Errorf("base schema: %w", err) + } + + // Create observations table (v3 schema) + obsExists := false + row := db.QueryRow("SELECT name FROM sqlite_master WHERE type='table' AND name='observations'") + var dummy string + if row.Scan(&dummy) == nil { + obsExists = true + } + + if !obsExists { + obs := ` + CREATE TABLE observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + transmission_id INTEGER NOT NULL REFERENCES transmissions(id), + observer_idx INTEGER, + direction TEXT, + snr REAL, + rssi REAL, + score INTEGER, + path_json TEXT, + timestamp INTEGER NOT NULL + ); + CREATE INDEX idx_observations_transmission_id ON observations(transmission_id); + CREATE INDEX idx_observations_observer_idx ON observations(observer_idx); + CREATE INDEX idx_observations_timestamp ON observations(timestamp); + CREATE UNIQUE INDEX IF NOT EXISTS idx_observations_dedup ON observations(transmission_id, observer_idx, COALESCE(path_json, '')); + ` + if _, err := db.Exec(obs); err != nil { + return fmt.Errorf("observations schema: %w", err) + } + } + + // Create/rebuild packets_v view (v3 schema: observer_idx → observers.rowid) + // The Go server reads this view; without it fresh installs get "no such table: packets_v". + db.Exec(`DROP VIEW IF EXISTS packets_v`) + _, vErr := db.Exec(` + CREATE VIEW packets_v AS + SELECT o.id, COALESCE(o.raw_hex, t.raw_hex) AS raw_hex, + datetime(o.timestamp, 'unixepoch') AS timestamp, + obs.id AS observer_id, obs.name AS observer_name, + o.direction, o.snr, o.rssi, o.score, t.hash, t.route_type, + t.payload_type, t.payload_version, o.path_json, t.decoded_json, + t.created_at + FROM observations o + JOIN transmissions t ON t.id = o.transmission_id + LEFT JOIN observers obs ON obs.rowid = o.observer_idx AND (obs.inactive IS NULL OR obs.inactive = 0) + `) + if vErr != nil { + return fmt.Errorf("packets_v view: %w", vErr) + } + + // One-time migration: recalculate advert_count to count unique transmissions only + db.Exec(`CREATE TABLE IF NOT EXISTS _migrations (name TEXT PRIMARY KEY)`) + var migDone int + row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'advert_count_unique_v1'") + if row.Scan(&migDone) != nil { + log.Println("[migration] Recalculating advert_count (unique transmissions only)...") + // Note: this migration is gated on a one-shot _migrations row, so it + // runs at most once per DB. The historical version used a LIKE-on-JSON + // substring match (#1143). Switching to from_pubkey here is safe even + // though the column may not yet be backfilled on legacy DBs: the + // migration is already marked done on those DBs and won't re-run. + db.Exec(` + UPDATE nodes SET advert_count = ( + SELECT COUNT(*) FROM transmissions t + WHERE t.payload_type = 4 + AND t.from_pubkey = nodes.public_key + ) + `) + db.Exec(`INSERT INTO _migrations (name) VALUES ('advert_count_unique_v1')`) + log.Println("[migration] advert_count recalculated") + } + + // One-time migration: change noise_floor from INTEGER to REAL affinity. + // SQLite doesn't support ALTER COLUMN, but existing float values are stored + // as REAL regardless of column affinity. New table definition already uses REAL. + // This migration casts any integer-stored noise_floor values to real. + row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'noise_floor_real_v1'") + if row.Scan(&migDone) != nil { + log.Println("[migration] Ensuring noise_floor values are stored as REAL...") + db.Exec(`UPDATE observers SET noise_floor = CAST(noise_floor AS REAL) WHERE noise_floor IS NOT NULL AND typeof(noise_floor) = 'integer'`) + db.Exec(`INSERT INTO _migrations (name) VALUES ('noise_floor_real_v1')`) + log.Println("[migration] noise_floor migration complete") + } + + // One-time migration: add telemetry columns to nodes and inactive_nodes tables. + row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'node_telemetry_v1'") + if row.Scan(&migDone) != nil { + log.Println("[migration] Adding telemetry columns to nodes/inactive_nodes...") + + // checkAndAddColumn checks whether `column` already exists in `table` + // using PRAGMA table_info, and adds it if missing. All call sites pass + // hardcoded table/column/type literals so there is no SQL injection risk. + checkAndAddColumn := func(table, column, colType string) error { + rows, err := db.Query(fmt.Sprintf("PRAGMA table_info(%s)", table)) + if err != nil { + return fmt.Errorf("querying table info for %s: %w", table, err) + } + defer rows.Close() + + exists := false + for rows.Next() { + var cid int + var name, ctype string + var notnull, pk int + var dfltValue sql.NullString + if err := rows.Scan(&cid, &name, &ctype, ¬null, &dfltValue, &pk); err != nil { + return fmt.Errorf("scanning table info for %s: %w", table, err) + } + if name == column { + exists = true + break + } + } + if err := rows.Err(); err != nil { + return fmt.Errorf("iterating table info for %s: %w", table, err) + } + if exists { + return nil + } + if _, err := db.Exec(fmt.Sprintf("ALTER TABLE %s ADD COLUMN %s %s", table, column, colType)); err != nil { + return fmt.Errorf("adding column %s to %s: %w", column, table, err) + } + return nil + } + + if err := checkAndAddColumn("nodes", "battery_mv", "INTEGER"); err != nil { + return err + } + if err := checkAndAddColumn("nodes", "temperature_c", "REAL"); err != nil { + return err + } + if err := checkAndAddColumn("inactive_nodes", "battery_mv", "INTEGER"); err != nil { + return err + } + if err := checkAndAddColumn("inactive_nodes", "temperature_c", "REAL"); err != nil { + return err + } + if _, err := db.Exec(`INSERT INTO _migrations (name) VALUES ('node_telemetry_v1')`); err != nil { + return fmt.Errorf("recording node_telemetry_v1 migration: %w", err) + } + log.Println("[migration] node telemetry columns added") + } + + // One-time migration: add timestamp index on observations for fast stats queries. + // Older databases created before this index was added suffer from full table scans + // on COUNT(*) WHERE timestamp > ?, causing /api/stats to take 30s+. + row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'obs_timestamp_index_v1'") + if row.Scan(&migDone) != nil { + log.Println("[migration] Adding timestamp index on observations...") + db.Exec(`CREATE INDEX IF NOT EXISTS idx_observations_timestamp ON observations(timestamp)`) + db.Exec(`INSERT INTO _migrations (name) VALUES ('obs_timestamp_index_v1')`) + log.Println("[migration] observations timestamp index created") + } + + // #1481 P0-3: covering index for GetObserverPacketCounts. The query + // joins observations → observers and GROUP BYs observer_idx with a + // timestamp WHERE filter; a composite (observer_idx, timestamp) + // index lets SQLite resolve the grouping + range filter from the + // index alone instead of a 1.9M-row scan. + // + // CONVERTED TO ASYNC (preflight-async-migration-gate). Scheduling + // happens in OpenStore() once the real *Store exists so the + // backfill WaitGroup is shared with the rest of the ingestor. + // The legacy `_migrations` gate is preserved by the async fn so + // DBs that already completed the sync build stay no-op. + + // #1483: normalize nodes.public_key to lowercase. The server's + // GetNodeLocationsByKeys lookup dropped LOWER(public_key) for perf + // (#1481 P0-3) and now relies on stored keys being lowercase. The + // decoder writes lowercase today, but legacy/admin/API inserts may + // have left mixed-case rows. Idempotent: counts and lowers any + // non-lowercase rows on every boot, runs once via _migrations gate + // for the bulk fix. Re-running stays cheap because subsequent + // passes match zero rows. + if r := db.QueryRow("SELECT COUNT(*) FROM nodes WHERE public_key != lower(public_key)"); r != nil { + var n int64 + _ = r.Scan(&n) + if n > 0 { + log.Printf("[migration] Normalizing %d nodes.public_key row(s) to lowercase (#1483)...", n) + if _, err := db.Exec(`UPDATE nodes SET public_key = lower(public_key) WHERE public_key != lower(public_key)`); err != nil { + log.Printf("[migration] public_key lowercase normalize failed: %v", err) + } else { + log.Printf("[migration] public_key lowercase normalize complete (%d rows)", n) + } + } + } + + // observer_metrics table for RF health dashboard + row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'observer_metrics_v1'") + if row.Scan(&migDone) != nil { + log.Println("[migration] Creating observer_metrics table...") + _, err := db.Exec(` + CREATE TABLE IF NOT EXISTS observer_metrics ( + observer_id TEXT NOT NULL, + timestamp TEXT NOT NULL, + noise_floor REAL, + tx_air_secs INTEGER, + rx_air_secs INTEGER, + recv_errors INTEGER, + battery_mv INTEGER, + PRIMARY KEY (observer_id, timestamp) + ) + `) + if err != nil { + return fmt.Errorf("observer_metrics schema: %w", err) + } + db.Exec(`INSERT INTO _migrations (name) VALUES ('observer_metrics_v1')`) + log.Println("[migration] observer_metrics table created") + } + + // Migration: add timestamp index for cross-observer time-range queries + row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'observer_metrics_ts_idx'") + if row.Scan(&migDone) != nil { + log.Println("[migration] Creating observer_metrics timestamp index...") + _, err := db.Exec(`CREATE INDEX IF NOT EXISTS idx_observer_metrics_timestamp ON observer_metrics(timestamp)`) + if err != nil { + return fmt.Errorf("observer_metrics timestamp index: %w", err) + } + db.Exec(`INSERT INTO _migrations (name) VALUES ('observer_metrics_ts_idx')`) + log.Println("[migration] observer_metrics timestamp index created") + } + + // Migration: add inactive column to observers for soft-delete retention + row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'observers_inactive_v1'") + if row.Scan(&migDone) != nil { + log.Println("[migration] Adding inactive column to observers...") + _, err := db.Exec(`ALTER TABLE observers ADD COLUMN inactive INTEGER DEFAULT 0`) + if err != nil { + // Column may already exist (e.g. fresh install with schema above) + log.Printf("[migration] observers.inactive: %v (may already exist)", err) + } + db.Exec(`INSERT INTO _migrations (name) VALUES ('observers_inactive_v1')`) + log.Println("[migration] observers.inactive column added") + } + + // Migration: add packets_sent and packets_recv columns to observer_metrics + row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'observer_metrics_packets_v1'") + if row.Scan(&migDone) != nil { + log.Println("[migration] Adding packets_sent/packets_recv columns to observer_metrics...") + db.Exec(`ALTER TABLE observer_metrics ADD COLUMN packets_sent INTEGER`) + db.Exec(`ALTER TABLE observer_metrics ADD COLUMN packets_recv INTEGER`) + db.Exec(`INSERT INTO _migrations (name) VALUES ('observer_metrics_packets_v1')`) + log.Println("[migration] packets_sent/packets_recv columns added") + } + + // Migration: add channel_hash column for fast channel queries (#762) + row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'channel_hash_v1'") + if row.Scan(&migDone) != nil { + log.Println("[migration] Adding channel_hash column to transmissions...") + db.Exec(`ALTER TABLE transmissions ADD COLUMN channel_hash TEXT DEFAULT NULL`) + db.Exec(`CREATE INDEX IF NOT EXISTS idx_tx_channel_hash ON transmissions(channel_hash) WHERE payload_type = 5`) + // Backfill: extract channel name for decrypted (CHAN) packets + res, err := db.Exec(`UPDATE transmissions SET channel_hash = json_extract(decoded_json, '$.channel') WHERE payload_type = 5 AND channel_hash IS NULL AND json_extract(decoded_json, '$.type') = 'CHAN'`) + if err == nil { + n, _ := res.RowsAffected() + log.Printf("[migration] Backfilled channel_hash for %d CHAN packets", n) + } + // Backfill: extract channelHashHex for encrypted (GRP_TXT) packets, prefixed with 'enc_' + res, err = db.Exec(`UPDATE transmissions SET channel_hash = 'enc_' || json_extract(decoded_json, '$.channelHashHex') WHERE payload_type = 5 AND channel_hash IS NULL AND json_extract(decoded_json, '$.type') = 'GRP_TXT'`) + if err == nil { + n, _ := res.RowsAffected() + log.Printf("[migration] Backfilled channel_hash for %d GRP_TXT packets", n) + } + db.Exec(`INSERT INTO _migrations (name) VALUES ('channel_hash_v1')`) + log.Println("[migration] channel_hash column added and backfilled") + } + + // Migration: dropped_packets table for signature validation failures (#793) + row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'dropped_packets_v1'") + if row.Scan(&migDone) != nil { + log.Println("[migration] Creating dropped_packets table...") + _, err := db.Exec(` + CREATE TABLE IF NOT EXISTS dropped_packets ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + hash TEXT, + raw_hex TEXT, + reason TEXT NOT NULL, + observer_id TEXT, + observer_name TEXT, + node_pubkey TEXT, + node_name TEXT, + dropped_at DATETIME DEFAULT CURRENT_TIMESTAMP + ); + CREATE INDEX IF NOT EXISTS idx_dropped_observer ON dropped_packets(observer_id); + CREATE INDEX IF NOT EXISTS idx_dropped_node ON dropped_packets(node_pubkey); + `) + if err != nil { + return fmt.Errorf("dropped_packets schema: %w", err) + } + db.Exec(`INSERT INTO _migrations (name) VALUES ('dropped_packets_v1')`) + log.Println("[migration] dropped_packets table created") + } + + // Migration: observations.raw_hex (#881) is now owned by + // internal/dbschema/dbschema.go (#1321). The server PRAGMA-detects + // this column as hasObsRawHex; keeping a single canonical Apply + // path closes the startup race where the server's detector ran + // before this ALTER finished. + + // Migration: transmissions.scope_name (#899) is now owned by + // internal/dbschema/dbschema.go (#1321). See above. + + // Migration: add last_packet_at column to observers (#last-packet-at) + row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'observers_last_packet_at_v1'") + if row.Scan(&migDone) != nil { + log.Println("[migration] Adding last_packet_at column to observers...") + _, alterErr := db.Exec(`ALTER TABLE observers ADD COLUMN last_packet_at TEXT DEFAULT NULL`) + if alterErr != nil && !strings.Contains(alterErr.Error(), "duplicate column") { + return fmt.Errorf("observers last_packet_at ALTER: %w", alterErr) + } + // Backfill: set last_packet_at = last_seen only for observers that actually have + // observation rows (packet_count alone is unreliable — UpsertObserver sets it to 1 + // on INSERT even for status-only observers). + res, err := db.Exec(`UPDATE observers SET last_packet_at = last_seen + WHERE last_packet_at IS NULL + AND rowid IN (SELECT DISTINCT observer_idx FROM observations WHERE observer_idx IS NOT NULL)`) + if err == nil { + n, _ := res.RowsAffected() + log.Printf("[migration] Backfilled last_packet_at for %d observers with packets", n) + } + db.Exec(`INSERT INTO _migrations (name) VALUES ('observers_last_packet_at_v1')`) + log.Println("[migration] observers.last_packet_at column added") + } + + // Migration: per-observer naive-clock skew tracking (#1478). + // When the ingestor clamps a packet's envelope timestamp because the + // observer emitted a zone-less local-time string off from UTC by >15min + // (resolveRxTime in main.go), we record the event here so the UI can + // surface a ⚠️ chip + banner. Decays after 24h via server-side read sweep. + row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'observers_clock_naive_v1'") + if row.Scan(&migDone) != nil { + log.Println("[migration] Adding clock-naive columns to observers (#1478)...") + // Each ALTER is independent — ignore "duplicate column" so reruns are safe. + for _, stmt := range []string{ + `ALTER TABLE observers ADD COLUMN clock_skew_seconds INTEGER DEFAULT NULL`, + `ALTER TABLE observers ADD COLUMN clock_skew_count_24h INTEGER DEFAULT 0`, + `ALTER TABLE observers ADD COLUMN clock_last_naive_at TEXT DEFAULT NULL`, + } { + if _, err := db.Exec(stmt); err != nil && !strings.Contains(err.Error(), "duplicate column") { + return fmt.Errorf("clock_naive migration: %w", err) + } + } + db.Exec(`INSERT INTO _migrations (name) VALUES ('observers_clock_naive_v1')`) + log.Println("[migration] observers.clock_naive columns added") + } + + // Migration: backfill observations.path_json from raw_hex (#888) + // NOTE: This runs ASYNC via BackfillPathJSONAsync() to avoid blocking MQTT startup. + // See staging outage where ~502K rows blocked ingest for 15+ hours. + + // One-time cleanup: delete legacy packets with empty hash or empty first_seen (#994) + row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'cleanup_legacy_null_hash_ts'") + if row.Scan(&migDone) != nil { + log.Println("[migration] Cleaning up legacy packets with empty hash/timestamp...") + db.Exec(`DELETE FROM observations WHERE transmission_id IN (SELECT id FROM transmissions WHERE hash = '' OR first_seen = '')`) + res, err := db.Exec(`DELETE FROM transmissions WHERE hash = '' OR first_seen = ''`) + if err == nil { + deleted, _ := res.RowsAffected() + log.Printf("[migration] deleted %d legacy packets with empty hash/timestamp", deleted) + } + db.Exec(`INSERT INTO _migrations (name) VALUES ('cleanup_legacy_null_hash_ts')`) + } + + // Migration: foreign_advert column on nodes/inactive_nodes (#730) + // Marks nodes whose ADVERT GPS lies outside the configured geofilter polygon. + // Default 0; set to 1 by the ingestor when GeoFilter is configured and + // PassesFilter() returns false. Allows operators to surface bridged/leaked + // adverts without silently dropping them. + row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'foreign_advert_v1'") + if row.Scan(&migDone) != nil { + log.Println("[migration] Adding foreign_advert column to nodes/inactive_nodes...") + if _, err := db.Exec(`ALTER TABLE nodes ADD COLUMN foreign_advert INTEGER DEFAULT 0`); err != nil { + log.Printf("[migration] nodes.foreign_advert: %v (may already exist)", err) + } + if _, err := db.Exec(`ALTER TABLE inactive_nodes ADD COLUMN foreign_advert INTEGER DEFAULT 0`); err != nil { + log.Printf("[migration] inactive_nodes.foreign_advert: %v (may already exist)", err) + } + db.Exec(`CREATE INDEX IF NOT EXISTS idx_nodes_foreign_advert ON nodes(foreign_advert) WHERE foreign_advert = 1`) + db.Exec(`INSERT INTO _migrations (name) VALUES ('foreign_advert_v1')`) + log.Println("[migration] foreign_advert column added") + } + + // Migration: from_pubkey column on transmissions (#1143). + // Replaces the unsound `decoded_json LIKE '%pubkey%'` attribution path with + // an exact-match indexed column. Synchronously adds the column + index; + // row-level backfill is run by the SERVER asynchronously + // (cmd/server/from_pubkey_migration.go) so we don't block ingestor boot. + row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'from_pubkey_v1'") + if row.Scan(&migDone) != nil { + log.Println("[migration] Adding from_pubkey column + index to transmissions (#1143)...") + if _, err := db.Exec(`ALTER TABLE transmissions ADD COLUMN from_pubkey TEXT`); err != nil { + log.Printf("[migration] transmissions.from_pubkey: %v (may already exist)", err) + } + if _, err := db.Exec(`CREATE INDEX IF NOT EXISTS idx_transmissions_from_pubkey ON transmissions(from_pubkey)`); err != nil { + log.Printf("[migration] idx_transmissions_from_pubkey: %v", err) + } + db.Exec(`INSERT INTO _migrations (name) VALUES ('from_pubkey_v1')`) + log.Println("[migration] from_pubkey column + index added") + } + + // Migration: nodes.default_scope (#899 Feature 3) is now owned by + // internal/dbschema/dbschema.go (#1321). The server PRAGMA-detects + // this column as hasDefaultScope; keeping a single canonical Apply + // path closes the startup race that #1321 documented. + + // Migration: normalize known channel_hash values for existing rows. + // Before this PR, config key "public" was stored as channel_hash="public". + // After this PR, new rows use channel_hash="Public". Without backfill, + // channel grouping queries split into two buckets across the upgrade boundary. + row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'channel_hash_casing_v1'") + if row.Scan(&migDone) != nil { + log.Println("[migration] Normalizing known channel_hash values...") + res, err := db.Exec(`UPDATE transmissions SET channel_hash = 'Public' WHERE channel_hash = 'public' AND payload_type = 5`) + if err != nil { + log.Printf("[migration] ERROR: failed to normalize channel_hash: %v", err) + return fmt.Errorf("migration channel_hash_casing_v1 UPDATE failed: %w", err) + } + n, _ := res.RowsAffected() + log.Printf("[migration] Normalized %d channel_hash rows from 'public' to 'Public'", n) + if _, err := db.Exec(`INSERT OR IGNORE INTO _migrations (name) VALUES ('channel_hash_casing_v1')`); err != nil { + log.Printf("[migration] WARNING: failed to record migration: %v", err) + } + log.Println("[migration] channel_hash casing normalization complete") + } + + // Migration: transmissions.code1/code2 — the transport codes were decoded + // and discarded; storing them makes scope forwarding queryable per repeater. + row = db.QueryRow("SELECT 1 FROM _migrations WHERE name = 'transport_codes_v1'") + if row.Scan(new(int)) != nil { + log.Println("[migration] Adding code1/code2 columns to transmissions...") + // Each ALTER is independent — ignore "duplicate column" so reruns (and + // the fresh-database path, where the base schema already created the + // columns) are safe. Any other failure must stop before the guard row + // is inserted, or prepareStatements' code1/code2 INSERT fails forever + // on every subsequent restart with no way to retry the migration. + for _, stmt := range []string{ + `ALTER TABLE transmissions ADD COLUMN code1 TEXT DEFAULT NULL`, + `ALTER TABLE transmissions ADD COLUMN code2 TEXT DEFAULT NULL`, + } { + if _, err := db.Exec(stmt); err != nil && !strings.Contains(err.Error(), "duplicate column") { + return fmt.Errorf("transport_codes_v1 migration: %w", err) + } + } + db.Exec(`CREATE INDEX IF NOT EXISTS idx_tx_code1 ON transmissions(code1) WHERE code1 IS NOT NULL`) + db.Exec(`INSERT INTO _migrations (name) VALUES ('transport_codes_v1')`) + log.Println("[migration] code1/code2 columns added") + } + + return nil +} + +// Normalize upgrades an offline SQLite working copy before import. +func Normalize(db *sql.DB, logf Logger) error { + if err := CheckLegacySource(db); err != nil { + return err + } + if err := ApplyBase(db); err != nil { + return err + } + return Apply(db, logf) +} diff --git a/internal/dbschema/legacy/check.go b/internal/dbschema/legacy/check.go new file mode 100644 index 000000000..636006fd2 --- /dev/null +++ b/internal/dbschema/legacy/check.go @@ -0,0 +1,50 @@ +package legacy + +import ( + "database/sql" + "fmt" +) + +// CheckLegacySource rejects layouts or data the historical normalizer would +// discard. The importer also rejects unknown tables/columns before copying. +// Supported layouts are v3 observations with the known optional columns absent. +func CheckLegacySource(db *sql.DB) error { + var integrity string + if err := db.QueryRow(`PRAGMA quick_check`).Scan(&integrity); err != nil { + return err + } + if integrity != "ok" { + return fmt.Errorf("legacy SQLite integrity check failed") + } + for _, table := range []string{"nodes", "observers", "transmissions", "observations"} { + var n int + if err := db.QueryRow(`SELECT count(*) FROM sqlite_master WHERE type='table' AND name=?`, table).Scan(&n); err != nil { + return err + } + if n != 1 { + return fmt.Errorf("unsupported legacy schema: missing %s", table) + } + } + has, err := TableHasColumn(db, "observations", "observer_idx") + if err != nil { + return err + } + if !has { + return fmt.Errorf("legacy v2 observer_id layout is unsupported; upgrade an offline copy with the last SQLite release before importing") + } + checks := []struct{ name, query string }{ + {"empty transmission hash/timestamp", `SELECT count(*) FROM transmissions WHERE hash IS NULL OR hash='' OR first_seen IS NULL OR first_seen=''`}, + {"case-colliding node public keys", `SELECT count(*) FROM (SELECT lower(public_key) FROM nodes GROUP BY lower(public_key) HAVING count(*)>1)`}, + {"duplicate observation identity", `SELECT count(*) FROM (SELECT transmission_id,observer_idx,coalesce(path_json,'') FROM observations WHERE observer_idx IS NOT NULL GROUP BY transmission_id,observer_idx,coalesce(path_json,'') HAVING count(*)>1)`}, + } + for _, check := range checks { + var n int + if err := db.QueryRow(check.query).Scan(&n); err != nil { + return err + } + if n > 0 { + return fmt.Errorf("legacy source contains %s (%d groups/rows); resolve explicitly before import", check.name, n) + } + } + return nil +} diff --git a/internal/dbschema/legacy/dbschema.go b/internal/dbschema/legacy/dbschema.go new file mode 100644 index 000000000..5c2987c34 --- /dev/null +++ b/internal/dbschema/legacy/dbschema.go @@ -0,0 +1,756 @@ +// Package dbschema centralizes schema migrations and read-side schema +// assertions for the CoreScope SQLite DB. Per issue #1287 the writer +// (cmd/ingestor) owns ALL CREATE/ALTER/INSERT/UPDATE/DELETE on schema +// objects; the server (cmd/server) only ASSERTS that the schema is in +// the expected shape and refuses to start otherwise. +// +// Apply(rw, log) runs from the ingestor at startup BEFORE subscribing to +// MQTT. AssertReady(ro) runs from the server at startup and returns an +// error listing every missing column/index/table. +// +// INVARIANT (#1321): Any optional column the SERVER detects via PRAGMA +// (e.g. cmd/server/db.go detectSchema → hasScopeName, hasDefaultScope, +// hasObsRawHex, hasResolvedPath) MUST be added here, in Apply, AND +// asserted in AssertReady. Adding it only to cmd/ingestor/db.go +// applySchema reintroduces the startup-race bug from #1321: the +// server's PRAGMA can run before the ingestor finishes applySchema, +// the boolean caches `false`, and feature endpoints permanently 500 +// until the server restarts. Source of truth lives here — full stop. +package legacy + +import ( + "database/sql" + "errors" + "fmt" + "strings" +) + +// Logger is the minimal logging surface used by Apply. Both cmd/server +// and cmd/ingestor satisfy this with the stdlib `log` package's Printf +// (passed as a closure to avoid an indirect log dependency here). +type Logger func(format string, args ...interface{}) + +// Apply runs every server-side ensure_* migration against the given +// read-write SQLite connection. Each operation is idempotent +// (IF NOT EXISTS / column-probe-before-ALTER). Safe to call repeatedly. +// +// Called by the ingestor at startup. The server MUST NOT call this — +// it only calls AssertReady. +func Apply(rw *sql.DB, logf Logger) error { + if logf == nil { + logf = func(string, ...interface{}) {} + } + if err := ensureServerIndexes(rw); err != nil { + return fmt.Errorf("ensure server indexes: %w", err) + } + if err := ensureObservationsDedupIndex(rw, logf); err != nil { + return fmt.Errorf("ensure observations dedup index: %w", err) + } + if err := ensureNeighborEdgesTable(rw); err != nil { + return fmt.Errorf("ensure neighbor_edges: %w", err) + } + if err := ensureInactiveNodesTable(rw); err != nil { + return fmt.Errorf("ensure inactive_nodes: %w", err) + } + if err := ensureResolvedPathColumn(rw, logf); err != nil { + return fmt.Errorf("ensure resolved_path: %w", err) + } + if err := ensureObserverInactiveColumn(rw, logf); err != nil { + return fmt.Errorf("ensure observers.inactive: %w", err) + } + if err := ensureLastPacketAtColumn(rw, logf); err != nil { + return fmt.Errorf("ensure observers.last_packet_at: %w", err) + } + if err := ensureObserverIATAColumn(rw, logf); err != nil { + return fmt.Errorf("ensure observers.iata: %w", err) + } + // Must run after ensureObserverIATAColumn: the expression index below + // is on observers.iata, which that step guarantees exists. + if err := ensureChannelIndexes(rw); err != nil { + return fmt.Errorf("ensure channel indexes: %w", err) + } + if err := ensureForeignAdvertColumn(rw, logf); err != nil { + return fmt.Errorf("ensure foreign_advert: %w", err) + } + if err := ensureFromPubkeyColumn(rw, logf); err != nil { + return fmt.Errorf("ensure from_pubkey: %w", err) + } + if err := ensureScopeNameColumn(rw, logf); err != nil { + return fmt.Errorf("ensure scope_name: %w", err) + } + if err := ensureDefaultScopeColumns(rw, logf); err != nil { + return fmt.Errorf("ensure default_scope: %w", err) + } + if err := ensureConfiguredScopeColumns(rw, logf); err != nil { + return fmt.Errorf("ensure configured_scope: %w", err) + } + if err := ensureObservationsRawHexColumn(rw, logf); err != nil { + return fmt.Errorf("ensure observations.raw_hex: %w", err) + } + if err := ensureMultibyteCapColumns(rw, logf); err != nil { + return fmt.Errorf("ensure multibyte_cap columns: %w", err) + } + if err := ensureObserverNaiveClockColumns(rw, logf); err != nil { + return fmt.Errorf("ensure observers naive-clock columns: %w", err) + } + if err := ensureObserverCanRelayColumn(rw, logf); err != nil { + return fmt.Errorf("ensure observers.can_relay: %w", err) + } + if err := ensureObserverCanRelaySeenColumn(rw, logf); err != nil { + return fmt.Errorf("ensure observers.can_relay_seen: %w", err) + } + // #1690: denormalized last_seen on transmissions so cold-load filters + // on effective recency rather than first-ever first_seen. The column + // add + index creation are cheap (single ALTER, indexed INTEGER + // column); the *backfill* from observations is potentially expensive + // and runs via RunAsyncMigration from the ingestor (see + // cmd/ingestor/db.go OpenStore). + if err := ensureTransmissionsLastSeenColumn(rw, logf); err != nil { + return fmt.Errorf("ensure transmissions.last_seen: %w", err) + } + if err := ensureAdvertEvidence(rw); err != nil { + return fmt.Errorf("ensure advert route evidence: %w", err) + } + if err := ensureKeyBackfill(rw); err != nil { + return fmt.Errorf("ensure key backfill: %w", err) + } + // Advert evidence is intentionally not required by AssertReady: a legacy + // read-only server reports unknown until this table/backfill is available, + // and re-probes absence instead of caching the startup race permanently. + return nil +} + +// AssertReady verifies the schema is in the expected shape. The server +// calls this at startup against a read-only connection; if it returns +// non-nil, the server MUST fatal-log and exit so the operator restarts +// the ingestor (which owns migrations). +func AssertReady(ro *sql.DB) error { + var missing []string + + mustCol := func(table, col string) { + has, err := TableHasColumn(ro, table, col) + if err != nil { + missing = append(missing, fmt.Sprintf("%s.%s (probe error: %v)", table, col, err)) + return + } + if !has { + missing = append(missing, fmt.Sprintf("%s.%s", table, col)) + } + } + mustTable := func(name string) { + var n int + err := ro.QueryRow(`SELECT 1 FROM sqlite_master WHERE type='table' AND name=?`, name).Scan(&n) + if errors.Is(err, sql.ErrNoRows) { + missing = append(missing, "table:"+name) + } else if err != nil { + missing = append(missing, fmt.Sprintf("table:%s (probe error: %v)", name, err)) + } + } + + mustTable("neighbor_edges") + mustCol("observations", "resolved_path") + mustCol("observers", "inactive") + mustCol("observers", "last_packet_at") + mustCol("observers", "iata") + mustCol("nodes", "foreign_advert") + mustCol("inactive_nodes", "foreign_advert") + mustCol("transmissions", "from_pubkey") + // #1690: denormalized recency axis for cold-load. Owned by ingestor + // (cmd/ingestor/db.go OpenStore: ALTER + index + async backfill); + // server reads it to filter the hot-window query. + mustCol("transmissions", "last_seen") + // #1321: server's detectSchema PRAGMA-detects these and caches a + // boolean. To kill the startup race they're owned + asserted here. + mustCol("transmissions", "scope_name") + mustCol("nodes", "default_scope") + mustCol("inactive_nodes", "default_scope") + // #1865: confirmed region scopes from the observer /neighbors report. + mustCol("nodes", "configured_scope") + mustCol("inactive_nodes", "configured_scope") + mustCol("observations", "raw_hex") + // Multi-byte capability cache (#1324 follow-up; PR #903 surface). + // Owned by ingestor — server reads these for O(1) /api/nodes + // enrichment, ingestor's RunMultibyteCapPersist is the only writer. + mustCol("nodes", "multibyte_sup") + mustCol("nodes", "multibyte_evidence") + mustCol("inactive_nodes", "multibyte_sup") + mustCol("inactive_nodes", "multibyte_evidence") + // Issue #1478: per-observer naive-clock skew tracking. Server reads + // all three to populate ObserverResp.ClockNaive / ClockSkew* fields. + mustCol("observers", "clock_skew_seconds") + mustCol("observers", "clock_skew_count_24h") + mustCol("observers", "clock_last_naive_at") + // Issue #1290: firmware 1.16 publishes a `repeat: on|off` flag in + // the MQTT /status JSON. Ingestor persists it as can_relay; server + // reads it to filter listener-only observers out of the path-hop + // disambiguator candidate set. Default 1 preserves prior behavior + // for legacy observers that never sent the field. + mustCol("observers", "can_relay") + // Issue #1290 follow-up (PR #1624 MAJOR-2): tri-state badge. The + // can_relay column defaults to 1 at INSERT, so we cannot distinguish + // "firmware confirmed repeater" from "legacy observer that never + // sent a repeat field". can_relay_seen=1 means the ingestor wrote + // an explicit value; can_relay_seen=0 means leave the UI badge + // unset (unknown state). + mustCol("observers", "can_relay_seen") + + if len(missing) > 0 { + return fmt.Errorf("schema not migrated by ingestor; restart ingestor first. missing: %s", + strings.Join(missing, ", ")) + } + return nil +} + +// Querier is the read surface shared by *sql.DB and *sql.Tx. +// +// Not *sql.Conn: it exposes only QueryContext/QueryRowContext, so it does not +// satisfy this. Widen the interface to the Context variants if a caller ever +// needs one. +// +// It exists so schema probes can run on whichever handle the caller already +// holds. Taking *sql.DB unconditionally is a deadlock waiting to happen: a +// caller inside a transaction has the connection checked out, and on a pool +// capped at one connection — which is what cmd/ingestor runs — a probe against +// the pool waits forever for the connection its own transaction is holding. +type Querier interface { + Query(query string, args ...any) (*sql.Rows, error) + QueryRow(query string, args ...any) *sql.Row +} + +// TableHasColumn reports whether the given table has the given column. +// Exported because tests and the read-side need it without re-implementing. +// +// Pass the transaction, not the pool, when you are inside one. See Querier. +func TableHasColumn(db Querier, table, column string) (bool, error) { + rows, err := db.Query(fmt.Sprintf("PRAGMA table_info(%s)", table)) + if err != nil { + return false, err + } + defer rows.Close() + for rows.Next() { + var cid int + var name string + var ctype sql.NullString + var notnull, pk int + var dflt sql.NullString + if err := rows.Scan(&cid, &name, &ctype, ¬null, &dflt, &pk); err != nil { + return false, err + } + if name == column { + return true, nil + } + } + return false, rows.Err() +} + +// ─── ensure_* helpers (writer side) ──────────────────────────────────────── + +func ensureServerIndexes(rw *sql.DB) error { + stmts := []string{ + `CREATE INDEX IF NOT EXISTS idx_transmissions_first_seen ON transmissions(first_seen)`, + `CREATE INDEX IF NOT EXISTS idx_transmissions_hash ON transmissions(hash)`, + `CREATE INDEX IF NOT EXISTS idx_transmissions_payload_type ON transmissions(payload_type)`, + `CREATE INDEX IF NOT EXISTS idx_observations_timestamp ON observations(timestamp)`, + `CREATE INDEX IF NOT EXISTS idx_observations_transmission_id ON observations(transmission_id)`, + // Composite covers GetChannelMessages' grouped MAX(timestamp) per + // transmission_id (issue #1366 / PR #1368). With this index sqlite can + // satisfy the aggregate index-only without touching the heap. + `CREATE INDEX IF NOT EXISTS idx_observations_tx_ts ON observations(transmission_id, timestamp)`, + } + for _, s := range stmts { + if _, err := rw.Exec(s); err != nil { + return fmt.Errorf("ensure index %q: %w", s, err) + } + } + // observer_idx (v3) vs observer_id (v2) — probe + index the matching one. + hasIdx, err := TableHasColumn(rw, "observations", "observer_idx") + if err != nil { + return err + } + if hasIdx { + if _, err := rw.Exec(`CREATE INDEX IF NOT EXISTS idx_observations_observer_idx ON observations(observer_idx)`); err != nil { + return err + } + } + hasID, err := TableHasColumn(rw, "observations", "observer_id") + if err != nil { + return err + } + if hasID { + if _, err := rw.Exec(`CREATE INDEX IF NOT EXISTS idx_observations_observer_id ON observations(observer_id)`); err != nil { + return err + } + } + return nil +} + +// ensureChannelIndexes speeds up /api/channels and +// /api/channels/{hash}/messages. +// +// - idx_transmissions_channel_hash_payload adds first_seen to the +// existing channel_hash index (cmd/ingestor/db.go migration +// 'channel_hash_v1', idx_tx_channel_hash) so GetChannels' "latest +// message" correlated subquery (channel_hash + payload_type, ORDER BY +// first_seen DESC LIMIT 1) and GetChannelMessages' count/page queries +// resolve as index-order scans instead of sorting per channel. +// - idx_observers_iata_norm indexes the UPPER(TRIM(iata)) expression +// that GetChannels/GetEncryptedChannels/GetChannelMessages use for +// region filtering; a plain index on iata can't be used under that +// expression, so the expression itself is indexed. +// +// transmissions.channel_hash itself is added by the ingestor's legacy +// 'channel_hash_v1' migration (cmd/ingestor/db.go), not by this package, +// so it isn't guaranteed to exist yet on every DB Apply runs against — +// probe before indexing, same pattern as the observer_idx/observer_id +// probe in ensureServerIndexes above. +func ensureChannelIndexes(rw *sql.DB) error { + hasChannelHash, err := TableHasColumn(rw, "transmissions", "channel_hash") + if err != nil { + return err + } + if hasChannelHash { + if _, err := rw.Exec(`CREATE INDEX IF NOT EXISTS idx_transmissions_channel_hash_payload ON transmissions(channel_hash, payload_type, first_seen)`); err != nil { + return fmt.Errorf("ensure idx_transmissions_channel_hash_payload: %w", err) + } + } + if _, err := rw.Exec(`CREATE INDEX IF NOT EXISTS idx_observers_iata_norm ON observers(UPPER(TRIM(iata)))`); err != nil { + return fmt.Errorf("ensure idx_observers_iata_norm: %w", err) + } + return nil +} + +func ensureNeighborEdgesTable(rw *sql.DB) error { + _, err := rw.Exec(`CREATE TABLE IF NOT EXISTS neighbor_edges ( + node_a TEXT NOT NULL, + node_b TEXT NOT NULL, + count INTEGER DEFAULT 1, + last_seen TEXT, + PRIMARY KEY (node_a, node_b) + )`) + return err +} + +// ensureInactiveNodesTable creates the inactive_nodes table if missing. +// The ingestor's applySchema also creates this table — duplicating it +// here makes dbschema.Apply self-sufficient when called against a +// fixture DB that pre-dates the soft-delete feature (e.g. CI's +// test-fixtures/e2e-fixture.db, which never had any inactive rows). +// Schema kept in sync with cmd/ingestor/db.go:applySchema. +func ensureInactiveNodesTable(rw *sql.DB) error { + _, err := rw.Exec(`CREATE TABLE IF NOT EXISTS inactive_nodes ( + public_key TEXT PRIMARY KEY, + name TEXT, + role TEXT, + lat REAL, + lon REAL, + last_seen TEXT, + first_seen TEXT, + advert_count INTEGER DEFAULT 0, + battery_mv INTEGER, + temperature_c REAL, + foreign_advert INTEGER DEFAULT 0 + )`) + if err != nil { + return err + } + _, err = rw.Exec(`CREATE INDEX IF NOT EXISTS idx_inactive_nodes_last_seen ON inactive_nodes(last_seen)`) + return err +} + +func ensureResolvedPathColumn(rw *sql.DB, logf Logger) error { + has, err := TableHasColumn(rw, "observations", "resolved_path") + if err != nil { + return err + } + if has { + return nil + } + if _, err := rw.Exec("ALTER TABLE observations ADD COLUMN resolved_path TEXT"); err != nil { + return err + } + logf("[dbschema] added resolved_path column to observations") + return nil +} + +func ensureObserverInactiveColumn(rw *sql.DB, logf Logger) error { + has, err := TableHasColumn(rw, "observers", "inactive") + if err != nil { + return err + } + if has { + return nil + } + if _, err := rw.Exec("ALTER TABLE observers ADD COLUMN inactive INTEGER DEFAULT 0"); err != nil { + return err + } + logf("[dbschema] added inactive column to observers") + return nil +} + +func ensureLastPacketAtColumn(rw *sql.DB, logf Logger) error { + has, err := TableHasColumn(rw, "observers", "last_packet_at") + if err != nil { + return err + } + if has { + return nil + } + if _, err := rw.Exec("ALTER TABLE observers ADD COLUMN last_packet_at TEXT"); err != nil { + return err + } + logf("[dbschema] added last_packet_at column to observers") + return nil +} + +func ensureObserverIATAColumn(rw *sql.DB, logf Logger) error { + has, err := TableHasColumn(rw, "observers", "iata") + if err != nil { + return err + } + if has { + return nil + } + if _, err := rw.Exec("ALTER TABLE observers ADD COLUMN iata TEXT"); err != nil { + return err + } + logf("[dbschema] added iata column to observers") + return nil +} + +func ensureForeignAdvertColumn(rw *sql.DB, logf Logger) error { + for _, table := range []string{"nodes", "inactive_nodes"} { + has, err := TableHasColumn(rw, table, "foreign_advert") + if err != nil { + return fmt.Errorf("inspect %s: %w", table, err) + } + if has { + continue + } + if _, err := rw.Exec(fmt.Sprintf("ALTER TABLE %s ADD COLUMN foreign_advert INTEGER DEFAULT 0", table)); err != nil { + return err + } + logf("[dbschema] added foreign_advert column to %s", table) + } + return nil +} + +func ensureFromPubkeyColumn(rw *sql.DB, logf Logger) error { + has, err := TableHasColumn(rw, "transmissions", "from_pubkey") + if err != nil { + return err + } + if !has { + if _, err := rw.Exec("ALTER TABLE transmissions ADD COLUMN from_pubkey TEXT"); err != nil { + return err + } + logf("[dbschema] added from_pubkey column to transmissions (#1143)") + } + if _, err := rw.Exec("CREATE INDEX IF NOT EXISTS idx_transmissions_from_pubkey ON transmissions(from_pubkey)"); err != nil { + return err + } + return nil +} + +// ensureScopeNameColumn adds transmissions.scope_name (+ partial index) and +// records the legacy `_migrations` marker so the ingestor's old gated path +// stays idempotent on existing DBs. Moved here from cmd/ingestor/db.go in +// #1321 to be the canonical source of truth for the optional column the +// server PRAGMA-detects as hasScopeName. +func ensureScopeNameColumn(rw *sql.DB, logf Logger) error { + if err := ensureMigrationsTable(rw); err != nil { + return err + } + has, err := TableHasColumn(rw, "transmissions", "scope_name") + if err != nil { + return err + } + if !has { + if _, err := rw.Exec(`ALTER TABLE transmissions ADD COLUMN scope_name TEXT DEFAULT NULL`); err != nil { + return fmt.Errorf("alter transmissions add scope_name: %w", err) + } + logf("[dbschema] added scope_name column to transmissions (#899)") + } + if _, err := rw.Exec(`CREATE INDEX IF NOT EXISTS idx_tx_scope_name ON transmissions(scope_name) WHERE scope_name IS NOT NULL`); err != nil { + return fmt.Errorf("create idx_tx_scope_name: %w", err) + } + if _, err := rw.Exec(`INSERT OR IGNORE INTO _migrations (name) VALUES ('scope_name_v1')`); err != nil { + return fmt.Errorf("record scope_name_v1: %w", err) + } + return nil +} + +// ensureDefaultScopeColumns adds default_scope to nodes + inactive_nodes +// and records the `_migrations` marker. Source of truth lives here per +// #1321 (was previously cmd/ingestor/db.go only). +func ensureDefaultScopeColumns(rw *sql.DB, logf Logger) error { + if err := ensureMigrationsTable(rw); err != nil { + return err + } + for _, table := range []string{"nodes", "inactive_nodes"} { + has, err := TableHasColumn(rw, table, "default_scope") + if err != nil { + return fmt.Errorf("inspect %s.default_scope: %w", table, err) + } + if has { + continue + } + if _, err := rw.Exec(fmt.Sprintf(`ALTER TABLE %s ADD COLUMN default_scope TEXT DEFAULT NULL`, table)); err != nil { + return fmt.Errorf("alter %s add default_scope: %w", table, err) + } + logf("[dbschema] added default_scope column to %s (#899)", table) + } + if _, err := rw.Exec(`INSERT OR IGNORE INTO _migrations (name) VALUES ('nodes_default_scope_v1')`); err != nil { + return fmt.Errorf("record nodes_default_scope_v1: %w", err) + } + return nil +} + +// ensureConfiguredScopeColumns adds nodes.configured_scope + +// nodes.configured_scope_at (and mirrors on inactive_nodes) for #1865. +// Unlike default_scope (inferred from observed advert transport scope, and +// overwritten on every observation), configured_scope holds the region scopes +// a node has CONFIGURED, taken as concrete evidence from the observer +// /neighbors report — written only for the observer's own `self` scopes and +// for neighbors whose OTA scope query returned status="responded". The +// server PRAGMA-detects configured_scope as hasConfiguredScope. +func ensureConfiguredScopeColumns(rw *sql.DB, logf Logger) error { + if err := ensureMigrationsTable(rw); err != nil { + return err + } + for _, table := range []string{"nodes", "inactive_nodes"} { + for _, col := range []string{"configured_scope", "configured_scope_at"} { + has, err := TableHasColumn(rw, table, col) + if err != nil { + return fmt.Errorf("inspect %s.%s: %w", table, col, err) + } + if has { + continue + } + if _, err := rw.Exec(fmt.Sprintf(`ALTER TABLE %s ADD COLUMN %s TEXT DEFAULT NULL`, table, col)); err != nil { + return fmt.Errorf("alter %s add %s: %w", table, col, err) + } + logf("[dbschema] added %s column to %s (#1865)", col, table) + } + } + if _, err := rw.Exec(`INSERT OR IGNORE INTO _migrations (name) VALUES ('nodes_configured_scope_v1')`); err != nil { + return fmt.Errorf("record nodes_configured_scope_v1: %w", err) + } + return nil +} + +// ensureObservationsRawHexColumn adds observations.raw_hex (#881). +// Source of truth lives here per #1321 (was previously cmd/ingestor/db.go only): +// the server PRAGMA-detects this column as hasObsRawHex. +func ensureObservationsRawHexColumn(rw *sql.DB, logf Logger) error { + if err := ensureMigrationsTable(rw); err != nil { + return err + } + has, err := TableHasColumn(rw, "observations", "raw_hex") + if err != nil { + return err + } + if !has { + if _, err := rw.Exec(`ALTER TABLE observations ADD COLUMN raw_hex TEXT`); err != nil { + return fmt.Errorf("alter observations add raw_hex: %w", err) + } + logf("[dbschema] added raw_hex column to observations (#881)") + } + if _, err := rw.Exec(`INSERT OR IGNORE INTO _migrations (name) VALUES ('observations_raw_hex_v1')`); err != nil { + return fmt.Errorf("record observations_raw_hex_v1: %w", err) + } + return nil +} + +// ensureMigrationsTable is idempotent — the ingestor's applySchema also +// creates this table on legacy paths. Keeping it here lets the gated +// `INSERT OR IGNORE` markers above stay self-sufficient even when Apply +// runs against a brand-new fixture DB (e.g. dbschema_test.go). +func ensureMigrationsTable(rw *sql.DB) error { + _, err := rw.Exec(`CREATE TABLE IF NOT EXISTS _migrations (name TEXT PRIMARY KEY)`) + return err +} + +// SoftDeleteBlacklistedObservers marks the given observer IDs as +// inactive=1 (case-insensitive match). Returns count affected. +// Writer-side helper; ingestor calls it at startup with the operator +// blacklist (read from config). +func SoftDeleteBlacklistedObservers(rw *sql.DB, blacklist []string) (int64, error) { + placeholders := make([]string, 0, len(blacklist)) + args := make([]interface{}, 0, len(blacklist)) + for _, pk := range blacklist { + t := strings.TrimSpace(pk) + if t == "" { + continue + } + placeholders = append(placeholders, "LOWER(?)") + args = append(args, t) + } + if len(placeholders) == 0 { + return 0, nil + } + q := "UPDATE observers SET inactive = 1 WHERE LOWER(id) IN (" + + strings.Join(placeholders, ",") + ") AND (inactive IS NULL OR inactive = 0)" + res, err := rw.Exec(q, args...) + if err != nil { + return 0, err + } + n, _ := res.RowsAffected() + return n, nil +} + +// ensureMultibyteCapColumns adds the multi-byte capability cache columns +// to nodes / inactive_nodes (PR #903, canonical owner per #1324 +// follow-up). These columns are populated by the ingestor's +// RunMultibyteCapPersist from snapshot files written by the server's +// analytics cycle; the server is read-only since #1289 and MUST NOT +// write here. The schema itself lives here in dbschema (the writer +// owns migrations, the read-only server merely AssertReady's them). +func ensureMultibyteCapColumns(rw *sql.DB, logf Logger) error { + for _, table := range []string{"nodes", "inactive_nodes"} { + hasSup, err := TableHasColumn(rw, table, "multibyte_sup") + if err != nil { + return fmt.Errorf("inspect %s.multibyte_sup: %w", table, err) + } + if !hasSup { + if _, err := rw.Exec(fmt.Sprintf( + "ALTER TABLE %s ADD COLUMN multibyte_sup INTEGER NOT NULL DEFAULT 0", table)); err != nil { + return fmt.Errorf("add %s.multibyte_sup: %w", table, err) + } + logf("[dbschema] added multibyte_sup column to %s", table) + } + hasEvid, err := TableHasColumn(rw, table, "multibyte_evidence") + if err != nil { + return fmt.Errorf("inspect %s.multibyte_evidence: %w", table, err) + } + if !hasEvid { + if _, err := rw.Exec(fmt.Sprintf( + "ALTER TABLE %s ADD COLUMN multibyte_evidence TEXT", table)); err != nil { + return fmt.Errorf("add %s.multibyte_evidence: %w", table, err) + } + logf("[dbschema] added multibyte_evidence column to %s", table) + } + } + return nil +} + +// ensureObserverNaiveClockColumns adds the three per-observer naive-clock +// skew tracking columns (#1478). Server reads them to populate the +// clock_naive / clock_skew_seconds / clock_skew_count_24h / +// clock_last_naive_at fields in /api/observers responses; ingestor writes +// them from resolveRxTime via Store.RecordNaiveSkew on each clamp event. +// Owned here per the #1321 source-of-truth invariant — the cmd/ingestor +// copy migrates real prod DBs, dbschema migrates fixture/staging DBs via +// cmd/migrate. +func ensureObserverNaiveClockColumns(rw *sql.DB, logf Logger) error { + type col struct{ name, ddl string } + cols := []col{ + {"clock_skew_seconds", "ALTER TABLE observers ADD COLUMN clock_skew_seconds INTEGER DEFAULT NULL"}, + {"clock_skew_count_24h", "ALTER TABLE observers ADD COLUMN clock_skew_count_24h INTEGER DEFAULT 0"}, + {"clock_last_naive_at", "ALTER TABLE observers ADD COLUMN clock_last_naive_at TEXT DEFAULT NULL"}, + } + for _, c := range cols { + has, err := TableHasColumn(rw, "observers", c.name) + if err != nil { + return fmt.Errorf("inspect observers.%s: %w", c.name, err) + } + if has { + continue + } + if _, err := rw.Exec(c.ddl); err != nil { + return fmt.Errorf("add observers.%s: %w", c.name, err) + } + logf("[dbschema] added %s column to observers", c.name) + } + return nil +} + +// ensureObserverCanRelayColumn adds the can_relay column to observers. +// Firmware 1.16 publishes a `repeat: on|off` flag in the MQTT /status +// JSON (#1290); the ingestor parses it and writes 0/1 here. The server's +// path-hop disambiguator (cmd/server/store.go pm.resolveWithContext) +// excludes observers with can_relay=0 from the candidate set. Default 1 +// preserves prior behavior for legacy observers (no repeat field). +func ensureObserverCanRelayColumn(rw *sql.DB, logf Logger) error { + has, err := TableHasColumn(rw, "observers", "can_relay") + if err != nil { + return err + } + if has { + return nil + } + // PREFLIGHT: async=true reason="single-column ALTER on observers (low-cardinality, ~1k rows in prod); DEFAULT 1 is a constant so SQLite does the rewrite as a metadata-only schema update, no row scan" + if _, err := rw.Exec("ALTER TABLE observers ADD COLUMN can_relay INTEGER DEFAULT 1"); err != nil { + return err + } + logf("[dbschema] added can_relay column to observers") + return nil +} + +// ensureObserverCanRelaySeenColumn adds the can_relay_seen tracking column. +// Issue #1290 follow-up (PR #1624 MAJOR-2): can_relay defaults to 1 at +// INSERT, which conflates "confirmed repeater" with "legacy observer +// that never sent the repeat field". can_relay_seen=1 is written by +// the ingestor whenever the firmware actually provided the field; the +// server's read layer returns CanRelay=nil whenever seen=0 so the UI +// can render the tri-state badge (no badge for unknown). +func ensureObserverCanRelaySeenColumn(rw *sql.DB, logf Logger) error { + has, err := TableHasColumn(rw, "observers", "can_relay_seen") + if err != nil { + return err + } + if has { + return nil + } + // PREFLIGHT: async=true reason="single-column ALTER on observers (low-cardinality, ~1k rows in prod); DEFAULT 0 is a constant so SQLite does the rewrite as a metadata-only schema update, no row scan" + if _, err := rw.Exec("ALTER TABLE observers ADD COLUMN can_relay_seen INTEGER DEFAULT 0"); err != nil { + return err + } + logf("[dbschema] added can_relay_seen column to observers") + return nil +} + +// ensureTransmissionsLastSeenColumn adds transmissions.last_seen (#1690), +// the denormalized "most recent observation timestamp" column the cold-load +// path filters on. Without it, hot-window queries that match +// `first_seen >= cutoff` exclude long-lived hashes whose hashes were first +// inserted weeks ago but are still active — see the issue for the full +// 0.3%-of-DB-loaded post-mortem. +// +// The backfill from `MAX(observations.timestamp) GROUP BY transmission_id` +// is potentially expensive (1.9M+ obs rows in prod) and runs as a separate +// async migration scheduled by cmd/ingestor/db.go::OpenStore via +// Store.RunAsyncMigration. +func ensureTransmissionsLastSeenColumn(rw *sql.DB, logf Logger) error { + if err := ensureMigrationsTable(rw); err != nil { + return err + } + has, err := TableHasColumn(rw, "transmissions", "last_seen") + if err != nil { + return err + } + if !has { + // PREFLIGHT: async=true reason="single-column ALTER TABLE on transmissions; DEFAULT 0 is a constant so SQLite does a metadata-only schema rewrite (no row scan) — cheap at any scale" + if _, err := rw.Exec(`ALTER TABLE transmissions ADD COLUMN last_seen INTEGER NOT NULL DEFAULT 0`); err != nil { + return fmt.Errorf("alter transmissions add last_seen: %w", err) + } + logf("[dbschema] added last_seen column to transmissions (#1690)") + } + // #1740 step (a): create the PARTIAL index first. Scans only rows with + // last_seen=0 — i.e. the un-backfilled hot subset the chunked backfill + // MAX(id) lookup walks. Degenerates to near-empty once the backfill + // converges, so it stops competing for page cache. + // + // PREFLIGHT: async=false reason="partial index on `WHERE last_seen=0`; subset is bounded by un-backfilled rows (≤0 in steady state, ≤ingest-rate during ops) — cheap at any scale even on a 1.9M-row prod table" + if _, err := rw.Exec(`CREATE INDEX IF NOT EXISTS idx_tx_last_seen_zero ON transmissions(id) WHERE last_seen=0`); err != nil { + return fmt.Errorf("create idx_tx_last_seen_zero: %w", err) + } + // #1740 step (b): gated DROP of the legacy full index. Only runs AFTER + // step (a) succeeded (sequential — we're past the CREATE above). The + // DROP is a metadata-only schema rewrite in SQLite; no row scan. + // + // PREFLIGHT: async=false reason="DROP INDEX is metadata-only in SQLite (no row scan, no rewrite); safe inline at any DB size" + if _, err := rw.Exec(`DROP INDEX IF EXISTS idx_tx_last_seen`); err != nil { + return fmt.Errorf("drop legacy idx_tx_last_seen: %w", err) + } + return nil +} diff --git a/internal/dbschema/dbschema_test.go b/internal/dbschema/legacy/dbschema_test.go similarity index 99% rename from internal/dbschema/dbschema_test.go rename to internal/dbschema/legacy/dbschema_test.go index 5145777b5..3723df9ef 100644 --- a/internal/dbschema/dbschema_test.go +++ b/internal/dbschema/legacy/dbschema_test.go @@ -1,4 +1,4 @@ -package dbschema +package legacy import ( "database/sql" diff --git a/internal/dbschema/dedup_index.go b/internal/dbschema/legacy/dedup_index.go similarity index 99% rename from internal/dbschema/dedup_index.go rename to internal/dbschema/legacy/dedup_index.go index fb0f728fc..080e97fbb 100644 --- a/internal/dbschema/dedup_index.go +++ b/internal/dbschema/legacy/dedup_index.go @@ -1,4 +1,4 @@ -package dbschema +package legacy import ( "database/sql" diff --git a/internal/dbschema/dedup_index_test.go b/internal/dbschema/legacy/dedup_index_test.go similarity index 99% rename from internal/dbschema/dedup_index_test.go rename to internal/dbschema/legacy/dedup_index_test.go index 550e328d9..fc4c6d36e 100644 --- a/internal/dbschema/dedup_index_test.go +++ b/internal/dbschema/legacy/dedup_index_test.go @@ -1,4 +1,4 @@ -package dbschema +package legacy import ( "database/sql" diff --git a/internal/dbschema/legacy/ingestor_migration_test.go b/internal/dbschema/legacy/ingestor_migration_test.go new file mode 100644 index 000000000..d277fc934 --- /dev/null +++ b/internal/dbschema/legacy/ingestor_migration_test.go @@ -0,0 +1,67 @@ +package legacy + +import ( + "database/sql" + "path/filepath" + "testing" +) + +func TestNormalizeRejectsDestructiveLegacyCleanup(t *testing.T) { + path := filepath.Join(t.TempDir(), "legacy.db") + + // Create a bare-bones DB with legacy bad data + db, err := sql.Open("sqlite3", path+"?_journal_mode=WAL&_busy_timeout=5000") + if err != nil { + t.Fatal(err) + } + db.Exec(`CREATE TABLE IF NOT EXISTS transmissions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + raw_hex TEXT NOT NULL, + hash TEXT NOT NULL, + first_seen TEXT NOT NULL, + route_type INTEGER, + payload_type INTEGER, + payload_version INTEGER, + decoded_json TEXT, + created_at TEXT DEFAULT (datetime('now')), + channel_hash TEXT DEFAULT NULL + )`) + db.Exec(`CREATE TABLE IF NOT EXISTS observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + transmission_id INTEGER NOT NULL REFERENCES transmissions(id), + observer_idx INTEGER, + direction TEXT, + snr REAL, + rssi REAL, + score INTEGER, + path_json TEXT, + timestamp INTEGER NOT NULL + )`) + db.Exec(`CREATE TABLE IF NOT EXISTS _migrations (name TEXT PRIMARY KEY)`) + db.Exec(`CREATE TABLE IF NOT EXISTS nodes (public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, advert_count INTEGER DEFAULT 0, battery_mv INTEGER, temperature_c REAL)`) + db.Exec(`CREATE TABLE IF NOT EXISTS observers (id TEXT PRIMARY KEY, name TEXT, iata TEXT, last_seen TEXT, first_seen TEXT, packet_count INTEGER DEFAULT 0, model TEXT, firmware TEXT, client_version TEXT, radio TEXT, battery_mv INTEGER, uptime_secs INTEGER, noise_floor REAL, inactive INTEGER DEFAULT 0, last_packet_at TEXT DEFAULT NULL)`) + + // Insert good transmission + db.Exec(`INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (1, 'aabb', 'abc123', '2024-01-01T00:00:00Z')`) + db.Exec(`INSERT INTO observations (transmission_id, observer_idx, timestamp) VALUES (1, 1, 1704067200)`) + + // Insert bad: empty hash + db.Exec(`INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (2, 'ccdd', '', '2024-01-01T00:00:00Z')`) + db.Exec(`INSERT INTO observations (transmission_id, observer_idx, timestamp) VALUES (2, 1, 1704067200)`) + + // Insert bad: empty first_seen + db.Exec(`INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (3, 'eeff', 'def456', '')`) + db.Exec(`INSERT INTO observations (transmission_id, observer_idx, timestamp) VALUES (3, 2, 1704067200)`) + + defer db.Close() + if err := Normalize(db, nil); err == nil { + t.Fatal("lossy legacy cleanup accepted") + } + var count int + if err := db.QueryRow(`SELECT count(*) FROM transmissions`).Scan(&count); err != nil { + t.Fatal(err) + } + if count != 3 { + t.Fatalf("source rows changed on refusal: %d", count) + } +} diff --git a/internal/dbschema/legacy/key_backfill.go b/internal/dbschema/legacy/key_backfill.go new file mode 100644 index 000000000..85eabfb23 --- /dev/null +++ b/internal/dbschema/legacy/key_backfill.go @@ -0,0 +1,52 @@ +package legacy + +import "database/sql" + +// ensureKeyBackfill creates the tables behind the key-set backfill (#2107). +// +// tx_rewrite_feed is the change feed: the ingestor appends one row per +// transmission it rewrote in place (a GRP_TXT it can now decrypt, a transport +// scope it can now name), in the same transaction as the rewrite itself. The +// read-only server tails it with its own cursor, exactly like +// advert_route_evidence, so a rewritten row reaches the in-memory store without +// a restart. AUTOINCREMENT keeps feed IDs monotonic after pruning; the +// created_at index serves the age-based prune, and the tx_id index serves the +// ON DELETE CASCADE, so retention deleting a transmission does not scan the +// whole feed. +// +// key_backfill_state holds at most one in-flight run per kind. run_fp is the +// fingerprint of the key set the run is working with: a run whose fingerprint +// no longer matches the live key set is restarted, which is what makes a key +// added mid-run safe. cursor advances in the same transaction as the rewrites +// it covers, so a cancelled run resumes exactly where it committed. +// +// key_backfill_done records which keys have had a complete pass over history, +// so a restart with an unchanged key set does no work at all. +// +// PREFLIGHT: async=true reason="creates three empty tables and two indexes on the empty feed; the backfill itself runs batched in the background after live ingest is ready" +func ensureKeyBackfill(rw *sql.DB) error { + _, err := rw.Exec(` + CREATE TABLE IF NOT EXISTS tx_rewrite_feed ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + tx_id INTEGER NOT NULL REFERENCES transmissions(id) ON DELETE CASCADE, + created_at INTEGER NOT NULL DEFAULT (CAST(strftime('%s','now') AS INTEGER)) + ); + CREATE INDEX IF NOT EXISTS idx_tx_rewrite_feed_created ON tx_rewrite_feed(created_at); + CREATE INDEX IF NOT EXISTS idx_tx_rewrite_feed_tx ON tx_rewrite_feed(tx_id); + CREATE TABLE IF NOT EXISTS key_backfill_state ( + kind TEXT PRIMARY KEY CHECK(kind IN ('channel','region')), + run_fp TEXT NOT NULL, + cursor INTEGER NOT NULL DEFAULT 0, + upper INTEGER NOT NULL DEFAULT 0, + rewritten INTEGER NOT NULL DEFAULT 0, + started_at INTEGER NOT NULL DEFAULT (CAST(strftime('%s','now') AS INTEGER)) + ); + CREATE TABLE IF NOT EXISTS key_backfill_done ( + kind TEXT NOT NULL, + key_fp TEXT NOT NULL, + done_at INTEGER NOT NULL DEFAULT (CAST(strftime('%s','now') AS INTEGER)), + PRIMARY KEY(kind, key_fp) + ) WITHOUT ROWID; + `) + return err +} diff --git a/cmd/ingestor/public_key_lowercase_migration_test.go b/internal/dbschema/legacy/public_key_test.go similarity index 84% rename from cmd/ingestor/public_key_lowercase_migration_test.go rename to internal/dbschema/legacy/public_key_test.go index 160ab4aae..8c6e66e32 100644 --- a/cmd/ingestor/public_key_lowercase_migration_test.go +++ b/internal/dbschema/legacy/public_key_test.go @@ -1,7 +1,8 @@ -package main +package legacy import ( "database/sql" + "path/filepath" "strings" "testing" ) @@ -11,8 +12,12 @@ import ( // The ingestor must normalize any legacy uppercase rows on boot so // the lookup remains correct. func TestPublicKeyLowercaseNormalizationMigration(t *testing.T) { - dbPath := tempDBPath(t) - s, err := OpenStore(dbPath) + dbPath := filepath.Join(t.TempDir(), "legacy.db") + db, err := sql.Open("sqlite3", dbPath) + s := struct{ db *sql.DB }{db} + if err == nil { + err = ApplyBase(db) + } if err != nil { t.Fatalf("first OpenStore: %v", err) } @@ -31,14 +36,18 @@ func TestPublicKeyLowercaseNormalizationMigration(t *testing.T) { if pk != "AABBCCDDEEFF11223344" { t.Fatalf("pre-check: expected uppercase, got %s", pk) } - s.Close() + s.db.Close() // Reopen — the boot-time migration should normalize the row. - s2, err := OpenStore(dbPath) + db2, err := sql.Open("sqlite3", dbPath) + s2 := struct{ db *sql.DB }{db2} + if err == nil { + err = Normalize(db2, nil) + } if err != nil { t.Fatalf("reopen: %v", err) } - defer s2.Close() + defer s2.db.Close() // The uppercase row should be gone. var still int diff --git a/internal/dbschema/writer_dsn.go b/internal/dbschema/legacy/writer_dsn.go similarity index 80% rename from internal/dbschema/writer_dsn.go rename to internal/dbschema/legacy/writer_dsn.go index fb0f7acd2..4a8d53bf4 100644 --- a/internal/dbschema/writer_dsn.go +++ b/internal/dbschema/legacy/writer_dsn.go @@ -1,4 +1,10 @@ -package dbschema +package legacy + +import ( + "net/url" + + "github.com/meshcore-analyzer/dbconfig" +) // WriterDSN builds the DSN every writer must open the database with. // @@ -24,12 +30,13 @@ package dbschema // - _cache_size=-2000 — 2000 KiB of page cache per connection. This is // SQLite's own default, pinned so it cannot drift: the page cache is a C // allocation and therefore sits outside GOMEMLIMIT. -func WriterDSN(path string) string { - return path + - "?_journal_mode=WAL" + - "&_synchronous=FULL" + - "&_auto_vacuum=incremental" + - "&_foreign_keys=on" + - "&_busy_timeout=5000" + - "&_cache_size=-2000" +func WriterDSN(path string) (string, error) { + return dbconfig.SQLiteURI(path, url.Values{ + "_journal_mode": {"WAL"}, + "_synchronous": {"FULL"}, + "_auto_vacuum": {"incremental"}, + "_foreign_keys": {"on"}, + "_busy_timeout": {"5000"}, + "_cache_size": {"-2000"}, + }) } diff --git a/internal/dbschema/legacy/writer_dsn_test.go b/internal/dbschema/legacy/writer_dsn_test.go new file mode 100644 index 000000000..12766fa54 --- /dev/null +++ b/internal/dbschema/legacy/writer_dsn_test.go @@ -0,0 +1,100 @@ +package legacy + +import ( + "database/sql" + "net/url" + "os" + "path/filepath" + "runtime" + "testing" + + _ "github.com/mattn/go-sqlite3" + "github.com/meshcore-analyzer/dbconfig" +) + +// TestWriterDSNPragmas pins what every writer connection actually gets. +// +// cmd/ingestor has its own end-to-end version of this through the store, but +// this one covers the DSN itself, which is what cmd/migrate also opens with. +// The synchronous line matters most: mattn defaults it to NORMAL and runs the +// pragma unconditionally, so if it ever drops out of the DSN the durability +// change is silent. +func TestWriterDSNPragmas(t *testing.T) { + dsn, err := WriterDSN(filepath.Join(t.TempDir(), "w.db")) + if err != nil { + t.Fatal(err) + } + db, err := sql.Open("sqlite3", dsn) + if err != nil { + t.Fatal(err) + } + defer db.Close() + db.SetMaxOpenConns(1) + + for _, want := range []struct{ pragma, value, why string }{ + {"journal_mode", "wal", "cmd/server reads concurrently"}, + {"synchronous", "2", "FULL; mattn defaults to 1 (NORMAL) unless the DSN says otherwise"}, + {"auto_vacuum", "2", "INCREMENTAL; maintenance.go drives incremental_vacuum"}, + {"foreign_keys", "1", "the schema relies on FK enforcement"}, + {"busy_timeout", "5000", "writers serialise behind the reader"}, + {"cache_size", "-2000", "C-allocated page cache, pinned: it sits outside GOMEMLIMIT"}, + } { + var got string + if err := db.QueryRow("PRAGMA " + want.pragma).Scan(&got); err != nil { + t.Errorf("PRAGMA %s: %v", want.pragma, err) + continue + } + if got != want.value { + t.Errorf("PRAGMA %s = %q, want %q (%s)", want.pragma, got, want.value, want.why) + } + } +} + +func TestWriterDSNLiteralPathAndReadOnlyURI(t *testing.T) { + names := []string{"literal #100%20 space µ.db"} + if runtime.GOOS != "windows" { + names = append(names, "literal ? query.db") + } + for _, name := range names { + t.Run(name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), name) + dsn, err := WriterDSN(path) + if err != nil { + t.Fatal(err) + } + db, err := sql.Open("sqlite3", dsn) + if err != nil { + t.Fatal(err) + } + if _, err = db.Exec(`CREATE TABLE kept(value TEXT);INSERT INTO kept VALUES('same file')`); err != nil { + db.Close() + t.Fatal(err) + } + if err = db.Close(); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(path); err != nil { + t.Fatal("literal filename was not created", err) + } + dsn, err = dbconfig.SQLiteURI(path, url.Values{"mode": {"ro"}}) + if err != nil { + t.Fatal(err) + } + ro, err := sql.Open("sqlite3", dsn) + if err != nil { + t.Fatal(err) + } + defer ro.Close() + var value string + if err = ro.QueryRow(`SELECT value FROM kept`).Scan(&value); err != nil || value != "same file" { + t.Fatalf("read-only URI opened another file: %q %v", value, err) + } + if _, err = ro.Exec(`INSERT INTO kept VALUES('forbidden')`); err == nil { + t.Fatal("read-only URI allowed writes") + } + }) + } + if _, err := WriterDSN(""); err == nil { + t.Fatal("empty path accepted") + } +} diff --git a/internal/dbschema/manifest.go b/internal/dbschema/manifest.go new file mode 100644 index 000000000..0bd19b851 --- /dev/null +++ b/internal/dbschema/manifest.go @@ -0,0 +1,39 @@ +package dbschema + +// Column describes the lossless import representation. Identity columns are reseeded after import. +type Column struct { + Name string + Type string + Identity bool +} +type Table struct { + Name string + Columns []Column +} + +// Tables is the complete telemetry import manifest, in dependency order. +var Tables = []Table{ + {Name: "nodes", Columns: []Column{{Name: "public_key", Type: "TEXT", Identity: false}, {Name: "name", Type: "TEXT", Identity: false}, {Name: "role", Type: "TEXT", Identity: false}, {Name: "lat", Type: "DOUBLE PRECISION", Identity: false}, {Name: "lon", Type: "DOUBLE PRECISION", Identity: false}, {Name: "last_seen", Type: "TEXT", Identity: false}, {Name: "first_seen", Type: "TEXT", Identity: false}, {Name: "advert_count", Type: "BIGINT", Identity: false}, {Name: "battery_mv", Type: "BIGINT", Identity: false}, {Name: "temperature_c", Type: "DOUBLE PRECISION", Identity: false}, {Name: "foreign_advert", Type: "BIGINT", Identity: false}, {Name: "default_scope", Type: "TEXT", Identity: false}, {Name: "configured_scope", Type: "TEXT", Identity: false}, {Name: "configured_scope_at", Type: "TEXT", Identity: false}, {Name: "multibyte_sup", Type: "BIGINT", Identity: false}, {Name: "multibyte_evidence", Type: "TEXT", Identity: false}}}, + {Name: "inactive_nodes", Columns: []Column{{Name: "public_key", Type: "TEXT", Identity: false}, {Name: "name", Type: "TEXT", Identity: false}, {Name: "role", Type: "TEXT", Identity: false}, {Name: "lat", Type: "DOUBLE PRECISION", Identity: false}, {Name: "lon", Type: "DOUBLE PRECISION", Identity: false}, {Name: "last_seen", Type: "TEXT", Identity: false}, {Name: "first_seen", Type: "TEXT", Identity: false}, {Name: "advert_count", Type: "BIGINT", Identity: false}, {Name: "battery_mv", Type: "BIGINT", Identity: false}, {Name: "temperature_c", Type: "DOUBLE PRECISION", Identity: false}, {Name: "foreign_advert", Type: "BIGINT", Identity: false}, {Name: "default_scope", Type: "TEXT", Identity: false}, {Name: "configured_scope", Type: "TEXT", Identity: false}, {Name: "configured_scope_at", Type: "TEXT", Identity: false}, {Name: "multibyte_sup", Type: "BIGINT", Identity: false}, {Name: "multibyte_evidence", Type: "TEXT", Identity: false}}}, + {Name: "observers", Columns: []Column{{Name: "rowid", Type: "BIGINT", Identity: true}, {Name: "id", Type: "TEXT", Identity: false}, {Name: "name", Type: "TEXT", Identity: false}, {Name: "iata", Type: "TEXT", Identity: false}, {Name: "last_seen", Type: "TEXT", Identity: false}, {Name: "first_seen", Type: "TEXT", Identity: false}, {Name: "packet_count", Type: "BIGINT", Identity: false}, {Name: "model", Type: "TEXT", Identity: false}, {Name: "firmware", Type: "TEXT", Identity: false}, {Name: "client_version", Type: "TEXT", Identity: false}, {Name: "radio", Type: "TEXT", Identity: false}, {Name: "battery_mv", Type: "BIGINT", Identity: false}, {Name: "uptime_secs", Type: "BIGINT", Identity: false}, {Name: "noise_floor", Type: "DOUBLE PRECISION", Identity: false}, {Name: "inactive", Type: "BIGINT", Identity: false}, {Name: "last_packet_at", Type: "TEXT", Identity: false}, {Name: "clock_skew_seconds", Type: "BIGINT", Identity: false}, {Name: "clock_skew_count_24h", Type: "BIGINT", Identity: false}, {Name: "clock_last_naive_at", Type: "TEXT", Identity: false}, {Name: "can_relay", Type: "BIGINT", Identity: false}, {Name: "can_relay_seen", Type: "BIGINT", Identity: false}}}, + {Name: "transmissions", Columns: []Column{{Name: "id", Type: "BIGINT", Identity: true}, {Name: "raw_hex", Type: "TEXT", Identity: false}, {Name: "hash", Type: "TEXT", Identity: false}, {Name: "first_seen", Type: "TEXT", Identity: false}, {Name: "route_type", Type: "BIGINT", Identity: false}, {Name: "payload_type", Type: "BIGINT", Identity: false}, {Name: "payload_version", Type: "BIGINT", Identity: false}, {Name: "decoded_json", Type: "TEXT", Identity: false}, {Name: "from_pubkey", Type: "TEXT", Identity: false}, {Name: "last_seen", Type: "BIGINT", Identity: false}, {Name: "created_at", Type: "TEXT", Identity: false}, {Name: "code1", Type: "TEXT", Identity: false}, {Name: "code2", Type: "TEXT", Identity: false}, {Name: "channel_hash", Type: "TEXT", Identity: false}, {Name: "scope_name", Type: "TEXT", Identity: false}}}, + {Name: "observations", Columns: []Column{{Name: "id", Type: "BIGINT", Identity: true}, {Name: "transmission_id", Type: "BIGINT", Identity: false}, {Name: "observer_idx", Type: "BIGINT", Identity: false}, {Name: "direction", Type: "TEXT", Identity: false}, {Name: "snr", Type: "DOUBLE PRECISION", Identity: false}, {Name: "rssi", Type: "DOUBLE PRECISION", Identity: false}, {Name: "score", Type: "DOUBLE PRECISION", Identity: false}, {Name: "path_json", Type: "TEXT", Identity: false}, {Name: "timestamp", Type: "BIGINT", Identity: false}, {Name: "raw_hex", Type: "TEXT", Identity: false}, {Name: "resolved_path", Type: "TEXT", Identity: false}}}, + {Name: "observer_metrics", Columns: []Column{{Name: "observer_id", Type: "TEXT", Identity: false}, {Name: "timestamp", Type: "TEXT", Identity: false}, {Name: "noise_floor", Type: "DOUBLE PRECISION", Identity: false}, {Name: "tx_air_secs", Type: "BIGINT", Identity: false}, {Name: "rx_air_secs", Type: "BIGINT", Identity: false}, {Name: "recv_errors", Type: "BIGINT", Identity: false}, {Name: "battery_mv", Type: "BIGINT", Identity: false}, {Name: "packets_sent", Type: "BIGINT", Identity: false}, {Name: "packets_recv", Type: "BIGINT", Identity: false}}}, + {Name: "neighbor_edges", Columns: []Column{{Name: "node_a", Type: "TEXT", Identity: false}, {Name: "node_b", Type: "TEXT", Identity: false}, {Name: "count", Type: "BIGINT", Identity: false}, {Name: "last_seen", Type: "TEXT", Identity: false}}}, + {Name: "dropped_packets", Columns: []Column{{Name: "id", Type: "BIGINT", Identity: true}, {Name: "hash", Type: "TEXT", Identity: false}, {Name: "raw_hex", Type: "TEXT", Identity: false}, {Name: "reason", Type: "TEXT", Identity: false}, {Name: "observer_id", Type: "TEXT", Identity: false}, {Name: "observer_name", Type: "TEXT", Identity: false}, {Name: "node_pubkey", Type: "TEXT", Identity: false}, {Name: "node_name", Type: "TEXT", Identity: false}, {Name: "dropped_at", Type: "TEXT", Identity: false}}}, + {Name: "client_receptions", Columns: []Column{{Name: "id", Type: "BIGINT", Identity: true}, {Name: "rx_pubkey", Type: "TEXT", Identity: false}, {Name: "heard_key", Type: "TEXT", Identity: false}, {Name: "heard_keylen", Type: "BIGINT", Identity: false}, {Name: "rssi", Type: "BIGINT", Identity: false}, {Name: "snr", Type: "DOUBLE PRECISION", Identity: false}, {Name: "lat", Type: "DOUBLE PRECISION", Identity: false}, {Name: "lon", Type: "DOUBLE PRECISION", Identity: false}, {Name: "pos_acc_m", Type: "DOUBLE PRECISION", Identity: false}, {Name: "rx_at", Type: "TEXT", Identity: false}, {Name: "ingested_at", Type: "TEXT", Identity: false}, {Name: "src", Type: "TEXT", Identity: false}}}, + {Name: "client_observers", Columns: []Column{{Name: "pubkey", Type: "TEXT", Identity: false}, {Name: "name", Type: "TEXT", Identity: false}, {Name: "last_seen", Type: "TEXT", Identity: false}}}, + {Name: "client_rx_observations", Columns: []Column{{Name: "id", Type: "BIGINT", Identity: true}, {Name: "rx_pubkey", Type: "TEXT", Identity: false}, {Name: "rx_at", Type: "TEXT", Identity: false}, {Name: "ingested_at", Type: "TEXT", Identity: false}, {Name: "pkt_hash", Type: "TEXT", Identity: false}, {Name: "route_type", Type: "BIGINT", Identity: false}, {Name: "payload_type", Type: "BIGINT", Identity: false}, {Name: "code1", Type: "TEXT", Identity: false}, {Name: "code2", Type: "TEXT", Identity: false}, {Name: "scope_name", Type: "TEXT", Identity: false}, {Name: "hash_size", Type: "BIGINT", Identity: false}, {Name: "hop_count", Type: "BIGINT", Identity: false}, {Name: "path_json", Type: "TEXT", Identity: false}, {Name: "forwarder", Type: "TEXT", Identity: false}, {Name: "snr", Type: "DOUBLE PRECISION", Identity: false}, {Name: "rssi", Type: "BIGINT", Identity: false}, {Name: "lat", Type: "DOUBLE PRECISION", Identity: false}, {Name: "lon", Type: "DOUBLE PRECISION", Identity: false}, {Name: "pos_acc_m", Type: "DOUBLE PRECISION", Identity: false}}}, + {Name: "client_rf_samples", Columns: []Column{{Name: "id", Type: "BIGINT", Identity: true}, {Name: "rx_pubkey", Type: "TEXT", Identity: false}, {Name: "sampled_at", Type: "TEXT", Identity: false}, {Name: "ingested_at", Type: "TEXT", Identity: false}, {Name: "lat", Type: "DOUBLE PRECISION", Identity: false}, {Name: "lon", Type: "DOUBLE PRECISION", Identity: false}, {Name: "pos_acc_m", Type: "DOUBLE PRECISION", Identity: false}, {Name: "stationary", Type: "BIGINT", Identity: false}, {Name: "uptime_secs", Type: "BIGINT", Identity: false}, {Name: "battery_mv", Type: "BIGINT", Identity: false}, {Name: "queue_len", Type: "BIGINT", Identity: false}, {Name: "errors", Type: "BIGINT", Identity: false}, {Name: "noise_floor", Type: "BIGINT", Identity: false}, {Name: "last_rssi", Type: "BIGINT", Identity: false}, {Name: "last_snr", Type: "DOUBLE PRECISION", Identity: false}, {Name: "tx_air_secs", Type: "BIGINT", Identity: false}, {Name: "rx_air_secs", Type: "BIGINT", Identity: false}, {Name: "recv", Type: "BIGINT", Identity: false}, {Name: "sent", Type: "BIGINT", Identity: false}, {Name: "flood_rx", Type: "BIGINT", Identity: false}, {Name: "direct_rx", Type: "BIGINT", Identity: false}, {Name: "flood_tx", Type: "BIGINT", Identity: false}, {Name: "direct_tx", Type: "BIGINT", Identity: false}, {Name: "recv_errors", Type: "BIGINT", Identity: false}}}, + {Name: "node_declared_regions", Columns: []Column{{Name: "id", Type: "BIGINT", Identity: true}, {Name: "target", Type: "TEXT", Identity: false}, {Name: "rx_pubkey", Type: "TEXT", Identity: false}, {Name: "observed_at", Type: "TEXT", Identity: false}, {Name: "ingested_at", Type: "TEXT", Identity: false}, {Name: "regions_csv", Type: "TEXT", Identity: false}, {Name: "truncated", Type: "BIGINT", Identity: false}, {Name: "lat", Type: "DOUBLE PRECISION", Identity: false}, {Name: "lon", Type: "DOUBLE PRECISION", Identity: false}, {Name: "pos_acc_m", Type: "DOUBLE PRECISION", Identity: false}, {Name: "repeater_clock", Type: "BIGINT", Identity: false}}}, + {Name: "scope_match_totals", Columns: []Column{{Name: "id", Type: "BIGINT", Identity: false}, {Name: "since_unix", Type: "BIGINT", Identity: false}, {Name: "unique_matches", Type: "BIGINT", Identity: false}, {Name: "explicit_over_derived", Type: "BIGINT", Identity: false}, {Name: "ambiguous", Type: "BIGINT", Identity: false}, {Name: "none_matches", Type: "BIGINT", Identity: false}, {Name: "updated_unix", Type: "BIGINT", Identity: false}}}, + {Name: "_migrations", Columns: []Column{{Name: "name", Type: "TEXT", Identity: false}}}, + {Name: "_async_migrations", Columns: []Column{{Name: "name", Type: "TEXT", Identity: false}, {Name: "status", Type: "TEXT", Identity: false}, {Name: "started_at", Type: "TEXT", Identity: false}, {Name: "ended_at", Type: "TEXT", Identity: false}, {Name: "error", Type: "TEXT", Identity: false}}}, + {Name: "advert_route_evidence", Columns: []Column{{Name: "id", Type: "BIGINT", Identity: true}, {Name: "tx_id", Type: "BIGINT", Identity: false}, {Name: "bit", Type: "BIGINT", Identity: false}}}, + {Name: "advert_evidence_backfill", Columns: []Column{{Name: "id", Type: "BIGINT", Identity: false}, {Name: "tx_cursor", Type: "BIGINT", Identity: false}, {Name: "obs_cursor", Type: "BIGINT", Identity: false}}}, + {Name: "tx_rewrite_feed", Columns: []Column{{Name: "id", Type: "BIGINT", Identity: true}, {Name: "tx_id", Type: "BIGINT"}, {Name: "created_at", Type: "BIGINT"}}}, + {Name: "key_backfill_state", Columns: []Column{{Name: "kind", Type: "TEXT"}, {Name: "run_fp", Type: "TEXT"}, {Name: "cursor", Type: "BIGINT"}, {Name: "upper", Type: "BIGINT"}, {Name: "rewritten", Type: "BIGINT"}, {Name: "started_at", Type: "BIGINT"}}}, + {Name: "key_backfill_done", Columns: []Column{{Name: "kind", Type: "TEXT"}, {Name: "key_fp", Type: "TEXT"}, {Name: "done_at", Type: "BIGINT"}}}, +} + +var Indexes = []string{"idx_tx_rewrite_feed_created", "idx_tx_rewrite_feed_tx", "idx_observations_observer_idx_timestamp", "idx_nodes_last_seen", "idx_observers_last_seen", "idx_inactive_nodes_last_seen", "idx_transmissions_hash", "idx_transmissions_first_seen", "idx_transmissions_payload_type", "idx_client_recept_heard_geo", "idx_client_recept_latlon", "idx_client_recept_rxat", "idx_cro_prune", "idx_cro_hash", "idx_cro_forwarder", "idx_cro_scope", "idx_crf_prune", "idx_crf_track", "idx_ndr_target", "idx_ndr_prune", "idx_observations_transmission_id", "idx_observations_observer_idx", "idx_observations_timestamp", "idx_observations_dedup", "idx_observer_metrics_timestamp", "idx_tx_channel_hash", "idx_dropped_observer", "idx_dropped_node", "idx_nodes_foreign_advert", "idx_transmissions_from_pubkey", "idx_tx_code1", "idx_observations_tx_ts", "idx_transmissions_channel_hash_payload", "idx_observers_iata_norm", "idx_tx_scope_name", "idx_tx_last_seen_zero"} diff --git a/internal/dbschema/observation_hops_bench_test.go b/internal/dbschema/observation_hops_bench_test.go index 7855b2dde..16e91ac60 100644 --- a/internal/dbschema/observation_hops_bench_test.go +++ b/internal/dbschema/observation_hops_bench_test.go @@ -4,6 +4,8 @@ import ( "database/sql" "path/filepath" "testing" + + "github.com/meshcore-analyzer/dbschema/legacy" ) // BenchmarkObservationInsert measures what the observation_hops triggers add @@ -20,7 +22,11 @@ func BenchmarkObservationInsert(b *testing.B) { name = "with_index" } b.Run(name, func(b *testing.B) { - db, err := sql.Open("sqlite3", WriterDSN(filepath.Join(b.TempDir(), "w.db"))) + dsn, err := legacy.WriterDSN(filepath.Join(b.TempDir(), "w.db")) + if err != nil { + b.Fatal(err) + } + db, err := sql.Open("sqlite3", dsn) if err != nil { b.Fatal(err) } diff --git a/internal/dbschema/observation_hops_test.go b/internal/dbschema/observation_hops_test.go index eac384865..a7aeffd11 100644 --- a/internal/dbschema/observation_hops_test.go +++ b/internal/dbschema/observation_hops_test.go @@ -4,6 +4,7 @@ import ( "database/sql" "reflect" "sort" + "strings" "testing" _ "github.com/mattn/go-sqlite3" @@ -112,7 +113,7 @@ func TestObservationHops_QueryUsesPrimaryKey(t *testing.T) { plan = append(plan, d) } sort.Strings(plan) - if len(plan) == 0 || !contains(plan[0], "PRIMARY KEY") || contains(plan[0], "SCAN") { + if len(plan) == 0 || !strings.Contains(plan[0], "PRIMARY KEY") || strings.Contains(plan[0], "SCAN") { t.Fatalf("expected PK SEARCH, got %v", plan) } } @@ -166,9 +167,8 @@ func hopObjects(t *testing.T, db *sql.DB) []string { // The index is opt-in: the shared schema (Apply, used by the ingestor and the // migrate tool) must not install any of it. func TestApplyDoesNotInstallObservationHops(t *testing.T) { - db := minimalDB(t) - defer db.Close() - if err := Apply(db, nil); err != nil { + db, _ := sqliteTestDB(t) + if err := ApplySQLite(db, nil); err != nil { t.Fatal(err) } if got := hopObjects(t, db); len(got) != 0 { diff --git a/internal/dbschema/parameter_conformance_test.go b/internal/dbschema/parameter_conformance_test.go new file mode 100644 index 000000000..4848deda4 --- /dev/null +++ b/internal/dbschema/parameter_conformance_test.go @@ -0,0 +1,62 @@ +package dbschema + +import ( + "database/sql" + "fmt" + "testing" + + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/pgutil" +) + +func testNumberedParameters(t *testing.T, db *sql.DB, backend dbconfig.Backend) { + t.Helper() + query := fmt.Sprintf("SELECT CAST(%s AS BIGINT),CAST(%s AS BIGINT),CAST(%s AS BIGINT)", backend.Parameter(2), backend.Parameter(1), backend.Parameter(2)) + for _, second := range []any{int64(22), nil} { + var a, b, c sql.NullInt64 + if err := db.QueryRow(query, int64(11), second).Scan(&a, &b, &c); err != nil { + t.Fatal(err) + } + if !b.Valid || b.Int64 != 11 || a != c { + t.Fatalf("reordered/repeated parameters: %v,%v,%v", a, b, c) + } + if second == nil { + if a.Valid { + t.Fatal("NULL changed") + } + } else if !a.Valid || a.Int64 != 22 { + t.Fatal("second argument bound by appearance, not position") + } + } +} + +func TestSQLiteNumberedParameterConformance(t *testing.T) { + db, _ := sqliteTestDB(t) + testNumberedParameters(t, db, dbconfig.SQLite) +} + +func TestPostgresNumberedParameterConformance(t *testing.T) { + db, err := pgutil.Open(postgresSchema(t), false) + if err != nil { + t.Fatal(err) + } + defer db.Close() + testNumberedParameters(t, db, dbconfig.Postgres) +} + +func TestPostgresNamedEntrypointsKeepImportGate(t *testing.T) { + db, err := pgutil.Open(postgresSchema(t), false) + if err != nil { + t.Fatal(err) + } + defer db.Close() + if err := ApplyForImport(db, nil); err != nil { + t.Fatal(err) + } + if err := AssertPostgresReady(db); err == nil { + t.Fatal("named reader entrypoint bypassed incomplete import") + } + if err := ApplyPostgres(db, nil); err == nil { + t.Fatal("named bootstrap entrypoint bypassed incomplete import") + } +} diff --git a/internal/dbschema/postgres_integration_test.go b/internal/dbschema/postgres_integration_test.go new file mode 100644 index 000000000..76416512d --- /dev/null +++ b/internal/dbschema/postgres_integration_test.go @@ -0,0 +1,179 @@ +package dbschema + +import ( + "database/sql" + "github.com/meshcore-analyzer/pgutil" + "github.com/meshcore-analyzer/pgutil/pgtest" + "sync" + "testing" +) + +func TestPostgresImportInitializationNeverReady(t *testing.T) { + db, err := pgutil.Open(postgresSchema(t), false) + if err != nil { + t.Fatal(err) + } + defer db.Close() + if err := ApplyForImport(db, nil); err != nil { + t.Fatal(err) + } + if err := AssertReady(db); err == nil { + t.Fatal("empty import target accepted before verification") + } + if err := Apply(db, nil); err == nil { + t.Fatal("normal initialization bypassed incomplete import") + } +} + +func TestPostgresConcurrentInitialization(t *testing.T) { + db, err := pgutil.Open(postgresSchema(t), false) + if err != nil { + t.Fatal(err) + } + defer db.Close() + var wg sync.WaitGroup + for i := 0; i < 3; i++ { + wg.Add(1) + go func() { + defer wg.Done() + if err := Apply(db, nil); err != nil { + t.Error(err) + } + }() + } + wg.Wait() + if err := AssertReady(db); err != nil { + t.Fatal(err) + } +} + +func TestPostgresIdentityRawValuesAndNullDedup(t *testing.T) { + db, err := pgutil.Open(postgresSchema(t), false) + if err != nil { + t.Fatal(err) + } + defer db.Close() + if err := Apply(db, nil); err != nil { + t.Fatal(err) + } + for _, q := range []string{ + `INSERT INTO observers(rowid,id) VALUES(0,'zero'),(77,'stable')`, + `INSERT INTO transmissions(id,raw_hex,hash,first_seen,decoded_json) VALUES(0,'aa','zero','2026-01-01T01:02:03.123456Z','{malformed')`, + `INSERT INTO observations(id,transmission_id,observer_idx,path_json,timestamp,score) VALUES(0,0,77,NULL,1,3.5),(50,0,NULL,NULL,2,NULL),(60,0,NULL,'',3,NULL)`, + } { + if _, err := db.Exec(q); err != nil { + t.Fatal(err) + } + } + if _, err := db.Exec(`INSERT INTO observations(transmission_id,observer_idx,path_json,timestamp) VALUES(0,77,'',4)`); err == nil { + t.Fatal("null/empty path identity diverged") + } + var score float64 + var raw, stamp string + if err := db.QueryRow(`SELECT o.score,t.decoded_json,t.first_seen FROM observations o JOIN transmissions t ON t.id=o.transmission_id WHERE o.id=0`).Scan(&score, &raw, &stamp); err != nil { + t.Fatal(err) + } + if score != 3.5 || raw != "{malformed" || stamp != "2026-01-01T01:02:03.123456Z" { + t.Fatal("raw values changed") + } + tx, err := db.Begin() + if err != nil { + t.Fatal(err) + } + defer tx.Rollback() + if err := ReseedIdentities(tx, map[string]int64{"observations.id": 500}); err != nil { + t.Fatal(err) + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + var id int64 + if err := db.QueryRow(`INSERT INTO observations(transmission_id,timestamp) VALUES(0,5) RETURNING id`).Scan(&id); err != nil { + t.Fatal(err) + } + if id != 501 { + t.Fatalf("deleted identity high-water lost: %d", id) + } +} + +func TestPostgresWriterRejectsOwnerAndReader(t *testing.T) { + dsn := postgresSchema(t) + owner, err := pgutil.Open(dsn, false) + if err != nil { + t.Fatal(err) + } + defer owner.Close() + if err := Apply(owner, nil); err != nil { + t.Fatal(err) + } + if err := AssertWriter(owner); err == nil { + t.Fatal("owner accepted as runtime writer") + } + reader, err := pgutil.Open(pgtest.ReadOnly(t, dsn), true) + if err != nil { + t.Fatal(err) + } + defer reader.Close() + if err := AssertReady(reader); err != nil { + t.Fatal(err) + } + if err := AssertWriter(reader); err == nil { + t.Fatal("SELECT-only role accepted as writer") + } + for _, q := range []string{`INSERT INTO nodes(public_key) VALUES('x')`, `UPDATE nodes SET name='x'`, `DELETE FROM nodes`, `CREATE TABLE forbidden(id int)`} { + if _, err := reader.Exec(q); err == nil { + t.Fatalf("reader permitted %s", q) + } + } +} + +func TestPostgresInitializationRejectsUnversionedData(t *testing.T) { + db, err := pgutil.Open(postgresSchema(t), false) + if err != nil { + t.Fatal(err) + } + defer db.Close() + if _, err := db.Exec(`CREATE TABLE existing(id int)`); err != nil { + t.Fatal(err) + } + if err := Apply(db, nil); err == nil { + t.Fatal("unversioned nonempty schema accepted") + } + var marker sql.NullString + if err := db.QueryRow(`SELECT to_regclass('corescope_schema')::text`).Scan(&marker); err != nil { + t.Fatal(err) + } + if marker.Valid { + t.Fatal("failed initialization left readiness metadata behind") + } +} + +func TestPostgresSchemaFreshRepeatedAndReadiness(t *testing.T) { + dsn := postgresSchema(t) + db, err := pgutil.Open(dsn, false) + if err != nil { + t.Fatal(err) + } + defer db.Close() + apply := Apply + for i := 0; i < 2; i++ { + if err := apply(db, nil); err != nil { + t.Fatalf("initialize PostgreSQL schema: %v", err) + } + } + if err := AssertReady(db); err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`UPDATE corescope_schema SET ready=false`); err != nil { + t.Fatal(err) + } + if err := AssertReady(db); err == nil { + t.Fatal("incomplete import accepted") + } + if _, err := db.Exec(`UPDATE corescope_schema SET ready=true,version=999`); err != nil { + t.Fatal(err) + } + if err := AssertReady(db); err == nil { + t.Fatal("newer schema accepted") + } +} diff --git a/internal/dbschema/postgres_test.go b/internal/dbschema/postgres_test.go new file mode 100644 index 000000000..8d1097982 --- /dev/null +++ b/internal/dbschema/postgres_test.go @@ -0,0 +1,29 @@ +package dbschema + +import ( + "strings" + "testing" +) + +func TestPostgresManifestPreservesTelemetry(t *testing.T) { + if len(Tables) != 21 { + t.Fatalf("telemetry manifest has %d tables, want 21", len(Tables)) + } + for _, name := range []string{"nodes", "inactive_nodes", "observers", "transmissions", "observations", "observer_metrics", "neighbor_edges", "dropped_packets", "client_receptions", "client_observers", "client_rx_observations", "client_rf_samples", "node_declared_regions", "scope_match_totals", "_migrations", "_async_migrations", "advert_route_evidence", "advert_evidence_backfill", "tx_rewrite_feed", "key_backfill_state", "key_backfill_done"} { + found := false + for _, table := range Tables { + if table.Name == name { + found = true + } + } + if !found { + t.Errorf("manifest missing %s", name) + } + } + if !strings.Contains(SchemaSQL, "rowid BIGINT GENERATED BY DEFAULT AS IDENTITY") { + t.Error("observer identity must be explicit and importable") + } + if strings.Contains(strings.ToUpper(SchemaSQL), "JSONB") { + t.Error("raw JSON must remain text") + } +} diff --git a/internal/dbschema/schema.sql b/internal/dbschema/schema.sql new file mode 100644 index 000000000..ebeabcdd4 --- /dev/null +++ b/internal/dbschema/schema.sql @@ -0,0 +1,347 @@ +-- Canonical PostgreSQL telemetry schema. Raw JSON and timestamp strings remain TEXT COLLATE "C". +CREATE TABLE nodes ( + public_key TEXT COLLATE "C" PRIMARY KEY, + name TEXT COLLATE "C", + role TEXT COLLATE "C", + lat DOUBLE PRECISION, + lon DOUBLE PRECISION, + last_seen TEXT COLLATE "C", + first_seen TEXT COLLATE "C", + advert_count BIGINT DEFAULT 0, + battery_mv BIGINT, + temperature_c DOUBLE PRECISION, + foreign_advert BIGINT DEFAULT 0, + default_scope TEXT COLLATE "C", + configured_scope TEXT COLLATE "C", + configured_scope_at TEXT COLLATE "C", + multibyte_sup BIGINT NOT NULL DEFAULT 0, + multibyte_evidence TEXT COLLATE "C" +); + +CREATE TABLE inactive_nodes ( + public_key TEXT COLLATE "C" PRIMARY KEY, + name TEXT COLLATE "C", + role TEXT COLLATE "C", + lat DOUBLE PRECISION, + lon DOUBLE PRECISION, + last_seen TEXT COLLATE "C", + first_seen TEXT COLLATE "C", + advert_count BIGINT DEFAULT 0, + battery_mv BIGINT, + temperature_c DOUBLE PRECISION, + foreign_advert BIGINT DEFAULT 0, + default_scope TEXT COLLATE "C", + configured_scope TEXT COLLATE "C", + configured_scope_at TEXT COLLATE "C", + multibyte_sup BIGINT NOT NULL DEFAULT 0, + multibyte_evidence TEXT COLLATE "C" +); + +CREATE TABLE observers ( + rowid BIGINT GENERATED BY DEFAULT AS IDENTITY UNIQUE, + id TEXT COLLATE "C" PRIMARY KEY, + name TEXT COLLATE "C", + iata TEXT COLLATE "C", + last_seen TEXT COLLATE "C", + first_seen TEXT COLLATE "C", + packet_count BIGINT DEFAULT 0, + model TEXT COLLATE "C", + firmware TEXT COLLATE "C", + client_version TEXT COLLATE "C", + radio TEXT COLLATE "C", + battery_mv BIGINT, + uptime_secs BIGINT, + noise_floor DOUBLE PRECISION, + inactive BIGINT DEFAULT 0, + last_packet_at TEXT COLLATE "C" DEFAULT NULL, + clock_skew_seconds BIGINT DEFAULT NULL, + clock_skew_count_24h BIGINT DEFAULT 0, + clock_last_naive_at TEXT COLLATE "C" DEFAULT NULL, + can_relay BIGINT DEFAULT 1, + can_relay_seen BIGINT DEFAULT 0 +); + +CREATE TABLE transmissions ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + raw_hex TEXT COLLATE "C" NOT NULL, + hash TEXT COLLATE "C" NOT NULL UNIQUE, + first_seen TEXT COLLATE "C" NOT NULL, + route_type BIGINT, + payload_type BIGINT, + payload_version BIGINT, + decoded_json TEXT COLLATE "C", + from_pubkey TEXT COLLATE "C", + last_seen BIGINT NOT NULL DEFAULT 0, + created_at TEXT COLLATE "C" DEFAULT (to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD HH24:MI:SS')), + code1 TEXT COLLATE "C", + code2 TEXT COLLATE "C", + channel_hash TEXT COLLATE "C", + scope_name TEXT COLLATE "C" +); + +CREATE TABLE observations ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + transmission_id BIGINT NOT NULL REFERENCES transmissions(id), + observer_idx BIGINT, + direction TEXT COLLATE "C", + snr DOUBLE PRECISION, + rssi DOUBLE PRECISION, + score DOUBLE PRECISION, + path_json TEXT COLLATE "C", + timestamp BIGINT NOT NULL, + raw_hex TEXT COLLATE "C", + resolved_path TEXT COLLATE "C" +); + +CREATE TABLE observer_metrics ( + observer_id TEXT COLLATE "C" NOT NULL, + timestamp TEXT COLLATE "C" NOT NULL, + noise_floor DOUBLE PRECISION, + tx_air_secs BIGINT, + rx_air_secs BIGINT, + recv_errors BIGINT, + battery_mv BIGINT, + packets_sent BIGINT, + packets_recv BIGINT, + PRIMARY KEY (observer_id, timestamp) +); + +CREATE TABLE neighbor_edges ( + node_a TEXT COLLATE "C" NOT NULL, + node_b TEXT COLLATE "C" NOT NULL, + count BIGINT DEFAULT 1, + last_seen TEXT COLLATE "C", + PRIMARY KEY (node_a, node_b) +); + +CREATE TABLE dropped_packets ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + hash TEXT COLLATE "C", + raw_hex TEXT COLLATE "C", + reason TEXT COLLATE "C" NOT NULL, + observer_id TEXT COLLATE "C", + observer_name TEXT COLLATE "C", + node_pubkey TEXT COLLATE "C", + node_name TEXT COLLATE "C", + dropped_at TEXT COLLATE "C" DEFAULT (to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD HH24:MI:SS')) +); + +CREATE TABLE client_receptions ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + rx_pubkey TEXT COLLATE "C" NOT NULL, + heard_key TEXT COLLATE "C" NOT NULL, + heard_keylen BIGINT NOT NULL, + rssi BIGINT, + snr DOUBLE PRECISION, + lat DOUBLE PRECISION NOT NULL, + lon DOUBLE PRECISION NOT NULL, + pos_acc_m DOUBLE PRECISION, + rx_at TEXT COLLATE "C" NOT NULL, + ingested_at TEXT COLLATE "C" NOT NULL, + src TEXT COLLATE "C" NOT NULL, + UNIQUE(rx_pubkey, heard_key, rx_at) +); + +CREATE TABLE client_observers ( + pubkey TEXT COLLATE "C" PRIMARY KEY, + name TEXT COLLATE "C", + last_seen TEXT COLLATE "C" +); + +CREATE TABLE client_rx_observations ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + rx_pubkey TEXT COLLATE "C" NOT NULL, + rx_at TEXT COLLATE "C" NOT NULL, + ingested_at TEXT COLLATE "C" NOT NULL, + pkt_hash TEXT COLLATE "C" NOT NULL, + route_type BIGINT NOT NULL, + payload_type BIGINT NOT NULL, + code1 TEXT COLLATE "C", + code2 TEXT COLLATE "C", + scope_name TEXT COLLATE "C", + hash_size BIGINT NOT NULL, + hop_count BIGINT NOT NULL, + path_json TEXT COLLATE "C", + forwarder TEXT COLLATE "C", + snr DOUBLE PRECISION, + rssi BIGINT, + lat DOUBLE PRECISION NOT NULL, + lon DOUBLE PRECISION NOT NULL, + pos_acc_m DOUBLE PRECISION, + UNIQUE(rx_pubkey, pkt_hash, rx_at) +); + +CREATE TABLE client_rf_samples ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + rx_pubkey TEXT COLLATE "C" NOT NULL, + sampled_at TEXT COLLATE "C" NOT NULL, + ingested_at TEXT COLLATE "C" NOT NULL, + lat DOUBLE PRECISION NOT NULL, + lon DOUBLE PRECISION NOT NULL, + pos_acc_m DOUBLE PRECISION, + stationary BIGINT NOT NULL DEFAULT 0, + uptime_secs BIGINT NOT NULL, + battery_mv BIGINT, + queue_len BIGINT, + errors BIGINT, + noise_floor BIGINT, + last_rssi BIGINT, + last_snr DOUBLE PRECISION, + tx_air_secs BIGINT, + rx_air_secs BIGINT, + recv BIGINT, + sent BIGINT, + flood_rx BIGINT, + direct_rx BIGINT, + flood_tx BIGINT, + direct_tx BIGINT, + recv_errors BIGINT, + UNIQUE(rx_pubkey, sampled_at) +); + +CREATE TABLE node_declared_regions ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + target TEXT COLLATE "C" NOT NULL, + rx_pubkey TEXT COLLATE "C" NOT NULL, + observed_at TEXT COLLATE "C" NOT NULL, + ingested_at TEXT COLLATE "C" NOT NULL, + regions_csv TEXT COLLATE "C" NOT NULL, + truncated BIGINT NOT NULL DEFAULT 0, + lat DOUBLE PRECISION, + lon DOUBLE PRECISION, + pos_acc_m DOUBLE PRECISION, + repeater_clock BIGINT, + UNIQUE(target, rx_pubkey, observed_at) +); + +CREATE TABLE scope_match_totals ( + id BIGINT PRIMARY KEY CHECK (id = 1), + since_unix BIGINT NOT NULL, + unique_matches BIGINT NOT NULL, + explicit_over_derived BIGINT NOT NULL, + ambiguous BIGINT NOT NULL, + none_matches BIGINT NOT NULL, + updated_unix BIGINT NOT NULL +); + +CREATE TABLE _migrations ( + name TEXT COLLATE "C" PRIMARY KEY +); + +CREATE TABLE _async_migrations ( + name TEXT COLLATE "C" PRIMARY KEY, + status TEXT COLLATE "C" NOT NULL, + started_at TEXT COLLATE "C" NOT NULL DEFAULT (to_char(CURRENT_TIMESTAMP AT TIME ZONE 'UTC', 'YYYY-MM-DD HH24:MI:SS')), + ended_at TEXT COLLATE "C", + error TEXT COLLATE "C" +); + +CREATE TABLE advert_route_evidence ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + tx_id BIGINT NOT NULL REFERENCES transmissions(id) ON DELETE CASCADE, + bit BIGINT NOT NULL CHECK(bit IN (1,2)), + UNIQUE(tx_id,bit) +); + +CREATE TABLE advert_evidence_backfill ( + id BIGINT PRIMARY KEY CHECK(id=1), + tx_cursor BIGINT NOT NULL DEFAULT 0, + obs_cursor BIGINT NOT NULL DEFAULT 0 +); + +CREATE TABLE tx_rewrite_feed ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + tx_id BIGINT NOT NULL REFERENCES transmissions(id) ON DELETE CASCADE, + created_at BIGINT NOT NULL DEFAULT FLOOR(EXTRACT(EPOCH FROM CURRENT_TIMESTAMP))::BIGINT +); +CREATE INDEX idx_tx_rewrite_feed_created ON tx_rewrite_feed(created_at); +CREATE INDEX idx_tx_rewrite_feed_tx ON tx_rewrite_feed(tx_id); +CREATE TABLE key_backfill_state ( + kind TEXT COLLATE "C" PRIMARY KEY CHECK(kind IN ('channel','region')), + run_fp TEXT COLLATE "C" NOT NULL, + cursor BIGINT NOT NULL DEFAULT 0, + upper BIGINT NOT NULL DEFAULT 0, + rewritten BIGINT NOT NULL DEFAULT 0, + started_at BIGINT NOT NULL DEFAULT FLOOR(EXTRACT(EPOCH FROM CURRENT_TIMESTAMP))::BIGINT +); +CREATE TABLE key_backfill_done ( + kind TEXT COLLATE "C" NOT NULL, + key_fp TEXT COLLATE "C" NOT NULL, + done_at BIGINT NOT NULL DEFAULT FLOOR(EXTRACT(EPOCH FROM CURRENT_TIMESTAMP))::BIGINT, + PRIMARY KEY(kind,key_fp) +); + +CREATE INDEX IF NOT EXISTS idx_observations_observer_idx_timestamp ON observations(observer_idx,timestamp); + +CREATE INDEX IF NOT EXISTS idx_nodes_last_seen ON nodes(last_seen); + +CREATE INDEX IF NOT EXISTS idx_observers_last_seen ON observers(last_seen); + +CREATE INDEX IF NOT EXISTS idx_inactive_nodes_last_seen ON inactive_nodes(last_seen); + +CREATE INDEX IF NOT EXISTS idx_transmissions_hash ON transmissions(hash); + +CREATE INDEX IF NOT EXISTS idx_transmissions_first_seen ON transmissions(first_seen,id); + +CREATE INDEX IF NOT EXISTS idx_transmissions_payload_type ON transmissions(payload_type); + +CREATE INDEX IF NOT EXISTS idx_client_recept_heard_geo ON client_receptions(heard_key, heard_keylen, lat, lon); + +CREATE INDEX IF NOT EXISTS idx_client_recept_latlon ON client_receptions(lat, lon); + +CREATE INDEX IF NOT EXISTS idx_client_recept_rxat ON client_receptions(rx_at); + +CREATE INDEX IF NOT EXISTS idx_cro_prune ON client_rx_observations(rx_at); + +CREATE INDEX IF NOT EXISTS idx_cro_hash ON client_rx_observations(pkt_hash, rx_at); + +CREATE INDEX IF NOT EXISTS idx_cro_forwarder ON client_rx_observations(forwarder, rx_at); + +CREATE INDEX IF NOT EXISTS idx_cro_scope ON client_rx_observations(scope_name, rx_at); + +CREATE INDEX IF NOT EXISTS idx_crf_prune ON client_rf_samples(sampled_at); + +CREATE INDEX IF NOT EXISTS idx_crf_track ON client_rf_samples(rx_pubkey, sampled_at); + +CREATE INDEX IF NOT EXISTS idx_ndr_target ON node_declared_regions(target, observed_at); + +CREATE INDEX IF NOT EXISTS idx_ndr_prune ON node_declared_regions(observed_at); + +CREATE INDEX IF NOT EXISTS idx_observations_transmission_id ON observations(transmission_id); + +CREATE INDEX IF NOT EXISTS idx_observations_observer_idx ON observations(observer_idx); + +CREATE INDEX IF NOT EXISTS idx_observations_timestamp ON observations(timestamp); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_observations_dedup ON observations(transmission_id,observer_idx,COALESCE(path_json,'')); + +CREATE INDEX IF NOT EXISTS idx_observer_metrics_timestamp ON observer_metrics(timestamp); + +CREATE INDEX IF NOT EXISTS idx_tx_channel_hash ON transmissions(channel_hash) WHERE payload_type = 5; + +CREATE INDEX IF NOT EXISTS idx_dropped_observer ON dropped_packets(observer_id); + +CREATE INDEX IF NOT EXISTS idx_dropped_node ON dropped_packets(node_pubkey); + +CREATE INDEX IF NOT EXISTS idx_nodes_foreign_advert ON nodes(foreign_advert) WHERE foreign_advert = 1; + +CREATE INDEX IF NOT EXISTS idx_transmissions_from_pubkey ON transmissions(from_pubkey); + +CREATE INDEX IF NOT EXISTS idx_tx_code1 ON transmissions(code1) WHERE code1 IS NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_observations_tx_ts ON observations(transmission_id, timestamp); + +CREATE INDEX IF NOT EXISTS idx_transmissions_channel_hash_payload ON transmissions(channel_hash, payload_type, first_seen); + +CREATE INDEX IF NOT EXISTS idx_observers_iata_norm ON observers(UPPER(TRIM(iata))); + +CREATE INDEX IF NOT EXISTS idx_tx_scope_name ON transmissions(scope_name) WHERE scope_name IS NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_tx_last_seen_zero ON transmissions(id) WHERE last_seen=0; + +CREATE VIEW packets_v AS SELECT o.id,COALESCE(o.raw_hex,t.raw_hex) AS raw_hex, +to_char(to_timestamp(o.timestamp) AT TIME ZONE 'UTC','YYYY-MM-DD HH24:MI:SS') AS timestamp, +obs.id AS observer_id,obs.name AS observer_name,o.direction,o.snr,o.rssi,o.score,t.hash, +t.route_type,t.payload_type,t.payload_version,o.path_json,t.decoded_json,t.created_at +FROM observations o JOIN transmissions t ON t.id=o.transmission_id +LEFT JOIN observers obs ON obs.rowid=o.observer_idx AND (obs.inactive IS NULL OR obs.inactive=0); diff --git a/internal/dbschema/sqlite.go b/internal/dbschema/sqlite.go new file mode 100644 index 000000000..13f3c8d39 --- /dev/null +++ b/internal/dbschema/sqlite.go @@ -0,0 +1,421 @@ +package dbschema + +import ( + "context" + "database/sql" + "errors" + "fmt" + "regexp" + "strings" + + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/dbschema/legacy" +) + +// The existing logical migration table records this SQLite physical-layout +// change. No new data table or PostgreSQL manifest column is introduced. +const SQLiteObserverIdentityMigration = "observers_identity_autoincrement_v1" + +// Existing entrypoints remain PostgreSQL-compatible while callers migrate to +// explicit engine selection. PostgreSQL grants/readiness checks are unchanged. +func ApplyPostgres(db *sql.DB, logf Logger) error { return Apply(db, logf) } +func AssertPostgresReady(db *sql.DB) error { return AssertReady(db) } + +// ApplySQLite is a writer-only startup/upgrade entrypoint. Existing v3 files +// are checked before historical migrations can perform cleanup. Older v2 or +// unknown layouts require an explicit supported offline upgrade first. +func ApplySQLite(db *sql.DB, logf Logger) error { + if err := dbconfig.AssertSQLiteImportComplete(db); err != nil { + return err + } + migrated, cols, err := sqliteObserverLayout(db) + if err != nil { + return err + } + if err := sqliteIdentityVersion(db); err != nil { + return err + } + var ledger, applied int + if err := db.QueryRow(`SELECT count(*) FROM sqlite_master WHERE type='table' AND name='_migrations'`).Scan(&ledger); err != nil { + return err + } + if ledger != 0 { + if err := db.QueryRow(`SELECT count(*) FROM _migrations WHERE name=?`, SQLiteObserverIdentityMigration).Scan(&applied); err != nil { + return err + } + } + if applied != 0 && !migrated { + return errors.New("SQLite observer identity marker does not match its physical schema") + } + needsIdentity := applied == 0 + var tables int + if err := db.QueryRow(`SELECT count(*) FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'`).Scan(&tables); err != nil { + return err + } + if tables != 0 && needsIdentity { + if len(cols) == 0 { + return errors.New("existing SQLite file is not a supported telemetry database") + } + if err := legacy.CheckLegacySource(db); err != nil { + return err + } + if err := sqliteForeignKeysValid(db); err != nil { + return err + } + } + if err := legacy.ApplyBase(db); err != nil { + return err + } + if err := legacy.Apply(db, legacy.Logger(logf)); err != nil { + return err + } + if err := EnsureSQLiteAsyncMigrations(db); err != nil { + return err + } + // The identity migration's corpus/FK scans run once. Ordinary restarts + // retain existing schema maintenance without rescanning all observations. + if needsIdentity { + if err := migrateSQLiteObserverIdentity(db); err != nil { + return err + } + } + // Rebuilding observers drops its sqlite_stat1 entries and can turn the + // recent-packet aggregate into a scan/sort of the observation window. + // Refresh only observers, also repairing previously rebuilt databases. + // Do not create sqlite_stat1 on a fresh database: its absence tells the + // ingestor that the initial whole-database analysis still needs to run. + var analyzed int + if err := db.QueryRow(`SELECT count(*) FROM sqlite_master WHERE type='table' AND name='sqlite_stat1'`).Scan(&analyzed); err != nil { + return err + } + if analyzed != 0 { + if _, err := db.Exec(`ANALYZE observers`); err != nil { + return fmt.Errorf("refresh SQLite observer planner statistics: %w", err) + } + } + return AssertSQLiteReady(db) +} + +// EnsureSQLiteAsyncMigrations retains the upstream ingestor's exact ledger +// layout. Setup and reverse conversion need the complete schema before the +// ingestor starts; that process also calls this same idempotent entrypoint. +func EnsureSQLiteAsyncMigrations(db *sql.DB) error { + _, err := db.Exec(`CREATE TABLE IF NOT EXISTS _async_migrations ( + name TEXT PRIMARY KEY, + status TEXT NOT NULL, + started_at TEXT NOT NULL DEFAULT (datetime('now')), + ended_at TEXT, + error TEXT + )`) + return err +} + +// AssertSQLiteReady only reads schema metadata. It is safe on a mode=ro handle; +// it never applies migrations or marks a partially initialized target ready. +func AssertSQLiteReady(db *sql.DB) error { + if err := dbconfig.AssertSQLiteImportComplete(db); err != nil { + return err + } + if err := legacy.AssertReady(db); err != nil { + return err + } + if err := sqliteIdentityVersion(db); err != nil { + return err + } + migrated, _, err := sqliteObserverLayout(db) + if err != nil { + return err + } + var count int + if err := db.QueryRow(`SELECT count(*) FROM _migrations WHERE name=?`, SQLiteObserverIdentityMigration).Scan(&count); err != nil { + return err + } + if !migrated || count != 1 { + return errors.New("SQLite observer identity schema is not migrated; run the writer upgrade") + } + return nil +} + +type sqliteColumn struct { + name, kind string + notNull, primaryKey, hidden int + defaultSQL sql.NullString +} + +var unsupportedObserverConstraint = regexp.MustCompile(`(?i)\b(CHECK|COLLATE|REFERENCES|CONSTRAINT|GENERATED|WITHOUT|STRICT)\b|\bON\s+CONFLICT\b`) + +func sqliteQuote(value string) string { return `"` + strings.ReplaceAll(value, `"`, `""`) + `"` } + +// Rebuilding unknown constraints can silently change semantics. Accept the +// ordinary supported layout; refuse extensions we cannot reproduce before any +// schema writes. Explicit indexes and triggers are preserved by the rebuild. +func sqliteObserverLayout(db Querier) (bool, []sqliteColumn, error) { + var ddl string + err := db.QueryRow(`SELECT sql FROM sqlite_master WHERE type='table' AND name='observers'`).Scan(&ddl) + if errors.Is(err, sql.ErrNoRows) { + return false, nil, nil + } + if err != nil { + return false, nil, err + } + if unsupportedObserverConstraint.MatchString(ddl) { + return false, nil, errors.New("unsupported SQLite observer constraints; source was not changed") + } + rows, err := db.Query(`PRAGMA table_xinfo(observers)`) + if err != nil { + return false, nil, err + } + var cols []sqliteColumn + for rows.Next() { + var cid int + var c sqliteColumn + if err = rows.Scan(&cid, &c.name, &c.kind, &c.notNull, &c.defaultSQL, &c.primaryKey, &c.hidden); err != nil { + rows.Close() + return false, nil, err + } + known := false + for _, table := range Tables { + if table.Name == "observers" { + for _, col := range table.Columns { + if col.Name == c.name { + known = true + } + } + } + } + c.kind = strings.ToUpper(c.kind) + if !known || c.hidden != 0 || (c.kind != "TEXT" && c.kind != "INTEGER" && c.kind != "REAL") { + rows.Close() + return false, nil, errors.New("unsupported SQLite observer column layout; source was not changed") + } + cols = append(cols, c) + } + if err = rows.Err(); err != nil { + rows.Close() + return false, nil, err + } + rows.Close() + var id, rowid *sqliteColumn + for i := range cols { + if cols[i].name == "id" { + id = &cols[i] + } + if cols[i].name == "rowid" { + rowid = &cols[i] + } + } + if id == nil || id.kind != "TEXT" { + return false, nil, errors.New("unsupported SQLite observer text identity") + } + if rowid == nil { + if id.primaryKey != 1 || strings.Contains(strings.ToUpper(ddl), "UNIQUE") { + return false, nil, errors.New("unsupported SQLite observer primary key") + } + for _, c := range cols { + if c.name != "id" && c.primaryKey != 0 { + return false, nil, errors.New("unsupported SQLite composite observer primary key") + } + } + return false, cols, nil + } + if rowid.kind != "INTEGER" || rowid.primaryKey != 1 || id.primaryKey != 0 || !strings.Contains(strings.ToUpper(ddl), "AUTOINCREMENT") { + return false, nil, errors.New("unsupported SQLite observer rowid identity") + } + // A text identity remains a BINARY unique key (including SQLite's existing + // multiple-NULL behavior), so foreign keys naming observers(id) still work. + idx, err := db.Query(`SELECT name FROM pragma_index_list('observers') WHERE "unique"=1 AND partial=0`) + if err != nil { + return false, nil, err + } + var names []string + for idx.Next() { + var name string + if err = idx.Scan(&name); err != nil { + idx.Close() + return false, nil, err + } + names = append(names, name) + } + if err = idx.Err(); err != nil { + idx.Close() + return false, nil, err + } + idx.Close() + for _, name := range names { + var total, matching int + err = db.QueryRow(`SELECT count(*),coalesce(sum(name='id' AND coll='BINARY'),0) FROM pragma_index_xinfo(?) WHERE key=1`, name).Scan(&total, &matching) + if err != nil { + return false, nil, err + } + if total == 1 && matching == 1 { + return true, cols, nil + } + } + return false, nil, errors.New("SQLite observer text identity is not a BINARY unique key") +} + +func sqliteIdentityVersion(db Querier) error { + var exists int + if err := db.QueryRow(`SELECT count(*) FROM sqlite_master WHERE type='table' AND name='_migrations'`).Scan(&exists); err != nil { + return err + } + if exists == 0 { + return nil + } + const prefix = "observers_identity_autoincrement_v" + rows, err := db.Query(`SELECT name FROM _migrations WHERE substr(name,1,?)=?`, len(prefix), prefix) + if err != nil { + return err + } + defer rows.Close() + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return err + } + if name != SQLiteObserverIdentityMigration { + return errors.New("unsupported newer SQLite observer identity migration") + } + } + return rows.Err() +} + +func migrateSQLiteObserverIdentity(db *sql.DB) (err error) { + ctx := context.Background() + conn, err := db.Conn(ctx) + if err != nil { + return err + } + defer conn.Close() + var foreignKeys, legacyAlter int + if err = conn.QueryRowContext(ctx, `PRAGMA foreign_keys`).Scan(&foreignKeys); err != nil { + return err + } + if err = conn.QueryRowContext(ctx, `PRAGMA legacy_alter_table`).Scan(&legacyAlter); err != nil { + return err + } + defer func() { + for _, q := range []string{fmt.Sprintf("PRAGMA legacy_alter_table=%d", legacyAlter), fmt.Sprintf("PRAGMA foreign_keys=%d", foreignKeys)} { + if _, restoreErr := conn.ExecContext(ctx, q); err == nil && restoreErr != nil { + err = restoreErr + } + } + }() + // Use the same reserved connection throughout. Disabling FK actions avoids + // cascades while replacing the parent; references are checked before commit. + if _, err = conn.ExecContext(ctx, `PRAGMA foreign_keys=OFF`); err != nil { + return err + } + if _, err = conn.ExecContext(ctx, `PRAGMA legacy_alter_table=ON`); err != nil { + return err + } + tx, err := conn.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + migrated, cols, err := sqliteObserverLayout(tx) + if err != nil { + return err + } + if err = sqliteForeignKeysValid(tx); err != nil { + return err + } + if !migrated { + var objects []string + rows, e := tx.Query(`SELECT sql FROM sqlite_master WHERE tbl_name='observers' AND type IN ('index','trigger') AND sql IS NOT NULL ORDER BY type,name`) + if e != nil { + return e + } + for rows.Next() { + var ddl string + if e = rows.Scan(&ddl); e != nil { + rows.Close() + return e + } + objects = append(objects, ddl) + } + if e = rows.Err(); e != nil { + rows.Close() + return e + } + rows.Close() + fields := []string{`rowid INTEGER PRIMARY KEY AUTOINCREMENT`} + names := []string{`rowid`} + for _, c := range cols { + field := sqliteQuote(c.name) + " " + c.kind + if c.name == "id" { + field += " UNIQUE" + } + if c.notNull != 0 { + field += " NOT NULL" + } + if c.defaultSQL.Valid { + field += " DEFAULT " + c.defaultSQL.String + } + fields = append(fields, field) + names = append(names, sqliteQuote(c.name)) + } + const replacement = "_corescope_observers_identity_v1" + if _, err = tx.Exec(`CREATE TABLE ` + replacement + ` (` + strings.Join(fields, ",") + `)`); err != nil { + return err + } + columns := strings.Join(names, ",") + if _, err = tx.Exec(`INSERT INTO ` + replacement + ` (` + columns + `) SELECT ` + columns + ` FROM observers`); err != nil { + return err + } + if _, err = tx.Exec(`DROP TABLE observers`); err != nil { + return err + } + if _, err = tx.Exec(`ALTER TABLE ` + replacement + ` RENAME TO observers`); err != nil { + return err + } + for _, ddl := range objects { + if _, err = tx.Exec(ddl); err != nil { + return err + } + } + } + // Include retained orphan links and the sequence itself, so deleted IDs can + // never be reassigned to a new observer. Other table sequences and cursors + // are untouched by this migration; reverse import must seed them separately. + var high int64 + if err = tx.QueryRow(`SELECT max(value) FROM ( + SELECT coalesce(max(rowid),0) AS value FROM observers UNION ALL + SELECT coalesce(max(observer_idx),0) FROM observations UNION ALL + SELECT coalesce(max(seq),0) FROM sqlite_sequence WHERE name='observers')`).Scan(&high); err != nil { + return err + } + res, err := tx.Exec(`UPDATE sqlite_sequence SET seq=? WHERE name='observers'`, high) + if err != nil { + return err + } + n, err := res.RowsAffected() + if err != nil { + return err + } + if n == 0 { + if _, err = tx.Exec(`INSERT INTO sqlite_sequence(name,seq) VALUES('observers',?)`, high); err != nil { + return err + } + } + if err = sqliteForeignKeysValid(tx); err != nil { + return err + } + if _, err = tx.Exec(`INSERT OR IGNORE INTO _migrations(name) VALUES(?)`, SQLiteObserverIdentityMigration); err != nil { + return err + } + return tx.Commit() +} + +func sqliteForeignKeysValid(db Querier) error { + rows, err := db.Query(`PRAGMA foreign_key_check`) + if err != nil { + return err + } + defer rows.Close() + if rows.Next() { + return errors.New("SQLite foreign-key violations prevent observer identity migration") + } + return rows.Err() +} diff --git a/internal/dbschema/sqlite_indexes.go b/internal/dbschema/sqlite_indexes.go new file mode 100644 index 000000000..459508043 --- /dev/null +++ b/internal/dbschema/sqlite_indexes.go @@ -0,0 +1,16 @@ +package dbschema + +import ( + "context" + "database/sql" +) + +// EnsureSQLiteObserverTimeIndex is shared by the online asynchronous migration +// and offline conversion. PostgreSQL creates its own index during owner setup. +// A transaction or reserved connection may be used during offline finalization. +func EnsureSQLiteObserverTimeIndex(ctx context.Context, db interface { + ExecContext(context.Context, string, ...any) (sql.Result, error) +}) error { + _, err := db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_observations_observer_idx_timestamp ON observations(observer_idx, timestamp)`) + return err +} diff --git a/internal/dbschema/sqlite_test.go b/internal/dbschema/sqlite_test.go new file mode 100644 index 000000000..74eac0e82 --- /dev/null +++ b/internal/dbschema/sqlite_test.go @@ -0,0 +1,449 @@ +package dbschema + +import ( + "context" + "database/sql" + "errors" + "fmt" + "path/filepath" + "strings" + "testing" + + "github.com/mattn/go-sqlite3" + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/dbschema/legacy" +) + +func sqliteTestDB(t *testing.T) (*sql.DB, string) { + t.Helper() + path := filepath.Join(t.TempDir(), "telemetry.db") + dsn, err := legacy.WriterDSN(path) + if err != nil { + t.Fatal(err) + } + db, err := sql.Open("sqlite3", dsn) + if err != nil { + t.Fatal(err) + } + db.SetMaxOpenConns(1) + t.Cleanup(func() { db.Close() }) + return db, path +} + +func sqliteExec(t *testing.T, db *sql.DB, query string, args ...any) { + t.Helper() + if _, err := db.Exec(query, args...); err != nil { + t.Fatal(err) + } +} + +func TestSQLiteSchemaOwnsAsyncMigrationLedger(t *testing.T) { + db, _ := sqliteTestDB(t) + if err := ApplySQLite(db, nil); err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`INSERT INTO _async_migrations(name,status,error) VALUES('retained','failed','exact error')`); err != nil { + t.Fatal(err) + } + if err := EnsureSQLiteAsyncMigrations(db); err != nil { + t.Fatal(err) + } + var status, started, detail string + if err := db.QueryRow(`SELECT status,started_at,error FROM _async_migrations WHERE name='retained'`).Scan(&status, &started, &detail); err != nil || status != "failed" || started == "" || detail != "exact error" { + t.Fatal("async ledger changed", err) + } +} + +func TestSQLitePendingConversionCannotBeAdoptedOrOpened(t *testing.T) { + db, _ := sqliteTestDB(t) + if err := ApplySQLite(db, nil); err != nil { + t.Fatal(err) + } + sqliteExec(t, db, `CREATE TABLE corescope_reverse_progress(table_name TEXT PRIMARY KEY,rows_copied INTEGER NOT NULL,sha256 TEXT NOT NULL,complete INTEGER NOT NULL)`) + for _, check := range []func() error{func() error { return ApplySQLite(db, nil) }, func() error { return AssertSQLiteReady(db) }, func() error { return dbconfig.AssertSQLiteImportComplete(db) }} { + if err := check(); !errors.Is(err, dbconfig.ErrSQLiteImportIncomplete) { + t.Fatal("incomplete target accepted", err) + } + } + var count int + if err := db.QueryRow(`SELECT count(*) FROM sqlite_master WHERE name='corescope_reverse_progress'`).Scan(&count); err != nil || count != 1 { + t.Fatal("pending marker changed", err) + } +} + +func TestSQLiteFreshAndReadOnlyReadiness(t *testing.T) { + db, path := sqliteTestDB(t) + if err := AssertSQLiteReady(db); err == nil { + t.Fatal("empty schema ready") + } + if err := ApplySQLite(db, nil); err != nil { + t.Fatal(err) + } + if err := ApplySQLite(db, nil); err != nil { + t.Fatal(err) + } + if err := AssertSQLiteReady(db); err != nil { + t.Fatal(err) + } + var marker int + if err := db.QueryRow(`SELECT count(*) FROM _migrations WHERE name=?`, SQLiteObserverIdentityMigration).Scan(&marker); err != nil || marker != 1 { + t.Fatalf("marker=%d err=%v", marker, err) + } + if err := db.Close(); err != nil { + t.Fatal(err) + } + ro, err := sql.Open("sqlite3", "file:"+filepath.ToSlash(path)+"?mode=ro") + if err != nil { + t.Fatal(err) + } + defer ro.Close() + ro.SetMaxOpenConns(1) + if err := AssertSQLiteReady(ro); err != nil { + t.Fatal(err) + } + if _, err := ro.Exec(`INSERT INTO observers(id) VALUES('reader-write')`); err == nil { + t.Fatal("reader wrote telemetry") + } + if err := ApplySQLite(ro, nil); err == nil { + t.Fatal("reader ran writer schema entrypoint") + } +} + +func TestSQLiteFreshSchemaDoesNotSuppressInitialPlannerAnalysis(t *testing.T) { + db, _ := sqliteTestDB(t) + for range 2 { + if err := ApplySQLite(db, nil); err != nil { + t.Fatal(err) + } + } + // The ingestor uses sqlite_stat1's presence to decide whether its initial + // whole-database ANALYZE is required. A table-only refresh must not mask it. + var tables int + if err := db.QueryRow(`SELECT count(*) FROM sqlite_master WHERE name='sqlite_stat1'`).Scan(&tables); err != nil || tables != 0 { + t.Fatalf("fresh database appears fully analyzed: tables=%d err=%v", tables, err) + } +} + +func TestSQLiteObserverIdentityPreservesOldRowsAndReferences(t *testing.T) { + db, _ := sqliteTestDB(t) + if err := legacy.ApplyBase(db); err != nil { + t.Fatal(err) + } + if err := legacy.Apply(db, nil); err != nil { + t.Fatal(err) + } + sqliteExec(t, db, `INSERT INTO observers(rowid,id,name) VALUES(0,'zero','zero'),(7,'Node','upper'),(8,'node','lower'),(20,NULL,'null-one'),(30,NULL,'null-two')`) + sqliteExec(t, db, `INSERT INTO transmissions(id,raw_hex,hash,first_seen) VALUES(1,'aa','hash','2026-01-01'),(77,'bb','deleted','2026-01-01')`) + sqliteExec(t, db, `DELETE FROM transmissions WHERE id=77`) + sqliteExec(t, db, `INSERT INTO observations(id,transmission_id,observer_idx,timestamp) VALUES(1,1,7,1),(2,1,99,1),(3,1,NULL,1),(88,1,8,2)`) + sqliteExec(t, db, `DELETE FROM observations WHERE id=88`) + sqliteExec(t, db, `INSERT INTO advert_evidence_backfill(id,tx_cursor,obs_cursor) VALUES(1,700,900)`) + sqliteExec(t, db, `CREATE TABLE observer_child(observer_id TEXT REFERENCES observers(id) ON DELETE CASCADE)`) + sqliteExec(t, db, `INSERT INTO observer_child VALUES('Node')`) + sqliteExec(t, db, `CREATE INDEX custom_observer_name ON observers(name)`) + sqliteExec(t, db, `CREATE TABLE observer_audit(value TEXT)`) + sqliteExec(t, db, `CREATE TRIGGER observer_name_changed AFTER UPDATE OF name ON observers BEGIN INSERT INTO observer_audit VALUES(new.name); END`) + sqliteExec(t, db, `CREATE VIEW observer_view AS SELECT rowid,id,name FROM observers`) + before := sqliteObserverRows(t, db) + if err := ApplySQLite(db, nil); err != nil { + t.Fatal(err) + } + if got := sqliteObserverRows(t, db); got != before { + t.Fatalf("rows changed: before=%s after=%s", before, got) + } + for query, want := range map[string]int64{ + `SELECT count(*) FROM observers`: 5, + `SELECT count(*) FROM observer_child`: 1, + `SELECT count(*) FROM observer_view`: 5, + `SELECT count(*) FROM observer_audit`: 0, + `SELECT count(*) FROM sqlite_master WHERE name='custom_observer_name'`: 1, + `SELECT count(*) FROM observations WHERE observer_idx=99`: 1, + `SELECT count(*) FROM observations WHERE observer_idx IS NULL`: 1, + `SELECT seq FROM sqlite_sequence WHERE name='observers'`: 99, + `SELECT seq FROM sqlite_sequence WHERE name='transmissions'`: 77, + `SELECT seq FROM sqlite_sequence WHERE name='observations'`: 88, + `SELECT tx_cursor FROM advert_evidence_backfill WHERE id=1`: 700, + `SELECT obs_cursor FROM advert_evidence_backfill WHERE id=1`: 900, + `PRAGMA foreign_keys`: 1, + `PRAGMA legacy_alter_table`: 0, + } { + var got int64 + if err := db.QueryRow(query).Scan(&got); err != nil || got != want { + t.Fatalf("%s: %d want %d err=%v", query, got, want, err) + } + } + if _, err := db.Exec(`INSERT INTO observers(id) VALUES('Node')`); err == nil { + t.Fatal("lost text-id uniqueness") + } + if _, err := db.Exec(`INSERT INTO observer_child VALUES('NODE')`); err == nil { + t.Fatal("lost binary foreign-key matching") + } + sqliteExec(t, db, `UPDATE observers SET name='updated' WHERE id='Node'`) + var value string + if err := db.QueryRow(`SELECT value FROM observer_audit`).Scan(&value); err != nil || value != "updated" { + t.Fatalf("trigger: %q %v", value, err) + } + sqliteExec(t, db, `DELETE FROM observers WHERE id='Node'`) + var count int + if err := db.QueryRow(`SELECT count(*) FROM observer_child`).Scan(&count); err != nil || count != 0 { + t.Fatal("foreign-key cascade changed") + } + for _, want := range []int64{100, 101} { + res, err := db.Exec(`INSERT INTO observers(id) VALUES('new')`) + if err != nil { + t.Fatal(err) + } + got, err := res.LastInsertId() + if err != nil || got != want { + t.Fatalf("next id=%d want=%d err=%v", got, want, err) + } + sqliteExec(t, db, `DELETE FROM observers WHERE id='new'`) + if err := ApplySQLite(db, nil); err != nil { + t.Fatal(err) + } + } +} + +func sqliteObserverRows(t *testing.T, db *sql.DB) string { + t.Helper() + rows, err := db.Query(`SELECT rowid,id,name FROM observers ORDER BY rowid`) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + var out strings.Builder + for rows.Next() { + var id int64 + var key, name sql.NullString + if err := rows.Scan(&id, &key, &name); err != nil { + t.Fatal(err) + } + fmt.Fprintf(&out, "%d:%#v:%#v\n", id, key, name) + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + return out.String() +} + +func TestSQLiteUnknownObserverLayoutRefusesBeforeMutation(t *testing.T) { + for _, tc := range []struct{ name, old, replacement string }{ + {"collation", "id TEXT PRIMARY KEY", "id TEXT COLLATE NOCASE PRIMARY KEY"}, + {"check", "name TEXT", "name TEXT CHECK(name<>'')"}, + {"shadow rowid", "id TEXT PRIMARY KEY", "rowid TEXT,id TEXT PRIMARY KEY"}, + {"conflict policy", "id TEXT PRIMARY KEY", "id TEXT PRIMARY KEY ON CONFLICT IGNORE"}, + {"composite key", "id TEXT PRIMARY KEY", "id TEXT"}, + } { + t.Run(tc.name, func(t *testing.T) { + db, _ := sqliteTestDB(t) + if err := legacy.ApplyBase(db); err != nil { + t.Fatal(err) + } + if err := legacy.Apply(db, nil); err != nil { + t.Fatal(err) + } + var ddl string + if err := db.QueryRow(`SELECT sql FROM sqlite_master WHERE name='observers' AND type='table'`).Scan(&ddl); err != nil { + t.Fatal(err) + } + if !strings.Contains(ddl, tc.old) { + t.Fatal("fixture does not contain expected declaration") + } + ddl = strings.Replace(ddl, tc.old, tc.replacement, 1) + if tc.name == "composite key" { + ddl = strings.TrimSuffix(strings.TrimSpace(ddl), ")") + ",PRIMARY KEY(id,name))" + } + sqliteExec(t, db, `DROP TABLE observers`) + sqliteExec(t, db, ddl) + sqliteExec(t, db, `INSERT INTO observers(id,name) VALUES('kept','value')`) + var before, after int + if err := db.QueryRow(`PRAGMA schema_version`).Scan(&before); err != nil { + t.Fatal(err) + } + if err := ApplySQLite(db, nil); err == nil { + t.Fatal("unsupported schema accepted") + } + if err := db.QueryRow(`PRAGMA schema_version`).Scan(&after); err != nil { + t.Fatal(err) + } + if before != after { + t.Fatal("refusal mutated schema") + } + var name string + if err := db.QueryRow(`SELECT name FROM observers`).Scan(&name); err != nil || name != "value" { + t.Fatal("refusal changed row") + } + }) + } +} + +func TestSQLiteObserverMigrationFailureRollsBackAndRestoresConnection(t *testing.T) { + db, _ := sqliteTestDB(t) + if err := legacy.ApplyBase(db); err != nil { + t.Fatal(err) + } + if err := legacy.Apply(db, nil); err != nil { + t.Fatal(err) + } + sqliteExec(t, db, `INSERT INTO observers(rowid,id,name) VALUES(12,'preserve','value')`) + // A conflicting pre-existing object is never overwritten or repurposed. + sqliteExec(t, db, `CREATE TABLE _corescope_observers_identity_v1(value TEXT)`) + sqliteExec(t, db, `INSERT INTO _corescope_observers_identity_v1 VALUES('owned elsewhere')`) + before := sqliteObserverRows(t, db) + if err := ApplySQLite(db, nil); err == nil { + t.Fatal("conflicting replacement table accepted") + } + if got := sqliteObserverRows(t, db); got != before { + t.Fatal("failed rebuild changed observers") + } + var got string + if err := db.QueryRow(`SELECT value FROM _corescope_observers_identity_v1`).Scan(&got); err != nil || got != "owned elsewhere" { + t.Fatal("failed rebuild modified existing object") + } + for query, want := range map[string]int{ + `SELECT count(*) FROM _migrations WHERE name='observers_identity_autoincrement_v1'`: 0, + `PRAGMA foreign_keys`: 1, + `PRAGMA legacy_alter_table`: 0, + } { + var got int + if err := db.QueryRow(query).Scan(&got); err != nil || got != want { + t.Fatalf("%s: %d want %d err=%v", query, got, want, err) + } + } +} + +func TestSQLiteFutureIdentityVersionRefusesBeforeMutation(t *testing.T) { + db, _ := sqliteTestDB(t) + if err := legacy.ApplyBase(db); err != nil { + t.Fatal(err) + } + if err := legacy.Apply(db, nil); err != nil { + t.Fatal(err) + } + sqliteExec(t, db, `INSERT INTO _migrations(name) VALUES('observers_identity_autoincrement_v2')`) + var before, after int + if err := db.QueryRow(`PRAGMA schema_version`).Scan(&before); err != nil { + t.Fatal(err) + } + if err := ApplySQLite(db, nil); err == nil { + t.Fatal("future version accepted") + } + if err := db.QueryRow(`PRAGMA schema_version`).Scan(&after); err != nil { + t.Fatal(err) + } + if before != after { + t.Fatal("refusal modified schema") + } +} + +func sqliteAuthorizer(t *testing.T, db *sql.DB, callback func(int, string, string, string) int) { + t.Helper() + conn, err := db.Conn(context.Background()) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + if err := conn.Raw(func(driver any) error { driver.(*sqlite3.SQLiteConn).RegisterAuthorizer(callback); return nil }); err != nil { + t.Fatal(err) + } +} + +func TestSQLiteIdentityRollbackAfterDroppingOriginal(t *testing.T) { + db, _ := sqliteTestDB(t) + if err := legacy.ApplyBase(db); err != nil { + t.Fatal(err) + } + if err := legacy.Apply(db, nil); err != nil { + t.Fatal(err) + } + sqliteExec(t, db, `INSERT INTO observers(rowid,id,name) VALUES(12,'preserve','value')`) + sqliteExec(t, db, `CREATE TABLE observer_child(observer_id TEXT REFERENCES observers(id) ON DELETE CASCADE)`) + sqliteExec(t, db, `INSERT INTO observer_child VALUES('preserve')`) + before := sqliteObserverRows(t, db) + dropped, denied := false, false + sqliteAuthorizer(t, db, func(op int, a, b, c string) int { + if op == sqlite3.SQLITE_DROP_TABLE && a == "observers" { + dropped = true + } + if op == sqlite3.SQLITE_ALTER_TABLE { + denied = true + return sqlite3.SQLITE_DENY + } + return sqlite3.SQLITE_OK + }) + err := ApplySQLite(db, nil) + sqliteAuthorizer(t, db, nil) + if err == nil || !dropped || !denied { + t.Fatalf("failure did not occur after drop: err=%v dropped=%v denied=%v", err, dropped, denied) + } + if got := sqliteObserverRows(t, db); got != before { + t.Fatal("rollback changed original rows") + } + for query, want := range map[string]int{ + `SELECT count(*) FROM observer_child`: 1, + `SELECT count(*) FROM sqlite_master WHERE name='_corescope_observers_identity_v1'`: 0, + `SELECT count(*) FROM _migrations WHERE name='observers_identity_autoincrement_v1'`: 0, + `PRAGMA foreign_keys`: 1, + `PRAGMA legacy_alter_table`: 0, + } { + var got int + if err := db.QueryRow(query).Scan(&got); err != nil || got != want { + t.Fatalf("%s: got=%d err=%v", query, got, err) + } + } +} + +func TestSQLiteRepeatedApplyDoesNotRepeatCorpusForeignKeyScan(t *testing.T) { + db, _ := sqliteTestDB(t) + if err := ApplySQLite(db, nil); err != nil { + t.Fatal(err) + } + sqliteAuthorizer(t, db, func(op int, a, b, c string) int { + if op == sqlite3.SQLITE_PRAGMA && a == "foreign_key_check" { + return sqlite3.SQLITE_DENY + } + return sqlite3.SQLITE_OK + }) + defer sqliteAuthorizer(t, db, nil) + if err := ApplySQLite(db, nil); err != nil { + t.Fatal(err) + } + if err := AssertSQLiteReady(db); err != nil { + t.Fatal(err) + } +} + +func TestSQLitePartialUniqueObserverKeyIsNotReady(t *testing.T) { + db, _ := sqliteTestDB(t) + if err := ApplySQLite(db, nil); err != nil { + t.Fatal(err) + } + var ddl string + if err := db.QueryRow(`SELECT sql FROM sqlite_master WHERE name='observers' AND type='table'`).Scan(&ddl); err != nil { + t.Fatal(err) + } + const fullKey = `"id" TEXT UNIQUE` + if !strings.Contains(ddl, fullKey) { + t.Fatal("fixture lacks full unique text key") + } + ddl = strings.Replace(ddl, fullKey, `"id" TEXT`, 1) + sqliteExec(t, db, `DROP TABLE observers`) + sqliteExec(t, db, ddl) + sqliteExec(t, db, `CREATE UNIQUE INDEX observers_partial_identity ON observers(id) WHERE id IS NOT NULL`) + if err := AssertSQLiteReady(db); err == nil { + t.Fatal("partial unique text key accepted as ready") + } + var before, after int + if err := db.QueryRow(`PRAGMA schema_version`).Scan(&before); err != nil { + t.Fatal(err) + } + if err := ApplySQLite(db, nil); err == nil { + t.Fatal("partial unique text key accepted for upgrade") + } + if err := db.QueryRow(`PRAGMA schema_version`).Scan(&after); err != nil { + t.Fatal(err) + } + if before != after { + t.Fatal("refusal mutated partial-key schema") + } +} diff --git a/internal/dbschema/writer_dsn_test.go b/internal/dbschema/writer_dsn_test.go deleted file mode 100644 index 1e83fa020..000000000 --- a/internal/dbschema/writer_dsn_test.go +++ /dev/null @@ -1,43 +0,0 @@ -package dbschema - -import ( - "database/sql" - "path/filepath" - "testing" - - _ "github.com/mattn/go-sqlite3" -) - -// TestWriterDSNPragmas pins what every writer connection actually gets. -// -// cmd/ingestor has its own end-to-end version of this through the store, but -// this one covers the DSN itself, which is what cmd/migrate also opens with. -// The synchronous line matters most: mattn defaults it to NORMAL and runs the -// pragma unconditionally, so if it ever drops out of the DSN the durability -// change is silent. -func TestWriterDSNPragmas(t *testing.T) { - db, err := sql.Open("sqlite3", WriterDSN(filepath.Join(t.TempDir(), "w.db"))) - if err != nil { - t.Fatal(err) - } - defer db.Close() - db.SetMaxOpenConns(1) - - for _, want := range []struct{ pragma, value, why string }{ - {"journal_mode", "wal", "cmd/server reads concurrently"}, - {"synchronous", "2", "FULL; mattn defaults to 1 (NORMAL) unless the DSN says otherwise"}, - {"auto_vacuum", "2", "INCREMENTAL; maintenance.go drives incremental_vacuum"}, - {"foreign_keys", "1", "the schema relies on FK enforcement"}, - {"busy_timeout", "5000", "writers serialise behind the reader"}, - {"cache_size", "-2000", "C-allocated page cache, pinned: it sits outside GOMEMLIMIT"}, - } { - var got string - if err := db.QueryRow("PRAGMA " + want.pragma).Scan(&got); err != nil { - t.Errorf("PRAGMA %s: %v", want.pragma, err) - continue - } - if got != want.value { - t.Errorf("PRAGMA %s = %q, want %q (%s)", want.pragma, got, want.value, want.why) - } - } -} diff --git a/internal/packetpath/hash.go b/internal/packetpath/hash.go new file mode 100644 index 000000000..9d4c4b051 --- /dev/null +++ b/internal/packetpath/hash.go @@ -0,0 +1,61 @@ +package packetpath + +import ( + "crypto/sha256" + "encoding/hex" +) + +// ContentHash preserves CoreScope packet identity. This is the existing server +// algorithm shared by runtime and offline fixture preparation. Transport routes +// and TRACE uint16 path_len were checked against MeshCore a366955 Packet.cpp/Packet.h. +func ContentHash(rawHex string) string { + buf, err := hex.DecodeString(rawHex) + if err != nil || len(buf) < 2 { + if len(rawHex) >= 16 { + return rawHex[:16] + } + return rawHex + } + + headerByte := buf[0] + offset := 1 + if IsTransportRoute(int(headerByte & 0x03)) { + offset += 4 + } + if offset >= len(buf) { + if len(rawHex) >= 16 { + return rawHex[:16] + } + return rawHex + } + pathByte := buf[offset] + offset++ + hashSize := int((pathByte>>6)&0x3) + 1 + hashCount := int(pathByte & 0x3F) + pathBytes := hashSize * hashCount + + payloadStart := offset + pathBytes + if payloadStart > len(buf) { + if len(rawHex) >= 16 { + return rawHex[:16] + } + return rawHex + } + + payload := buf[payloadStart:] + + // Hash payload-type byte only (bits 2-5 of header), not the full header. + // Firmware: SHA256(payload_type + [path_len for TRACE] + payload) + // Using the full header caused different hashes for the same logical packet + // when route type or version bits differed. See issue #786. + payloadType := (headerByte >> 2) & 0x0F + toHash := []byte{payloadType} + if int(payloadType) == 0x09 { + // Firmware uses uint16_t path_len (2 bytes, little-endian) + toHash = append(toHash, pathByte, 0x00) + } + toHash = append(toHash, payload...) + + h := sha256.Sum256(toHash) + return hex.EncodeToString(h[:])[:16] +} diff --git a/internal/packetpath/hash_test.go b/internal/packetpath/hash_test.go new file mode 100644 index 000000000..386ba5943 --- /dev/null +++ b/internal/packetpath/hash_test.go @@ -0,0 +1,45 @@ +package packetpath + +import "testing" + +func TestContentHashAllRoutesKeepBaselineGolden(t *testing.T) { + // Packet.cpp and Packet.h at MeshCore a366955: only routes 0/3 carry + // transport codes; those codes and hop bytes do not enter packet identity. + for _, raw := range []string{ + "1401020304000102030405060708090a0b0c0d0e0f", + "15000102030405060708090a0b0c0d0e0f", + "16000102030405060708090a0b0c0d0e0f", + "1701020304000102030405060708090a0b0c0d0e0f", + "1502aabb0102030405060708090a0b0c0d0e0f", + "1541aabb0102030405060708090a0b0c0d0e0f", + "1581aabbcc0102030405060708090a0b0c0d0e0f", + } { + if got := ContentHash(raw); got != "fae0c9e6d357a814" { + t.Fatalf("hash %s=%s", raw, got) + } + } +} + +func TestContentHashMalformedFallbackPreserved(t *testing.T) { + for _, raw := range []string{"", "a", "zz", "00112233", "150faabb", "invalid-long-raw-frame"} { + want := raw + if len(want) > 16 { + want = want[:16] + } + if got := ContentHash(raw); got != want { + t.Fatalf("malformed fallback %q=%q; want %q", raw, got, want) + } + } +} + +func TestContentHashTraceAndAnonymousRequestGoldens(t *testing.T) { + for _, test := range []struct{ raw, want string }{ + {"250001020304", "240ed67f297f83f9"}, + {"2501aa01020304", "dd79cbe99f3fc8f5"}, + {"1d0001020304", "14d68bbc63e0902d"}, + } { + if got := ContentHash(test.raw); got != test.want { + t.Fatalf("hash %s=%s; want %s", test.raw, got, test.want) + } + } +} diff --git a/internal/pgutil/dump.go b/internal/pgutil/dump.go new file mode 100644 index 000000000..135396dd5 --- /dev/null +++ b/internal/pgutil/dump.go @@ -0,0 +1,85 @@ +package pgutil + +import ( + "context" + "errors" + "fmt" + "net/url" + "os" + "os/exec" + "strconv" + "strings" + "time" +) + +// CommandEnv supplies libpq credentials through the child environment, never +// through argv. It removes inherited PG settings so a dump cannot reach a +// different database than the application. Callers must never log this slice. +func CommandEnv(dsn string) ([]string, error) { + c, err := ParseConfig(dsn) + if err != nil { + return nil, err + } + u, _ := url.Parse(c.ConnString()) + env := make([]string, 0, len(os.Environ())+12) + for _, e := range os.Environ() { + if !strings.HasPrefix(strings.ToUpper(e), "PG") { + env = append(env, e) + } + } + env = append(env, "PGHOST="+c.Host, "PGPORT="+strconv.Itoa(int(c.Port)), "PGDATABASE="+c.Database, + "PGUSER="+c.User, "PGPASSWORD="+c.Password, "PGCONNECT_TIMEOUT="+strconv.Itoa(int(c.ConnectTimeout/time.Second))) + for _, pair := range [][2]string{{"sslmode", "PGSSLMODE"}, {"sslrootcert", "PGSSLROOTCERT"}, + {"sslcert", "PGSSLCERT"}, {"sslkey", "PGSSLKEY"}, {"sslpassword", "PGSSLPASSWORD"}, + {"target_session_attrs", "PGTARGETSESSIONATTRS"}, {"application_name", "PGAPPNAME"}} { + if value := u.Query().Get(pair[0]); value != "" { + env = append(env, pair[1]+"="+value) + } + } + return env, nil +} + +// Dump writes a consistent native custom-format archive with owner-only mode. +// It never overwrites an existing file, and removes its own partial output on +// any failure or cancellation. The PostgreSQL client executable must be on PATH. +func Dump(ctx context.Context, dsn, path string) (err error) { + ctx, cancel := context.WithTimeout(ctx, 10*time.Minute) + defer cancel() + env, err := CommandEnv(dsn) + if err != nil { + return err + } + u, _ := url.Parse(dsn) + // Installed selections pin the default schema explicitly. pg_dump still + // exports the whole database; do not turn this into a schema-filtered dump. + switch strings.TrimSpace(u.Query().Get("search_path")) { + case "", "public", `"public"`: + default: + return errors.New("postgres: native backup requires the default public schema") + } + f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + if errors.Is(err, os.ErrExist) { + return errors.New("postgres: snapshot target already exists") + } + return fmt.Errorf("postgres: create snapshot: %w", err) + } + defer func() { + if closeErr := f.Close(); err == nil { + err = closeErr + } + if err != nil { + _ = os.Remove(path) + } + }() + cmd := exec.CommandContext(ctx, "pg_dump", "--format=custom", "--no-password", "--no-owner", "--no-privileges") + cmd.Env, cmd.Stdout = env, f + // Never return client stderr: libpq errors can include connection settings. + if runErr := cmd.Run(); runErr != nil { + if ctx.Err() != nil { + return ctx.Err() + } + return errors.New("postgres: pg_dump failed; check client installation, version and backup privileges") + } + return f.Sync() +} diff --git a/internal/pgutil/dump_schema_test.go b/internal/pgutil/dump_schema_test.go new file mode 100644 index 000000000..c9f5bf1d8 --- /dev/null +++ b/internal/pgutil/dump_schema_test.go @@ -0,0 +1,90 @@ +package pgutil_test + +import ( + "context" + "net/url" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/meshcore-analyzer/pgutil" + "github.com/meshcore-analyzer/pgutil/pgtest" +) + +func TestDumpPublicSchemaOwnerRestoresWholeDatabase(t *testing.T) { + for _, schema := range []string{"public", `"public"`} { + t.Run(schema, func(t *testing.T) { + dsn := pgtest.NewDatabase(t) + u, _ := url.Parse(dsn) + q := u.Query() + q.Set("search_path", schema) + u.RawQuery = q.Encode() + dsn = u.String() + db, err := pgutil.Open(dsn, false) + if err != nil { + t.Fatal(err) + } + defer db.Close() + if _, err := db.Exec(`CREATE TABLE sample(id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, value TEXT); + INSERT INTO sample(value) VALUES('kept'),('deleted'); DELETE FROM sample WHERE id=2; + CREATE SCHEMA auxiliary; CREATE TABLE auxiliary.retained(value TEXT); INSERT INTO auxiliary.retained VALUES('whole database')`); err != nil { + t.Fatal(err) + } + path := filepath.Join(t.TempDir(), "owner.dump") + if err := pgutil.Dump(context.Background(), dsn, path); err != nil { + t.Fatal(err) + } + target := pgtest.NewDatabase(t) + env, err := pgutil.CommandEnv(target) + if err != nil { + t.Fatal(err) + } + cfg, err := pgutil.ParseConfig(target) + if err != nil { + t.Fatal(err) + } + cmd := exec.Command("pg_restore", "--no-password", "--exit-on-error", "--no-owner", "--no-privileges", "--dbname="+cfg.Database, path) + cmd.Env = env + if err := cmd.Run(); err != nil { + t.Fatal("native whole-database restore failed") + } + restored, err := pgutil.Open(target, false) + if err != nil { + t.Fatal(err) + } + defer restored.Close() + var kept, auxiliary string + if err := restored.QueryRow(`SELECT value FROM sample WHERE id=1`).Scan(&kept); err != nil || kept != "kept" { + t.Fatal("restored public data changed", err) + } + if err := restored.QueryRow(`SELECT value FROM auxiliary.retained`).Scan(&auxiliary); err != nil || auxiliary != "whole database" { + t.Fatal("public search_path incorrectly filtered the native archive", err) + } + var nextID int + if err := restored.QueryRow(`INSERT INTO sample(value) VALUES('next') RETURNING id`).Scan(&nextID); err != nil || nextID != 3 { + t.Fatal("restored identity high-water changed", err) + } + }) + } +} + +func TestDumpRejectsCustomOrMultipleSchemasBeforeOutput(t *testing.T) { + for _, schema := range []string{"custom", "public,custom", `"public","custom"`, `"Public"`, "$user,public", "pg_catalog"} { + t.Run(schema, func(t *testing.T) { + u, _ := url.Parse("postgresql://owner:private-sentinel@localhost/example?sslmode=disable") + q := u.Query() + q.Set("search_path", schema) + u.RawQuery = q.Encode() + path := filepath.Join(t.TempDir(), "refused.dump") + err := pgutil.Dump(context.Background(), u.String(), path) + if err == nil || !strings.Contains(err.Error(), "default public schema") || strings.Contains(err.Error(), "private-sentinel") { + t.Fatal("unsupported backup scope was accepted or exposed credentials") + } + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Fatal("rejected schema left an output file") + } + }) + } +} diff --git a/internal/pgutil/go.mod b/internal/pgutil/go.mod new file mode 100644 index 000000000..a26277f9d --- /dev/null +++ b/internal/pgutil/go.mod @@ -0,0 +1,14 @@ +module github.com/meshcore-analyzer/pgutil + +go 1.25.0 + +require github.com/jackc/pgx/v5 v5.7.6 + +require ( + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + golang.org/x/crypto v0.37.0 // indirect + golang.org/x/sync v0.13.0 // indirect + golang.org/x/text v0.24.0 // indirect +) diff --git a/internal/pgutil/go.sum b/internal/pgutil/go.sum new file mode 100644 index 000000000..7600ae8d6 --- /dev/null +++ b/internal/pgutil/go.sum @@ -0,0 +1,28 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.7.6 h1:rWQc5FwZSPX58r1OQmkuaNicxdmExaEz5A2DO2hUuTk= +github.com/jackc/pgx/v5 v5.7.6/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= +golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= +golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= +golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= +golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/pgutil/pgtest/pgtest.go b/internal/pgutil/pgtest/pgtest.go new file mode 100644 index 000000000..1e17cff05 --- /dev/null +++ b/internal/pgutil/pgtest/pgtest.go @@ -0,0 +1,141 @@ +// Package pgtest provisions disposable PostgreSQL test namespaces. Tests fail +// rather than silently skipping when CORESCOPE_TEST_POSTGRES_URL is missing. +package pgtest + +import ( + "crypto/rand" + "database/sql" + "encoding/hex" + "net/url" + "os" + "strings" + "testing" + + "github.com/meshcore-analyzer/pgutil" +) + +func name() string { + var b [12]byte + if _, err := rand.Read(b[:]); err != nil { + panic(err) + } + return "corescope_test_" + hex.EncodeToString(b[:]) +} + +func admin(t testing.TB) (*sql.DB, *url.URL) { + t.Helper() + dsn := os.Getenv("CORESCOPE_TEST_POSTGRES_URL") + if dsn == "" { + t.Fatal("set CORESCOPE_TEST_POSTGRES_URL to a disposable PostgreSQL 18 administrator URL") + } + u, err := url.Parse(dsn) + if err != nil { + t.Fatal("invalid CORESCOPE_TEST_POSTGRES_URL") + } + db, err := pgutil.Open(dsn, false) + if err != nil { + t.Fatal(err) + } + return db, u +} + +// NewSchema returns a URL selecting a fresh empty schema. No fixture tables +// are created. Register pool cleanup after this call so pools close first. +func NewSchema(t testing.TB) string { + t.Helper() + db, u := admin(t) + schema := name() + if _, err := db.Exec(`CREATE SCHEMA ` + schema); err != nil { + db.Close() + t.Fatal(err) + } + t.Cleanup(func() { + defer db.Close() + if _, err := db.Exec(`DROP SCHEMA ` + schema + ` CASCADE`); err != nil { + t.Errorf("drop test schema: %v", err) + } + }) + q := u.Query() + q.Set("search_path", schema) + u.RawQuery = q.Encode() + return u.String() +} + +// NewDatabase returns a fresh empty logical database for isolation and restore +// tests. Only this randomly named database is removed by cleanup. +func NewDatabase(t testing.TB) string { + t.Helper() + db, u := admin(t) + database := name() + if _, err := db.Exec(`CREATE DATABASE ` + database + ` TEMPLATE template0`); err != nil { + db.Close() + t.Fatal(err) + } + t.Cleanup(func() { + defer db.Close() + if _, err := db.Exec(`DROP DATABASE ` + database + ` WITH (FORCE)`); err != nil { + t.Errorf("drop test database: %v", err) + } + }) + u.Path = "/" + database + q := u.Query() + q.Del("search_path") + u.RawQuery = q.Encode() + return u.String() +} + +// ReadOnly provisions a reader for the selected schema and future tables. +// It grants neither schema creation nor any write privilege. +func ReadOnly(t testing.TB, writerDSN string) string { + return role(t, writerDSN, false) +} + +// Writer provisions DML and sequence privileges, without schema ownership or DDL. +// A store can revoke writes on its readiness metadata after provisioning. +func Writer(t testing.TB, ownerDSN string) string { + return role(t, ownerDSN, true) +} + +func role(t testing.TB, writerDSN string, write bool) string { + t.Helper() + db, err := pgutil.Open(writerDSN, false) + if err != nil { + t.Fatal(err) + } + role, password := name(), name() + var schema string + if err := db.QueryRow(`SELECT current_schema()`).Scan(&schema); err != nil { + db.Close() + t.Fatal(err) + } + quotedSchema := `"` + schema + `"` + privileges := "SELECT" + if write { + privileges = "SELECT,INSERT,UPDATE,DELETE" + } + statements := []string{`CREATE ROLE ` + role + ` LOGIN PASSWORD '` + password + `'`, + `GRANT USAGE ON SCHEMA ` + quotedSchema + ` TO ` + role, + `GRANT ` + privileges + ` ON ALL TABLES IN SCHEMA ` + quotedSchema + ` TO ` + role, + `ALTER DEFAULT PRIVILEGES IN SCHEMA ` + quotedSchema + ` GRANT ` + privileges + ` ON TABLES TO ` + role} + sequencePrivileges := "SELECT" + if write { + sequencePrivileges = "USAGE,SELECT" + } + statements = append(statements, `GRANT `+sequencePrivileges+` ON ALL SEQUENCES IN SCHEMA `+quotedSchema+` TO `+role, + `ALTER DEFAULT PRIVILEGES IN SCHEMA `+quotedSchema+` GRANT `+sequencePrivileges+` ON SEQUENCES TO `+role) + // One implicit transaction preserves atomicity and avoids a durable commit + // per grant in suites with hundreds of isolated roles. + if _, err := db.Exec(strings.Join(statements, ";")); err != nil { + db.Close() + t.Fatal(err) + } + t.Cleanup(func() { + defer db.Close() + if _, err := db.Exec(`DROP OWNED BY ` + role + `;DROP ROLE ` + role); err != nil { + t.Errorf("drop test role: %v", err) + } + }) + u, _ := url.Parse(writerDSN) + u.User = url.UserPassword(role, password) + return u.String() +} diff --git a/internal/pgutil/pgutil.go b/internal/pgutil/pgutil.go new file mode 100644 index 000000000..38e733beb --- /dev/null +++ b/internal/pgutil/pgutil.go @@ -0,0 +1,150 @@ +// Package pgutil contains PostgreSQL connection and native backup helpers. +package pgutil + +import ( + "context" + "crypto/rand" + "database/sql" + "encoding/binary" + "errors" + "fmt" + "net/url" + "os" + "strings" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/stdlib" +) + +// ParseConfig requires an explicit PostgreSQL URL. Errors never include the +// URL: driver parse errors can contain credentials, including URL parameters. +func ParseConfig(dsn string) (*pgx.ConnConfig, error) { + u, err := url.Parse(dsn) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") { + return nil, errors.New("postgres: expected a PostgreSQL database URL") + } + if u.Hostname() == "" || strings.Contains(u.Host, ",") || u.User == nil || u.User.Username() == "" || strings.Trim(u.Path, "/") == "" { + return nil, errors.New("postgres: database URL requires one host, username and database") + } + allowed := map[string]bool{"sslmode": true, "sslrootcert": true, "sslcert": true, "sslkey": true, "sslpassword": true, + "target_session_attrs": true, "connect_timeout": true, "application_name": true, "search_path": true} + q, err := url.ParseQuery(u.RawQuery) + if err != nil { + return nil, errors.New("postgres: invalid database URL query") + } + for key, values := range q { + if !allowed[key] || len(values) != 1 { + return nil, errors.New("postgres: unsupported or repeated database URL parameter") + } + } + // Normalize TLS environment defaults into the parsed URL so libpq and pgx + // receive the same connection policy. Service files/options and multi-host + // URLs are deliberately unsupported rather than silently changing a backup. + if os.Getenv("PGSERVICE") != "" || os.Getenv("PGOPTIONS") != "" { + return nil, errors.New("postgres: use explicit database URL settings instead of PGSERVICE or PGOPTIONS") + } + for _, pair := range [][2]string{{"sslmode", "PGSSLMODE"}, {"sslrootcert", "PGSSLROOTCERT"}, {"sslcert", "PGSSLCERT"}, {"sslkey", "PGSSLKEY"}, {"sslpassword", "PGSSLPASSWORD"}} { + if !q.Has(pair[0]) && os.Getenv(pair[1]) != "" { + q.Set(pair[0], os.Getenv(pair[1])) + } + } + if !q.Has("sslmode") { + q.Set("sslmode", "prefer") + } + u.RawQuery = q.Encode() + c, err := pgx.ParseConfig(u.String()) + if err != nil || c.Database == "" { + return nil, errors.New("postgres: invalid database URL or missing database name") + } + // Ten seconds is the connection upper bound; retain a shorter explicit limit. + if c.ConnectTimeout <= 0 || c.ConnectTimeout > 10*time.Second { + c.ConnectTimeout = 10 * time.Second + } + return c, nil +} + +// Open returns a bounded pgx database/sql pool. readOnly is defense in depth; +// deployments must also grant the runtime role only its required privileges. +func Open(dsn string, readOnly bool) (*sql.DB, error) { + c, err := ParseConfig(dsn) + if err != nil { + return nil, err + } + c.RuntimeParams["timezone"] = "UTC" + if readOnly { + c.RuntimeParams["default_transaction_read_only"] = "on" + } + db := stdlib.OpenDB(*c) + db.SetMaxOpenConns(8) + db.SetMaxIdleConns(2) + db.SetConnMaxIdleTime(5 * time.Minute) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := db.PingContext(ctx); err != nil { + db.Close() + return nil, errors.New("postgres: connection failed; check database URL, credentials and server availability") + } + return db, nil +} + +// AssertReadOnly checks effective grants, including inherited grants. The +// session's read-only setting alone is insufficient: a client can change it. +func AssertReadOnly(db *sql.DB) error { + var writable bool + err := db.QueryRow(`SELECT + EXISTS (SELECT 1 FROM pg_roles WHERE rolname=current_user AND (rolsuper OR rolcreaterole OR rolcreatedb)) + OR has_database_privilege(current_database(), 'CREATE') + OR EXISTS (SELECT 1 FROM pg_namespace WHERE nspname !~ '^pg_' AND nspname <> 'information_schema' + AND has_schema_privilege(oid, 'CREATE')) + OR EXISTS (SELECT 1 FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace + WHERE n.nspname !~ '^pg_' AND n.nspname <> 'information_schema' AND c.relkind IN ('r','p','v','m','f') + AND has_table_privilege(c.oid, 'INSERT,UPDATE,DELETE,TRUNCATE,TRIGGER'))`).Scan(&writable) + if err != nil { + return errors.New("postgres: cannot verify reader privileges") + } + if writable { + return errors.New("postgres: reader role has write or schema privileges") + } + return nil +} + +// SameDatabase compares the actual databases reached by two URLs, independent +// of aliases, proxies, usernames, search_path, and URL spelling. Advisory locks +// are database-local and need no DDL/data write privilege. Both transactions +// always roll back; the random lock leaves no persistent state. +func SameDatabase(ctx context.Context, left, right string) (bool, error) { + a, err := Open(left, true) + if err != nil { + return false, err + } + defer a.Close() + b, err := Open(right, true) + if err != nil { + return false, err + } + defer b.Close() + ta, err := a.BeginTx(ctx, nil) + if err != nil { + return false, errors.New("postgres: database identity check failed") + } + defer ta.Rollback() + tb, err := b.BeginTx(ctx, nil) + if err != nil { + return false, errors.New("postgres: database identity check failed") + } + defer tb.Rollback() + var key [8]byte + if _, err := rand.Read(key[:]); err != nil { + return false, err + } + id := int64(binary.BigEndian.Uint64(key[:])) + var locked bool + if err := ta.QueryRowContext(ctx, `SELECT pg_try_advisory_xact_lock($1)`, id).Scan(&locked); err != nil || !locked { + return false, errors.New("postgres: database identity lock unavailable") + } + if err := tb.QueryRowContext(ctx, `SELECT pg_try_advisory_xact_lock($1)`, id).Scan(&locked); err != nil { + return false, fmt.Errorf("postgres: database identity probe failed") + } + return !locked, nil +} diff --git a/internal/pgutil/pgutil_test.go b/internal/pgutil/pgutil_test.go new file mode 100644 index 000000000..7df8c60da --- /dev/null +++ b/internal/pgutil/pgutil_test.go @@ -0,0 +1,232 @@ +package pgutil_test + +import ( + "context" + "net/url" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/meshcore-analyzer/pgutil" + "github.com/meshcore-analyzer/pgutil/pgtest" +) + +func TestParseConfigSecretsAndSupportedURL(t *testing.T) { + for _, dsn := range []string{ + "postgresql://hidden-user:hidden-password@host:bad/db", + "postgresql://hidden-user:hidden-password@host/db?password=hidden-password", + "postgresql://hidden-user:hidden-password@one,two/db", + "postgresql://hidden-user:hidden-password@host/db?hostaddr=elsewhere", + "postgresql://hidden-user:hidden-password@host/db?options=-csearch_path=elsewhere", + "postgresql://hidden-user:hidden-password@host/db?sslmode=disable&sslmode=require", + } { + _, err := pgutil.ParseConfig(dsn) + if err == nil { + t.Fatal("unsupported connection URL accepted") + } + if strings.Contains(err.Error(), "hidden-") { + t.Fatal("connection error leaked a credential") + } + } + c, err := pgutil.ParseConfig("postgresql://user:p%40ss@localhost:5433/accounts?sslmode=disable&search_path=test_schema") + if err != nil { + t.Fatal(err) + } + if c.Password != "p@ss" || c.Database != "accounts" || c.Port != 5433 || c.RuntimeParams["search_path"] != "test_schema" { + t.Fatal("URL parameters changed") + } +} + +func TestParseConfigRejectsMalformedQueryWithoutTLSDowngrade(t *testing.T) { + for _, key := range []string{"PGSSLMODE", "PGSSLROOTCERT", "PGSSLCERT", "PGSSLKEY", "PGSSLPASSWORD", "PGSERVICE", "PGOPTIONS"} { + t.Setenv(key, "") + } + for _, test := range []struct{ name, query string }{ + {"bad_value_escape", "sslmode=verify-full%ZZ"}, + {"unescaped_separator", "sslmode=verify-full;application_name=hidden-param"}, + {"bad_key_escape", "ssl%ZZmode=verify-full"}, + {"bad_other_parameter", "sslmode=verify-full&sslpassword=hidden-param%ZZ"}, + } { + t.Run(test.name, func(t *testing.T) { + dsn := "postgresql://hidden-user:hidden-password@localhost/accounts?" + test.query + if _, err := pgutil.ParseConfig(dsn); err == nil { + t.Fatal("malformed query accepted instead of preserving the requested connection policy") + } else if strings.Contains(err.Error(), "hidden-") || strings.Contains(err.Error(), test.query) { + t.Fatal("malformed query error leaked connection settings") + } + if _, err := pgutil.CommandEnv(dsn); err == nil { + t.Fatal("native client accepted the malformed query") + } + }) + } + c, err := pgutil.ParseConfig("postgresql://encoded%20user:p%40ss%25@localhost/accounts?ssl%6dode=verify%2Dfull&application_name=synthetic%3Btls%3Dstrict%26name%3Dread%2Bonly") + if err != nil { + t.Fatal("valid encoded URL refused", err) + } + if c.User != "encoded user" || c.Password != "p@ss%" || c.RuntimeParams["application_name"] != "synthetic;tls=strict&name=read+only" { + t.Fatal("valid encoded URL changed") + } + if c.TLSConfig == nil || c.TLSConfig.InsecureSkipVerify || len(c.Fallbacks) != 0 { + t.Fatal("valid strict-TLS URL was downgraded") + } +} + +func TestCommandEnvMatchesConnection(t *testing.T) { + dsn := "postgresql://someone:secret%40value@localhost:5433/accounts?sslmode=require" + env, err := pgutil.CommandEnv(dsn) + if err != nil { + t.Fatal(err) + } + values := map[string]string{} + for _, e := range env { + k, v, _ := strings.Cut(e, "=") + values[k] = v + } + for k, want := range map[string]string{"PGHOST": "localhost", "PGPORT": "5433", "PGDATABASE": "accounts", "PGUSER": "someone", "PGPASSWORD": "secret@value", "PGSSLMODE": "require"} { + if values[k] != want { + t.Errorf("%s did not preserve connection settings", k) + } + } +} + +func TestConnectTimeoutKeepsShorterLimitAndCapsLonger(t *testing.T) { + for _, test := range []struct { + seconds string + want time.Duration + }{{"1", time.Second}, {"30", 10 * time.Second}, {"0", 10 * time.Second}} { + c, err := pgutil.ParseConfig("postgresql://user:password@localhost/database?connect_timeout=" + test.seconds) + if err != nil { + t.Fatal(err) + } + if c.ConnectTimeout != test.want { + t.Fatalf("timeout %s became %s; want %s", test.seconds, c.ConnectTimeout, test.want) + } + } +} + +func TestSameDatabaseChecksActualTarget(t *testing.T) { + dsn := pgtest.NewSchema(t) + reader := pgtest.ReadOnly(t, dsn) + u, _ := url.Parse(reader) + q := u.Query() + q.Set("application_name", "identity-test") + q.Del("search_path") + u.RawQuery = q.Encode() + same, err := pgutil.SameDatabase(context.Background(), dsn, u.String()) + if err != nil || !same { + t.Fatalf("same database across roles/search paths=%v,%v", same, err) + } + other := pgtest.NewDatabase(t) + same, err = pgutil.SameDatabase(context.Background(), dsn, other) + if err != nil || same { + t.Fatalf("separate databases=%v,%v", same, err) + } +} + +func TestReadOnlyRoleDeniesWritesAndDDL(t *testing.T) { + dsn := pgtest.NewSchema(t) + owner, err := pgutil.Open(dsn, false) + if err != nil { + t.Fatal(err) + } + defer owner.Close() + if _, err := owner.Exec(`CREATE TABLE sample(id BIGINT PRIMARY KEY)`); err != nil { + t.Fatal(err) + } + readerDSN := pgtest.ReadOnly(t, dsn) + db, err := pgutil.Open(readerDSN, true) + if err != nil { + t.Fatal(err) + } + defer db.Close() + if err := pgutil.AssertReadOnly(db); err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`SET default_transaction_read_only=off`); err != nil { + t.Fatal(err) + } + for _, stmt := range []string{`INSERT INTO sample VALUES(1)`, `UPDATE sample SET id=2`, `DELETE FROM sample`, `CREATE TABLE forbidden(id int)`} { + if _, err := db.Exec(stmt); err == nil { + t.Fatalf("reader accepted write: %s", stmt) + } + } + u, _ := url.Parse(readerDSN) + var database string + if err := owner.QueryRow(`SELECT current_database()`).Scan(&database); err != nil { + t.Fatal(err) + } + if _, err := owner.Exec(`GRANT CREATE ON DATABASE "` + database + `" TO "` + u.User.Username() + `"`); err != nil { + t.Fatal(err) + } + if err := pgutil.AssertReadOnly(db); err == nil { + t.Fatal("database CREATE privilege was missed") + } +} + +func TestDumpRestoreAndFailureCleanup(t *testing.T) { + dsn := pgtest.NewDatabase(t) + db, err := pgutil.Open(dsn, false) + if err != nil { + t.Fatal(err) + } + defer db.Close() + if _, err := db.Exec(`CREATE TABLE sample(id BIGINT PRIMARY KEY,value TEXT); INSERT INTO sample VALUES(17,'unchanged')`); err != nil { + t.Fatal(err) + } + path := filepath.Join(t.TempDir(), "snapshot.dump") + if err := pgutil.Dump(context.Background(), dsn, path); err != nil { + t.Fatal(err) + } + content, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if !strings.HasPrefix(string(content), "PGDMP") { + t.Fatal("backup is not a native custom archive") + } + target := pgtest.NewDatabase(t) + env, err := pgutil.CommandEnv(target) + if err != nil { + t.Fatal(err) + } + cfg, err := pgutil.ParseConfig(target) + if err != nil { + t.Fatal(err) + } + cmd := exec.Command("pg_restore", "--no-password", "--exit-on-error", "--no-owner", "--no-privileges", "--dbname="+cfg.Database, path) + cmd.Env = env + if err := cmd.Run(); err != nil { + t.Fatal("native restore failed") + } + restored, err := pgutil.Open(target, false) + if err != nil { + t.Fatal(err) + } + defer restored.Close() + var value string + if err := restored.QueryRow(`SELECT value FROM sample WHERE id=17`).Scan(&value); err != nil || value != "unchanged" { + t.Fatalf("restored data=%q,%v", value, err) + } + if err := pgutil.Dump(context.Background(), dsn, path); err == nil { + t.Fatal("existing backup overwritten") + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + partial := filepath.Join(t.TempDir(), "partial.dump") + if err := pgutil.Dump(ctx, dsn, partial); err == nil { + t.Fatal("canceled dump succeeded") + } + if _, err := os.Stat(partial); !os.IsNotExist(err) { + t.Fatal("partial backup remains") + } + t.Setenv("PATH", t.TempDir()) + if err := pgutil.Dump(context.Background(), dsn, partial); err == nil { + t.Fatal("dump succeeded without pg_dump") + } + if _, err := os.Stat(partial); !os.IsNotExist(err) { + t.Fatal("failed dump left partial backup") + } +} diff --git a/internal/sqliteutil/go.mod b/internal/sqliteutil/go.mod new file mode 100644 index 000000000..c8085b762 --- /dev/null +++ b/internal/sqliteutil/go.mod @@ -0,0 +1,10 @@ +module github.com/meshcore-analyzer/sqliteutil + +go 1.25.0 + +require ( + github.com/mattn/go-sqlite3 v1.14.52 + github.com/meshcore-analyzer/dbconfig v0.0.0 +) + +replace github.com/meshcore-analyzer/dbconfig => ../dbconfig diff --git a/internal/sqliteutil/go.sum b/internal/sqliteutil/go.sum new file mode 100644 index 000000000..75bf6eb53 --- /dev/null +++ b/internal/sqliteutil/go.sum @@ -0,0 +1,2 @@ +github.com/mattn/go-sqlite3 v1.14.52 h1:wVbm2Qnf4OXkqhBTSPuCRZDRnxfbVrrmiCEroVdog8U= +github.com/mattn/go-sqlite3 v1.14.52/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w= diff --git a/internal/sqliteutil/snapshot.go b/internal/sqliteutil/snapshot.go new file mode 100644 index 000000000..f8c9c330c --- /dev/null +++ b/internal/sqliteutil/snapshot.go @@ -0,0 +1,138 @@ +// Package sqliteutil provides native snapshots without a writable source handle. +package sqliteutil + +import ( + "context" + "database/sql" + "errors" + "fmt" + "github.com/mattn/go-sqlite3" + "github.com/meshcore-analyzer/dbconfig" + "net/url" + "os" + "strings" + "time" +) + +// Snapshot copies one consistent SQLite image, including committed WAL pages, +// hidden rowids and AUTOINCREMENT sequences, to a new private file. It never +// writes the source. The caller may impose a shorter deadline; ten minutes is +// the upper bound. Failure removes only the new destination and its sidecars. +// Offline importers must separately enforce their source-immutability policy. +func Snapshot(ctx context.Context, source, destination string) (err error) { + if err := ctx.Err(); err != nil { + return err + } + ctx, cancel := context.WithTimeout(ctx, 10*time.Minute) + defer cancel() + if source == "" || destination == "" || strings.HasPrefix(source, "file:") || strings.Contains(source, "://") || strings.HasPrefix(destination, "file:") || strings.Contains(destination, "://") { + return errors.New("SQLite snapshot requires filesystem paths") + } + sourceURI, err := dbconfig.SQLiteURI(source, url.Values{"mode": {"ro"}, "_query_only": {"on"}, "_busy_timeout": {"100"}}) + if err != nil { + return err + } + destinationURI, err := dbconfig.SQLiteURI(destination, url.Values{"mode": {"rw"}, "_busy_timeout": {"100"}}) + if err != nil { + return err + } + sourceDB, err := sql.Open("sqlite3", sourceURI) + if err != nil { + return err + } + defer sourceDB.Close() + sourceDB.SetMaxOpenConns(1) + if _, err := sourceDB.ExecContext(ctx, `PRAGMA trusted_schema=OFF`); err != nil { + return err + } + var schemaVersion int + if err := sourceDB.QueryRowContext(ctx, `PRAGMA schema_version`).Scan(&schemaVersion); err != nil { + return fmt.Errorf("read SQLite source: %w", err) + } + reserved, err := os.OpenFile(destination, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if err != nil { + return fmt.Errorf("create new recovery snapshot: %w", err) + } + if err := reserved.Close(); err != nil { + _ = os.Remove(destination) + return err + } + defer func() { + if err != nil { + for _, suffix := range []string{"", "-wal", "-shm", "-journal"} { + _ = os.Remove(destination + suffix) + } + } + }() + destinationDB, err := sql.Open("sqlite3", destinationURI) + if err != nil { + return err + } + defer destinationDB.Close() + destinationDB.SetMaxOpenConns(1) + src, err := sourceDB.Conn(ctx) + if err != nil { + return err + } + defer src.Close() + dst, err := destinationDB.Conn(ctx) + if err != nil { + return err + } + defer dst.Close() + err = src.Raw(func(sourceDriver any) error { + return dst.Raw(func(destinationDriver any) (err error) { + backup, err := destinationDriver.(*sqlite3.SQLiteConn).Backup("main", sourceDriver.(*sqlite3.SQLiteConn), "main") + if err != nil { + return err + } + defer func() { + if finishErr := backup.Finish(); err == nil { + err = finishErr + } + }() + remaining, progressAt := -1, time.Now() + for { + if err := ctx.Err(); err != nil { + return err + } + done, err := backup.Step(256) + if err != nil { + return err + } + if done { + return nil + } + if current := backup.Remaining(); current != remaining { + remaining, progressAt = current, time.Now() + continue + } + if time.Since(progressAt) > 10*time.Second { + return errors.New("SQLite snapshot made no progress") + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(10 * time.Millisecond): + } + } + }) + }) + if err != nil { + return fmt.Errorf("copy SQLite recovery snapshot: %w", err) + } + var check string + if err := dst.QueryRowContext(ctx, `PRAGMA quick_check`).Scan(&check); err != nil { + return err + } + if check != "ok" { + return errors.New("SQLite recovery snapshot failed integrity verification") + } + if err := dst.Close(); err != nil { + return err + } + if err := destinationDB.Close(); err != nil { + return err + } + return nil +} diff --git a/internal/sqliteutil/snapshot_test.go b/internal/sqliteutil/snapshot_test.go new file mode 100644 index 000000000..9f850b155 --- /dev/null +++ b/internal/sqliteutil/snapshot_test.go @@ -0,0 +1,124 @@ +package sqliteutil + +import ( + "context" + "database/sql" + "errors" + "net/url" + "os" + "path/filepath" + "testing" + "time" + + _ "github.com/mattn/go-sqlite3" + "github.com/meshcore-analyzer/dbconfig" +) + +func open(t *testing.T, path, mode string) *sql.DB { + t.Helper() + dsn, err := dbconfig.SQLiteURI(path, url.Values{"mode": {mode}, "_busy_timeout": {"50"}}) + if err != nil { + t.Fatal(err) + } + db, err := sql.Open("sqlite3", dsn) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { db.Close() }) + return db +} + +func TestSnapshotPreservesWALRowIDsAndSequence(t *testing.T) { + dir := t.TempDir() + source := filepath.Join(dir, "live # %.db") + destination := filepath.Join(dir, "snapshot # %.db") + db := open(t, source, "rwc") + if _, err := db.Exec(`PRAGMA journal_mode=WAL; PRAGMA wal_autocheckpoint=0; + CREATE TABLE legacy(id TEXT); INSERT INTO legacy(rowid,id) VALUES(3,'a'),(101,'b'); + CREATE TABLE identities(id INTEGER PRIMARY KEY AUTOINCREMENT,value TEXT); + INSERT INTO identities(id,value) VALUES(900,'deleted'); DELETE FROM identities; + INSERT INTO identities(value) VALUES('last');`); err != nil { + t.Fatal(err) + } + if info, err := os.Stat(source + "-wal"); err != nil || info.Size() == 0 { + t.Fatal("fixture must contain uncheckpointed WAL") + } + if err := Snapshot(context.Background(), source, destination); err != nil { + t.Fatal(err) + } + copy := open(t, destination, "ro") + var rowid, id int + if err := copy.QueryRow(`SELECT rowid FROM legacy WHERE id='b'`).Scan(&rowid); err != nil || rowid != 101 { + t.Fatalf("rowid=%d: %v", rowid, err) + } + if err := copy.QueryRow(`SELECT id FROM identities`).Scan(&id); err != nil || id != 901 { + t.Fatalf("identity=%d: %v", id, err) + } + if err := copy.QueryRow(`SELECT seq FROM sqlite_sequence WHERE name='identities'`).Scan(&id); err != nil || id != 901 { + t.Fatalf("sequence=%d: %v", id, err) + } + if _, err := copy.Exec(`CREATE TABLE forbidden(id)`); err == nil { + t.Fatal("readonly snapshot handle allowed write") + } + if _, err := open(t, source, "ro").Exec(`DELETE FROM legacy`); err == nil { + t.Fatal("readonly source allowed write") + } +} + +func TestSnapshotRefusesExistingAndMissingSource(t *testing.T) { + dir := t.TempDir() + source := filepath.Join(dir, "source.db") + if _, err := open(t, source, "rwc").Exec(`CREATE TABLE t(id);INSERT INTO t VALUES(1)`); err != nil { + t.Fatal(err) + } + for _, destination := range []string{source, filepath.Join(dir, "existing.db")} { + if destination != source { + if err := os.WriteFile(destination, []byte("keep"), 0600); err != nil { + t.Fatal(err) + } + } + before, err := os.ReadFile(destination) + if err != nil { + t.Fatal(err) + } + if err := Snapshot(context.Background(), source, destination); err == nil { + t.Fatal("overwrote existing target") + } + after, err := os.ReadFile(destination) + if err != nil || string(after) != string(before) { + t.Fatal("existing target changed") + } + } + missing := filepath.Join(dir, "missing.db") + destination := filepath.Join(dir, "new.db") + if err := Snapshot(context.Background(), missing, destination); err == nil { + t.Fatal("missing source accepted") + } + for _, p := range []string{missing, destination} { + if _, err := os.Stat(p); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("unexpected file %q: %v", p, err) + } + } +} + +func TestSnapshotCancellationRemovesPartialOutput(t *testing.T) { + dir := t.TempDir() + source := filepath.Join(dir, "busy.db") + destination := filepath.Join(dir, "cancelled.db") + db := open(t, source, "rwc") + db.SetMaxOpenConns(1) + if _, err := db.Exec(`CREATE TABLE t(id);INSERT INTO t VALUES(1);BEGIN EXCLUSIVE`); err != nil { + t.Fatal(err) + } + defer db.Exec(`ROLLBACK`) + ctx, cancel := context.WithTimeout(context.Background(), 75*time.Millisecond) + defer cancel() + if err := Snapshot(ctx, source, destination); err == nil { + t.Fatal("snapshot ignored locked source/cancellation") + } + for _, suffix := range []string{"", "-wal", "-shm", "-journal"} { + if _, err := os.Stat(destination + suffix); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("partial snapshot remains: %v", err) + } + } +} diff --git a/internal/users/audit.go b/internal/users/audit.go index ff3f47e6b..e88fcb23d 100644 --- a/internal/users/audit.go +++ b/internal/users/audit.go @@ -3,6 +3,7 @@ package users import ( "database/sql" "encoding/json" + "fmt" "strings" "time" ) @@ -44,7 +45,7 @@ func (s *Store) Audit(actor *int64, action string, target *int64, detail map[str if err != nil { return err } - _, err = s.db.Exec(`INSERT INTO audit_log (at, actor_user_id, action, target_user_id, detail) VALUES (?, ?, ?, ?, ?)`, + _, err = s.db.Exec(`INSERT INTO audit_log (at, actor_user_id, action, target_user_id, detail) VALUES (`+s.p(1)+`, `+s.p(2)+`, `+s.p(3)+`, `+s.p(4)+`, `+s.p(5)+`)`, unix(s.now()), nullInt(actor), action, nullInt(target), string(b)) return err } @@ -60,12 +61,12 @@ func (s *Store) AuditFor(userID int64, limit int) ([]AuditEntry, error) { // AuditAllFor is AuditFor without a cap (the account export). func (s *Store) AuditAllFor(userID int64) ([]AuditEntry, error) { - return s.auditFor(userID, noLimit) + return s.auditFor(userID, s.noLimit()) } -func (s *Store) auditFor(userID int64, limit int) ([]AuditEntry, error) { +func (s *Store) auditFor(userID int64, limit any) ([]AuditEntry, error) { rows, err := s.db.Query(`SELECT id, at, actor_user_id, action, target_user_id, detail FROM audit_log - WHERE target_user_id = ? OR actor_user_id = ? ORDER BY at DESC, id DESC LIMIT ?`, userID, userID, limit) + WHERE target_user_id = `+s.p(1)+` OR actor_user_id = `+s.p(2)+` ORDER BY at DESC, id DESC LIMIT `+s.p(3), userID, userID, limit) if err != nil { return nil, err } @@ -80,29 +81,29 @@ func (s *Store) AuditList(f AuditFilter) ([]AuditEntry, error) { var ors []string for _, a := range f.Actions { if group, ok := strings.CutSuffix(a, ".*"); ok { - ors = append(ors, `action = ? OR action LIKE ? ESCAPE '\'`) + ors = append(ors, fmt.Sprintf(`action = %s OR action LIKE %s ESCAPE '\'`, s.p(len(args)+1), s.p(len(args)+2))) args = append(args, group, escapeLike(group)+".%") } else { - ors = append(ors, `action = ?`) + ors = append(ors, fmt.Sprintf(`action = %s`, s.p(len(args)+1))) args = append(args, a) } } q += ` AND (` + strings.Join(ors, ` OR `) + `)` } if f.UserID != nil { - q += ` AND (actor_user_id = ? OR target_user_id = ?)` + q += fmt.Sprintf(` AND (actor_user_id = %s OR target_user_id = %s)`, s.p(len(args)+1), s.p(len(args)+2)) args = append(args, *f.UserID, *f.UserID) } if f.From != nil { - q += ` AND at >= ?` + q += fmt.Sprintf(` AND at >= %s`, s.p(len(args)+1)) args = append(args, unix(*f.From)) } if f.To != nil { - q += ` AND at <= ?` + q += fmt.Sprintf(` AND at <= %s`, s.p(len(args)+1)) args = append(args, unix(*f.To)) } if f.BeforeID > 0 { - q += ` AND id < ?` + q += fmt.Sprintf(` AND id < %s`, s.p(len(args)+1)) args = append(args, f.BeforeID) } limit := f.Limit @@ -112,7 +113,7 @@ func (s *Store) AuditList(f AuditFilter) ([]AuditEntry, error) { if limit > AuditListMax { limit = AuditListMax } - q += ` ORDER BY id DESC LIMIT ?` + q += fmt.Sprintf(` ORDER BY id DESC LIMIT %s`, s.p(len(args)+1)) args = append(args, limit) rows, err := s.db.Query(q, args...) if err != nil { @@ -132,8 +133,8 @@ func (s *Store) PruneAudit(actions []string, maxAge time.Duration) (int64, error args = append(args, a) } args = append(args, unix(s.now())-int64(maxAge/time.Second)) - ph := placeholders(len(actions)) - res, err := s.db.Exec(`DELETE FROM audit_log WHERE action IN (`+ph+`) AND at < ?`, args...) + ph := s.placeholders(len(actions)) + res, err := s.db.Exec(`DELETE FROM audit_log WHERE action IN (`+ph+`) AND at < `+s.p(len(args)), args...) if err != nil { return 0, err } diff --git a/internal/users/audit_list_test.go b/internal/users/audit_list_test.go index a74f33eac..407479a29 100644 --- a/internal/users/audit_list_test.go +++ b/internal/users/audit_list_test.go @@ -1,9 +1,8 @@ package users import ( - "database/sql" "fmt" - "path/filepath" + "github.com/meshcore-analyzer/dbconfig" "strings" "testing" "time" @@ -147,7 +146,11 @@ func TestUsersByID(t *testing.T) { func hasIndex(t *testing.T, st *Store, name string) bool { t.Helper() var n int - if err := st.db.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE type = 'index' AND name = ?`, name).Scan(&n); err != nil { + q := `SELECT COUNT(*) FROM pg_indexes WHERE schemaname=current_schema() AND indexname=$1` + if st.backend == dbconfig.SQLite { + q = `SELECT COUNT(*) FROM sqlite_schema WHERE type='index' AND name=?1` + } + if err := st.db.QueryRow(q, name).Scan(&n); err != nil { t.Fatal(err) } return n == 1 @@ -160,34 +163,11 @@ func TestFreshDatabaseHasAuditTimeIndex(t *testing.T) { } } -// A users.db written by a v2 binary gains the audit_at index and keeps its rows. +// A native account schema at v2 binary gains the audit_at index and keeps its rows. func TestMigrateV2DatabaseToV3(t *testing.T) { - path := filepath.Join(t.TempDir(), "users.db") - db, err := sql.Open("sqlite", path) - if err != nil { - t.Fatal(err) - } - stmts := []string{ - `CREATE TABLE schema_version (version INTEGER NOT NULL)`, - `INSERT INTO schema_version (version) VALUES (2)`, - } - stmts = append(stmts, migrations[0]...) - stmts = append(stmts, migrations[1]...) - stmts = append(stmts, `INSERT INTO audit_log (at, action, detail) VALUES (1, 'user.register', '{}')`) - for _, s := range stmts { - if _, err := db.Exec(s); err != nil { - t.Fatalf("build v2 db: %v", err) - } - } - db.Close() - - st, err := Open(path) - if err != nil { - t.Fatalf("Open v2 db: %v", err) - } - defer st.Close() - if v, err := st.SchemaVersion(); err != nil || v != len(migrations) { - t.Fatalf("SchemaVersion = %d, %v; want %d", v, err, len(migrations)) + st := migratedTestStore(t, 2, `INSERT INTO audit_log (at,action,detail) VALUES (1,'user.register','{}')`) + if v, err := st.SchemaVersion(); err != nil || v != schemaVersion(t) { + t.Fatalf("SchemaVersion = %d, %v; want %d", v, err, schemaVersion(t)) } if !hasIndex(t, st, "audit_at") { t.Fatal("audit_at index missing after migration") diff --git a/internal/users/challenges.go b/internal/users/challenges.go index 4dc8bcd77..c582c9bc0 100644 --- a/internal/users/challenges.go +++ b/internal/users/challenges.go @@ -32,7 +32,7 @@ func (s *Store) CreateLinkChallenge(userID int64, pubkey string) (string, time.T return "", time.Time{}, fmt.Errorf("users: challenge: %w", err) } exp := unix(s.now().Add(LinkChallengeTTL)) - if _, err := s.db.Exec(`INSERT INTO link_challenges (challenge_hash, user_id, pubkey, expires_at) VALUES (?, ?, ?, ?)`, + if _, err := s.db.Exec(`INSERT INTO link_challenges (challenge_hash, user_id, pubkey, expires_at) VALUES (`+s.p(1)+`, `+s.p(2)+`, `+s.p(3)+`, `+s.p(4)+`)`, challengeHash(raw), userID, pk, exp); err != nil { return "", time.Time{}, fmt.Errorf("users: create link challenge: %w", err) } @@ -51,7 +51,7 @@ func (s *Store) ConsumeLinkChallenge(userID int64, pubkey, challenge string) err } var owner, exp int64 var pk string - err = s.db.QueryRow(`DELETE FROM link_challenges WHERE challenge_hash = ? RETURNING user_id, pubkey, expires_at`, + err = s.db.QueryRow(`DELETE FROM link_challenges WHERE challenge_hash = `+s.p(1)+` RETURNING user_id, pubkey, expires_at`, challengeHash(raw)).Scan(&owner, &pk, &exp) if errors.Is(err, sql.ErrNoRows) { return ErrChallengeMissing @@ -71,7 +71,7 @@ func (s *Store) ConsumeLinkChallenge(userID int64, pubkey, challenge string) err // PruneLinkChallenges deletes challenges that have expired. func (s *Store) PruneLinkChallenges() (int64, error) { - res, err := s.db.Exec(`DELETE FROM link_challenges WHERE expires_at <= ?`, unix(s.now())) + res, err := s.db.Exec(`DELETE FROM link_challenges WHERE expires_at <= `+s.p(1)+``, unix(s.now())) if err != nil { return 0, err } diff --git a/internal/users/challenges_test.go b/internal/users/challenges_test.go index 8e4c9b9ce..3de67dd4b 100644 --- a/internal/users/challenges_test.go +++ b/internal/users/challenges_test.go @@ -33,7 +33,7 @@ func TestLinkChallengeLifecycle(t *testing.T) { t.Fatalf("CreateLinkChallenge = %q, %v, %v", ch, exp, err) } var n int - if err := st.db.QueryRow(`SELECT COUNT(*) FROM link_challenges WHERE challenge_hash = ? AND pubkey = ? AND user_id = ?`, + if err := st.db.QueryRow(`SELECT COUNT(*) FROM link_challenges WHERE challenge_hash = $1 AND pubkey = $2 AND user_id = $3`, challengeHash(raw), pk, a.ID).Scan(&n); err != nil || n != 1 { t.Fatalf("stored challenge rows = %d, %v (want SHA-256 of the raw bytes)", n, err) } diff --git a/internal/users/companions.go b/internal/users/companions.go index 756e2b6c2..8b050fc62 100644 --- a/internal/users/companions.go +++ b/internal/users/companions.go @@ -7,6 +7,8 @@ import ( "fmt" "strings" "time" + + "github.com/meshcore-analyzer/dbconfig" ) // NormalizePubkey returns a companion's public key as 64 lowercase hex @@ -59,8 +61,16 @@ func (s *Store) UpsertCompanionLink(userID int64, pubkey, name string) (*Compani if err != nil { return nil, 0, err } + defer tx.Rollback() + // A row lock cannot protect the first link. Serialize a pubkey's transfer + // across processes, including competing first links, before reading its owner. + if s.backend == dbconfig.Postgres { + if _, err := tx.Exec(`SELECT pg_advisory_xact_lock(hashtextextended(current_schema() || ':' || $1,7289041404))`, pk); err != nil { + return nil, 0, err + } + } var prev, linkedAt int64 - err = tx.QueryRow(`SELECT user_id, linked_at FROM companion_links WHERE pubkey = ?`, pk).Scan(&prev, &linkedAt) + err = tx.QueryRow(`SELECT user_id, linked_at FROM companion_links WHERE pubkey = `+s.p(1)+``, pk).Scan(&prev, &linkedAt) switch { case errors.Is(err, sql.ErrNoRows): prev, linkedAt = 0, unix(s.now()) @@ -70,7 +80,7 @@ func (s *Store) UpsertCompanionLink(userID int64, pubkey, name string) (*Compani case prev != userID: linkedAt = unix(s.now()) } - if _, err := tx.Exec(`INSERT INTO companion_links (pubkey, user_id, name, linked_at) VALUES (?, ?, ?, ?) + if _, err := tx.Exec(`INSERT INTO companion_links (pubkey, user_id, name, linked_at) VALUES (`+s.p(1)+`, `+s.p(2)+`, `+s.p(3)+`, `+s.p(4)+`) ON CONFLICT(pubkey) DO UPDATE SET user_id = excluded.user_id, name = excluded.name, linked_at = excluded.linked_at`, pk, userID, name, linkedAt); err != nil { tx.Rollback() @@ -87,7 +97,7 @@ func (s *Store) UpsertCompanionLink(userID int64, pubkey, name string) (*Compani // ListCompanionLinks returns a user's linked companions, newest first. func (s *Store) ListCompanionLinks(userID int64) ([]CompanionLink, error) { - rows, err := s.db.Query(`SELECT `+companionCols+` FROM companion_links WHERE user_id = ? ORDER BY linked_at DESC, pubkey`, userID) + rows, err := s.db.Query(`SELECT `+companionCols+` FROM companion_links WHERE user_id = `+s.p(1)+` ORDER BY linked_at DESC, pubkey`, userID) if err != nil { return nil, err } @@ -109,7 +119,7 @@ func (s *Store) GetCompanionLink(pubkey string) (*CompanionLink, error) { if err != nil { return nil, err } - l, err := scanCompanion(s.db.QueryRow(`SELECT `+companionCols+` FROM companion_links WHERE pubkey = ?`, pk)) + l, err := scanCompanion(s.db.QueryRow(`SELECT `+companionCols+` FROM companion_links WHERE pubkey = `+s.p(1)+``, pk)) if errors.Is(err, sql.ErrNoRows) { return nil, ErrNotFound } @@ -125,7 +135,7 @@ func (s *Store) DeleteCompanionLink(userID int64, pubkey string) error { if err != nil { return err } - return expectOne(s.db.Exec(`DELETE FROM companion_links WHERE pubkey = ? AND user_id = ?`, pk, userID)) + return expectOne(s.db.Exec(`DELETE FROM companion_links WHERE pubkey = `+s.p(1)+` AND user_id = `+s.p(2)+``, pk, userID)) } // LinkedPubkeys returns every linked pubkey, sorted. (The ingestor does not diff --git a/internal/users/companions_test.go b/internal/users/companions_test.go index 8db75134c..6e1c02d5c 100644 --- a/internal/users/companions_test.go +++ b/internal/users/companions_test.go @@ -4,10 +4,57 @@ import ( "errors" "reflect" "strings" + "sync" "testing" "time" ) +func TestCompanionFirstLinkAcrossPools(t *testing.T) { + st, _ := newTestStore(t) + other, err := Open(st.target) + if err != nil { + t.Fatal(err) + } + defer other.Close() + a := mustCreate(t, st, "first@example.org", "First") + b := mustCreate(t, st, "second@example.org", "Second") + pk := strings.Repeat("cd", 32) + start := make(chan struct{}) + results := make(chan int64, 2) + var done sync.WaitGroup + for i, store := range []*Store{st, other} { + done.Add(1) + go func(i int, store *Store) { + defer done.Done() + <-start + id := a.ID + if i == 1 { + id = b.ID + } + _, prev, err := store.UpsertCompanionLink(id, pk, "Linked") + if err != nil { + t.Error(err) + return + } + results <- prev + }(i, store) + } + close(start) + done.Wait() + close(results) + first, transfer := 0, int64(0) + for prev := range results { + if prev == 0 { + first++ + } else { + transfer = prev + } + } + if first != 1 || (transfer != a.ID && transfer != b.ID) { + t.Fatalf("first links=%d transfer predecessor=%d", first, transfer) + } +} + func TestCompanionLinkUpsertAndTransfer(t *testing.T) { st, clk := newTestStore(t) a := mustCreate(t, st, "a@example.org", "A") @@ -117,7 +164,7 @@ func TestCompanionDataCascadesOnUserDelete(t *testing.T) { t.Fatalf("LinkedPubkeys after delete = %v", all) } var n int - if err := st.db.QueryRow(`SELECT COUNT(*) FROM link_challenges WHERE user_id = ?`, a.ID).Scan(&n); err != nil || n != 0 { + if err := st.db.QueryRow(`SELECT COUNT(*) FROM link_challenges WHERE user_id = $1`, a.ID).Scan(&n); err != nil || n != 0 { t.Fatalf("challenges after user delete = %d, %v", n, err) } if _, err := st.LookupSession(raw); !errors.Is(err, ErrNotFound) { diff --git a/internal/users/export_test.go b/internal/users/export_test.go index 8da4e2b24..8dc3861af 100644 --- a/internal/users/export_test.go +++ b/internal/users/export_test.go @@ -66,7 +66,7 @@ func TestAllProposalsByUserHasNoCap(t *testing.T) { t.Fatal(err) } for i := 0; i <= ProposalListMax; i++ { - if _, err := tx.Exec(`INSERT INTO proposals (kind, subject, status, proposer_id, created_at) VALUES (?, ?, 'rejected', ?, ?)`, + if _, err := tx.Exec(`INSERT INTO proposals (kind, subject, status, proposer_id, created_at) VALUES ($1, $2, 'rejected', $3, $4)`, KindHashtagChannel, fmt.Sprintf("#p%d", i), u.ID, unix(clk.Now())); err != nil { tx.Rollback() t.Fatal(err) diff --git a/internal/users/go.mod b/internal/users/go.mod index ac4edb30b..2756be8a4 100644 --- a/internal/users/go.mod +++ b/internal/users/go.mod @@ -1,20 +1,35 @@ module github.com/meshcore-analyzer/users -go 1.22 +go 1.25.0 require ( - golang.org/x/crypto v0.31.0 + github.com/jackc/pgx/v5 v5.7.6 + golang.org/x/crypto v0.37.0 modernc.org/sqlite v1.34.5 ) require ( github.com/dustin/go-humanize v1.0.1 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/ncruces/go-strftime v0.1.9 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect - golang.org/x/sys v0.28.0 // indirect + golang.org/x/sync v0.13.0 // indirect + golang.org/x/text v0.24.0 // indirect modernc.org/libc v1.55.3 // indirect modernc.org/mathutil v1.6.0 // indirect modernc.org/memory v1.8.0 // indirect ) + +require ( + github.com/meshcore-analyzer/dbconfig v0.0.0 + github.com/meshcore-analyzer/pgutil v0.0.0 + golang.org/x/sys v0.32.0 // indirect +) + +replace github.com/meshcore-analyzer/pgutil => ../pgutil + +replace github.com/meshcore-analyzer/dbconfig => ../dbconfig diff --git a/internal/users/go.sum b/internal/users/go.sum index aa3cb2b45..5cf6e6372 100644 --- a/internal/users/go.sum +++ b/internal/users/go.sum @@ -1,24 +1,50 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo= github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.7.6 h1:rWQc5FwZSPX58r1OQmkuaNicxdmExaEz5A2DO2hUuTk= +github.com/jackc/pgx/v5 v5.7.6/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= -golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= -golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= -golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= +golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= +golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= -golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw= -golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc= +golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= +golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= +golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= modernc.org/cc/v4 v4.21.4 h1:3Be/Rdo1fpr8GrQ7IVw9OHtplU4gWbb+wNgeoBMmGLQ= modernc.org/cc/v4 v4.21.4/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ= modernc.org/ccgo/v4 v4.19.2 h1:lwQZgvboKD0jBwdaeVCTouxhxAyN6iawF3STraAal8Y= diff --git a/internal/users/helpers_test.go b/internal/users/helpers_test.go index e6efef154..67373ee06 100644 --- a/internal/users/helpers_test.go +++ b/internal/users/helpers_test.go @@ -1,6 +1,12 @@ package users import ( + "database/sql" + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/pgutil" + "github.com/meshcore-analyzer/pgutil/pgtest" + "net/url" + "os" "path/filepath" "testing" "time" @@ -11,10 +17,65 @@ type fakeClock struct{ t time.Time } func (c *fakeClock) Now() time.Time { return c.t } func (c *fakeClock) Advance(d time.Duration) { c.t = c.t.Add(d) } -// newTestStore opens a fresh users.db in a temp dir with a controllable clock. +// Ordinary tests always exercise a real SQLite database. The explicit PostgreSQL +// matrix requires its service; it never silently substitutes SQLite or skips it. +func testBackend(t *testing.T) dbconfig.Backend { + t.Helper() + switch os.Getenv("CORESCOPE_TEST_BACKEND") { + case "", "sqlite": + return dbconfig.SQLite + case "postgres": + return dbconfig.Postgres + default: + t.Fatal("CORESCOPE_TEST_BACKEND must be sqlite or postgres") + return "" + } +} +func requirePostgres(t *testing.T) { + t.Helper() + if testBackend(t) != dbconfig.Postgres { + t.Skip("PostgreSQL-specific contract; run CORESCOPE_TEST_BACKEND=postgres") + } +} +func testTarget(t *testing.T, wholeDatabase bool) string { + t.Helper() + if testBackend(t) == dbconfig.SQLite { + return filepath.Join(t.TempDir(), "accounts.db") + } + if wholeDatabase { + return pgtest.NewDatabase(t) + } + return pgtest.NewSchema(t) +} +func schemaVersion(t *testing.T) int { + if testBackend(t) == dbconfig.SQLite { + return SQLiteSchemaVersion + } + return CurrentSchemaVersion +} +func applyTestSchema(t *testing.T, db *sql.DB) error { + if testBackend(t) == dbconfig.SQLite { + return ApplySQLite(db) + } + return ApplyPostgres(db) +} + +// newTestStore bootstraps an isolated database with a controllable clock. func newTestStore(t *testing.T) (*Store, *fakeClock) { t.Helper() - st, err := Open(filepath.Join(t.TempDir(), "users.db")) + return newTestStoreAt(t, testTarget(t, false)) +} + +func newTestStoreAt(t *testing.T, dsn string) (*Store, *fakeClock) { + t.Helper() + if testBackend(t) == dbconfig.Postgres { + db := testOwner(t, dsn) + if err := Apply(db); err != nil { + t.Fatal(err) + } + db.Close() + } + st, err := Open(testRuntimeURL(t, dsn)) if err != nil { t.Fatalf("Open: %v", err) } @@ -23,3 +84,77 @@ func newTestStore(t *testing.T) (*Store, *fakeClock) { t.Cleanup(func() { st.Close() }) return st, clk } + +func testRuntimeURL(t *testing.T, ownerURL string) string { + t.Helper() + if testBackend(t) == dbconfig.SQLite { + return ownerURL + } + dsn := pgtest.Writer(t, ownerURL) + u, _ := url.Parse(dsn) + db, err := pgutil.Open(ownerURL, false) + if err != nil { + t.Fatal(err) + } + defer db.Close() + if _, err := db.Exec(`REVOKE INSERT,UPDATE,DELETE ON corescope_schema,schema_version FROM "` + u.User.Username() + `"`); err != nil { + t.Fatal(err) + } + return dsn +} + +// migratedTestStore exercises every intermediate native account layout while +// SQLite source normalization is covered by the offline importer's fixtures. +func migratedTestStore(t *testing.T, version int, seed string) *Store { + t.Helper() + dsn := testTarget(t, false) + db := testOwner(t, dsn) + defer db.Close() + if _, err := db.Exec(`CREATE TABLE schema_version(version INTEGER NOT NULL)`); err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`INSERT INTO schema_version VALUES ($1)`, version); err != nil { + t.Fatal(err) + } + history := migrations + if testBackend(t) == dbconfig.SQLite { + history = legacyMigrations + } + for _, migration := range history[:version] { + for _, stmt := range migration { + if _, err := db.Exec(stmt); err != nil { + t.Fatal(err) + } + } + } + if seed != "" { + if _, err := db.Exec(seed); err != nil { + t.Fatal(err) + } + } + if err := applyTestSchema(t, db); err != nil { + t.Fatal(err) + } + st, err := Open(testRuntimeURL(t, dsn)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { st.Close() }) + return st +} + +func testOwner(t *testing.T, dsn string) *sql.DB { + t.Helper() + var db *sql.DB + var err error + if testBackend(t) == dbconfig.SQLite { + db, err = sql.Open("sqlite", dsn) + } else { + db, err = pgutil.Open(dsn, false) + } + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { db.Close() }) + return db +} diff --git a/internal/users/legacy_schema.go b/internal/users/legacy_schema.go new file mode 100644 index 000000000..ac91332e9 --- /dev/null +++ b/internal/users/legacy_schema.go @@ -0,0 +1,149 @@ +package users + +// legacyMigrations is the original append-only SQLite upgrade history shared by +// native account upgrades and the offline importer. Keep the upstream definitions unchanged. +var legacyMigrations = [][]string{ + { // v1: sub-project A + `CREATE TABLE users ( + id INTEGER PRIMARY KEY, + email TEXT NOT NULL UNIQUE, + display_name TEXT NOT NULL, + password_hash TEXT NOT NULL, + role TEXT NOT NULL DEFAULT 'user' CHECK (role IN ('user','admin')), + status TEXT NOT NULL DEFAULT 'pending' CHECK (status IN ('pending','active','disabled')), + created_at INTEGER NOT NULL, + activated_at INTEGER, + activated_by INTEGER, + last_login_at INTEGER, + email_bouncing INTEGER NOT NULL DEFAULT 0 + )`, + `CREATE TABLE sessions ( + id INTEGER PRIMARY KEY, + token_hash TEXT NOT NULL UNIQUE, + user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE, + csrf_token TEXT NOT NULL, + created_at INTEGER NOT NULL, + expires_at INTEGER NOT NULL, + last_seen_at INTEGER NOT NULL, + user_agent TEXT NOT NULL DEFAULT '' + )`, + `CREATE INDEX sessions_user ON sessions(user_id)`, + `CREATE TABLE tokens ( + token_hash TEXT PRIMARY KEY, + user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE, + purpose TEXT NOT NULL CHECK (purpose IN ('activate','reset','email_change')), + new_email TEXT, + expires_at INTEGER NOT NULL, + used_at INTEGER + )`, + `CREATE INDEX tokens_user ON tokens(user_id, purpose)`, + `CREATE TABLE audit_log ( + id INTEGER PRIMARY KEY, + at INTEGER NOT NULL, + actor_user_id INTEGER, + action TEXT NOT NULL, + target_user_id INTEGER, + detail TEXT NOT NULL DEFAULT '{}' + )`, + `CREATE INDEX audit_target ON audit_log(target_user_id)`, + `CREATE INDEX audit_actor ON audit_log(actor_user_id)`, + `CREATE TABLE mail_log ( + id INTEGER PRIMARY KEY, + user_id INTEGER REFERENCES users(id) ON DELETE SET NULL, + to_email TEXT NOT NULL, + purpose TEXT NOT NULL, + provider_message_id TEXT, + sent_at INTEGER NOT NULL, + last_event TEXT NOT NULL DEFAULT 'sent', + last_event_at INTEGER NOT NULL, + last_reason TEXT NOT NULL DEFAULT '' + )`, + `CREATE INDEX mail_user ON mail_log(user_id)`, + `CREATE UNIQUE INDEX mail_msgid ON mail_log(provider_message_id) WHERE provider_message_id IS NOT NULL`, + `CREATE TABLE mail_events ( + mail_id INTEGER NOT NULL REFERENCES mail_log(id) ON DELETE CASCADE, + event TEXT NOT NULL, + at INTEGER NOT NULL, + reason TEXT NOT NULL DEFAULT '', + UNIQUE (mail_id, event, at) + )`, + }, + { // v2: sub-project B, settings sync (one document per user) + `CREATE TABLE user_settings ( + user_id INTEGER PRIMARY KEY REFERENCES users(id) ON DELETE CASCADE, + doc TEXT NOT NULL, + revision INTEGER NOT NULL, + generation TEXT NOT NULL, + updated_at INTEGER NOT NULL + )`, + }, + { // v3: sub-project C, global audit list ordered and filtered by time + `CREATE INDEX audit_at ON audit_log(at)`, + }, + { // v4: sub-project D, proposals (one generic table; this version ships the kind 'hashtag_channel') + `CREATE TABLE proposals ( + id INTEGER PRIMARY KEY, + kind TEXT NOT NULL, + subject TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('pending','approved','rejected','revoked')), + proposer_id INTEGER REFERENCES users(id) ON DELETE SET NULL, + reviewer_id INTEGER REFERENCES users(id) ON DELETE SET NULL, + note TEXT NOT NULL DEFAULT '', + created_at INTEGER NOT NULL, + decided_at INTEGER + )`, + `CREATE UNIQUE INDEX proposals_kind_subject ON proposals(kind, subject)`, + `CREATE INDEX proposals_status ON proposals(status)`, + }, + { // v5: sub-project E, node notifications + `CREATE TABLE notification_prefs ( + user_id INTEGER PRIMARY KEY REFERENCES users(id) ON DELETE CASCADE, + enabled INTEGER NOT NULL DEFAULT 1, + events TEXT NOT NULL DEFAULT '', + unsub_token TEXT NOT NULL UNIQUE, + updated_at INTEGER NOT NULL + )`, + `CREATE TABLE notification_watches ( + user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE, + pubkey TEXT NOT NULL, + created_at INTEGER NOT NULL, + PRIMARY KEY (user_id, pubkey) + )`, + `CREATE TABLE notification_state ( + user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE, + event TEXT NOT NULL, + subject TEXT NOT NULL, + state TEXT NOT NULL, + changed_at INTEGER NOT NULL, + PRIMARY KEY (user_id, event, subject) + )`, + }, + { // v6: sub-project F, companion linking (device sessions, linked companions) + `ALTER TABLE sessions ADD COLUMN kind TEXT NOT NULL DEFAULT 'web' + CHECK (kind IN ('web','device'))`, + `ALTER TABLE sessions ADD COLUMN label TEXT NOT NULL DEFAULT ''`, + `ALTER TABLE sessions ADD COLUMN scopes TEXT NOT NULL DEFAULT ''`, + `CREATE TABLE companion_links ( + pubkey TEXT PRIMARY KEY, + user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE, + name TEXT NOT NULL DEFAULT '', + linked_at INTEGER NOT NULL + )`, + `CREATE INDEX companion_links_user ON companion_links(user_id)`, + `CREATE TABLE link_challenges ( + challenge_hash TEXT PRIMARY KEY, + user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE, + pubkey TEXT NOT NULL, + expires_at INTEGER NOT NULL + )`, + }, +} + +// LegacyMigrations returns a copy of the append-only SQLite migrations. +func LegacyMigrations() [][]string { + out := make([][]string, len(legacyMigrations)) + for i, m := range legacyMigrations { + out[i] = append([]string(nil), m...) + } + return out +} diff --git a/internal/users/mail.go b/internal/users/mail.go index 3bb2ca8b4..65a272f07 100644 --- a/internal/users/mail.go +++ b/internal/users/mail.go @@ -56,12 +56,13 @@ func (s *Store) LogMail(userID *int64, to, purpose, messageID string) (int64, er if messageID != "" { mid = messageID } - res, err := s.db.Exec(`INSERT INTO mail_log (user_id, to_email, purpose, provider_message_id, sent_at, last_event, last_event_at) - VALUES (?, ?, ?, ?, ?, 'sent', ?)`, nullInt(userID), to, purpose, mid, now, now) + var id int64 + err := s.db.QueryRow(`INSERT INTO mail_log (user_id, to_email, purpose, provider_message_id, sent_at, last_event, last_event_at) + VALUES (`+s.p(1)+`, `+s.p(2)+`, `+s.p(3)+`, `+s.p(4)+`, `+s.p(5)+`, 'sent', `+s.p(6)+`) RETURNING id`, nullInt(userID), to, purpose, mid, now, now).Scan(&id) if err != nil { return 0, err } - return res.LastInsertId() + return id, nil } // RecordMailEvent appends a provider event to the mail with that provider @@ -75,7 +76,7 @@ func (s *Store) RecordMailEvent(messageID, event string, at time.Time, reason st defer tx.Rollback() var id, lastAt int64 var uid sql.NullInt64 - err = tx.QueryRow(`SELECT id, user_id, last_event_at FROM mail_log WHERE provider_message_id = ?`, messageID). + err = tx.QueryRow(`SELECT id, user_id, last_event_at FROM mail_log WHERE provider_message_id = `+s.p(1)+s.forUpdate(), messageID). Scan(&id, &uid, &lastAt) if errors.Is(err, sql.ErrNoRows) { return nil, false, nil @@ -83,12 +84,12 @@ func (s *Store) RecordMailEvent(messageID, event string, at time.Time, reason st if err != nil { return nil, false, err } - if _, err := tx.Exec(`INSERT OR IGNORE INTO mail_events (mail_id, event, at, reason) VALUES (?, ?, ?, ?)`, + if _, err := tx.Exec(`INSERT INTO mail_events (mail_id, event, at, reason) VALUES (`+s.p(1)+`, `+s.p(2)+`, `+s.p(3)+`, `+s.p(4)+`) ON CONFLICT (mail_id,event,at) DO NOTHING`, id, event, unix(at), reason); err != nil { return nil, false, err } if unix(at) >= lastAt { - if _, err := tx.Exec(`UPDATE mail_log SET last_event = ?, last_event_at = ?, last_reason = ? WHERE id = ?`, + if _, err := tx.Exec(`UPDATE mail_log SET last_event = `+s.p(1)+`, last_event_at = `+s.p(2)+`, last_reason = `+s.p(3)+` WHERE id = `+s.p(4), event, unix(at), reason, id); err != nil { return nil, false, err } @@ -104,7 +105,7 @@ func (s *Store) RecordMailEvent(messageID, event string, at time.Time, reason st } func (s *Store) eventsFor(mailID int64) ([]MailEvent, error) { - rows, err := s.db.Query(`SELECT event, at, reason FROM mail_events WHERE mail_id = ? ORDER BY at, rowid`, mailID) + rows, err := s.db.Query(`SELECT event, at, reason FROM mail_events WHERE mail_id = `+s.p(1)+` ORDER BY at, id`, mailID) if err != nil { return nil, err } @@ -124,7 +125,7 @@ func (s *Store) eventsFor(mailID int64) ([]MailEvent, error) { // MailByID returns one record with its events. func (s *Store) MailByID(id int64) (*MailRecord, error) { - m, err := scanMail(s.db.QueryRow(`SELECT `+mailCols+` FROM mail_log WHERE id = ?`, id)) + m, err := scanMail(s.db.QueryRow(`SELECT `+mailCols+` FROM mail_log WHERE id = `+s.p(1), id)) if err != nil { return nil, err } @@ -145,11 +146,11 @@ func (s *Store) MailForUser(userID int64, limit int) ([]MailRecord, error) { // MailAllForUser is MailForUser without a cap (the account export). func (s *Store) MailAllForUser(userID int64) ([]MailRecord, error) { - return s.mailForUser(userID, noLimit) + return s.mailForUser(userID, s.noLimit()) } -func (s *Store) mailForUser(userID int64, limit int) ([]MailRecord, error) { - rows, err := s.db.Query(`SELECT `+mailCols+` FROM mail_log WHERE user_id = ? ORDER BY sent_at DESC, id DESC LIMIT ?`, userID, limit) +func (s *Store) mailForUser(userID int64, limit any) ([]MailRecord, error) { + rows, err := s.db.Query(`SELECT `+mailCols+` FROM mail_log WHERE user_id = `+s.p(1)+` ORDER BY sent_at DESC, id DESC LIMIT `+s.p(2), userID, limit) if err != nil { return nil, err } @@ -195,7 +196,7 @@ func (s *Store) LatestMailByUser() (map[int64]MailRecord, error) { // PruneMail deletes mail records (and their events) older than maxAge. func (s *Store) PruneMail(maxAge time.Duration) (int64, error) { - res, err := s.db.Exec(`DELETE FROM mail_log WHERE sent_at < ?`, unix(s.now())-int64(maxAge/time.Second)) + res, err := s.db.Exec(`DELETE FROM mail_log WHERE sent_at < `+s.p(1), unix(s.now())-int64(maxAge/time.Second)) if err != nil { return 0, err } diff --git a/internal/users/notify.go b/internal/users/notify.go index a646e0b49..1ce98e2f3 100644 --- a/internal/users/notify.go +++ b/internal/users/notify.go @@ -3,6 +3,8 @@ package users import ( "database/sql" "errors" + "github.com/meshcore-analyzer/dbconfig" + "sort" "strings" "time" ) @@ -116,7 +118,17 @@ func canonicalEvents(list []string) []string { return out } -func placeholders(n int) string { return strings.TrimSuffix(strings.Repeat("?,", n), ",") } +func (s *Store) placeholders(n int, offset ...int) string { + start := 1 + if len(offset) > 0 { + start += offset[0] + } + out := make([]string, n) + for i := range out { + out[i] = s.p(start + i) + } + return strings.Join(out, ",") +} const prefsCols = `user_id, enabled, events, unsub_token, updated_at` @@ -138,7 +150,7 @@ func scanPrefs(row rowScanner) (*NotifyPrefs, error) { } func (s *Store) getPrefs(userID int64) (*NotifyPrefs, error) { - return scanPrefs(s.db.QueryRow(`SELECT `+prefsCols+` FROM notification_prefs WHERE user_id = ?`, userID)) + return scanPrefs(s.db.QueryRow(`SELECT `+prefsCols+` FROM notification_prefs WHERE user_id = `+s.p(1), userID)) } // NotifyPrefsFor returns the user's preferences, creating the default row @@ -155,8 +167,8 @@ func (s *Store) NotifyPrefsFor(userID int64) (NotifyPrefs, error) { return NotifyPrefs{}, err } d := DefaultNotifyPrefs(userID) - if _, err := s.db.Exec(`INSERT OR IGNORE INTO notification_prefs (user_id, enabled, events, unsub_token, updated_at) - VALUES (?, ?, ?, ?, ?)`, userID, d.Enabled, strings.Join(d.Events, ","), tok, unix(s.now())); err != nil { + if _, err := s.db.Exec(`INSERT INTO notification_prefs (user_id, enabled, events, unsub_token, updated_at) + VALUES (`+s.p(1)+`, `+s.p(2)+`, `+s.p(3)+`, `+s.p(4)+`, `+s.p(5)+`) ON CONFLICT(user_id) DO NOTHING`, userID, 1, strings.Join(d.Events, ","), tok, unix(s.now())); err != nil { return NotifyPrefs{}, err } p, err := s.getPrefs(userID) @@ -194,14 +206,14 @@ func (s *Store) SetNotifyPrefs(userID int64, enabled bool, events []string) (Not return NotifyPrefs{}, err } defer tx.Rollback() - if _, err := tx.Exec(`UPDATE notification_prefs SET enabled = ?, events = ?, updated_at = ? WHERE user_id = ?`, + if _, err := tx.Exec(`UPDATE notification_prefs SET enabled = `+s.p(1)+`, events = `+s.p(2)+`, updated_at = `+s.p(3)+` WHERE user_id = `+s.p(4), en, strings.Join(ev, ","), unix(s.now()), userID); err != nil { return NotifyPrefs{}, err } - q := `DELETE FROM notification_state WHERE user_id = ?` + q := `DELETE FROM notification_state WHERE user_id = ` + s.p(1) args := []any{userID} if len(ev) > 0 { - q += ` AND event NOT IN (` + placeholders(len(ev)) + `)` + q += ` AND event NOT IN (` + s.placeholders(len(ev), 1) + `)` for _, e := range ev { args = append(args, e) } @@ -222,8 +234,13 @@ func (s *Store) DisableNotifyByToken(token string) (userID int64, wasEnabled boo if token == "" { return 0, false, ErrTokenInvalid } + tx, err := s.db.Begin() + if err != nil { + return 0, false, err + } + defer tx.Rollback() var en int - err = s.db.QueryRow(`SELECT user_id, enabled FROM notification_prefs WHERE unsub_token = ?`, token).Scan(&userID, &en) + err = tx.QueryRow(`SELECT user_id,enabled FROM notification_prefs WHERE unsub_token=`+s.p(1)+s.forUpdate(), token).Scan(&userID, &en) if errors.Is(err, sql.ErrNoRows) { return 0, false, ErrTokenInvalid } @@ -231,11 +248,11 @@ func (s *Store) DisableNotifyByToken(token string) (userID int64, wasEnabled boo return 0, false, err } if en != 0 { - if _, err := s.db.Exec(`UPDATE notification_prefs SET enabled = 0, updated_at = ? WHERE user_id = ?`, unix(s.now()), userID); err != nil { + if _, err = tx.Exec(`UPDATE notification_prefs SET enabled=0,updated_at=`+s.p(1)+` WHERE user_id=`+s.p(2), unix(s.now()), userID); err != nil { return 0, false, err } } - return userID, en != 0, nil + return userID, en != 0, tx.Commit() } // AllNotifyPrefs returns every preferences row, by user id. @@ -266,8 +283,11 @@ func (s *Store) AddWatches(userID int64, pubkeys []string, max int) (added, alre return 0, 0, 0, err } defer tx.Rollback() + if err := s.lockUser(tx, userID); err != nil { + return 0, 0, 0, err + } have := map[string]bool{} - rows, err := tx.Query(`SELECT pubkey FROM notification_watches WHERE user_id = ?`, userID) + rows, err := tx.Query(`SELECT pubkey FROM notification_watches WHERE user_id = `+s.p(1), userID) if err != nil { return 0, 0, 0, err } @@ -291,7 +311,7 @@ func (s *Store) AddWatches(userID int64, pubkeys []string, max int) (added, alre case max > 0 && len(have) >= max: overLimit++ default: - if _, err := tx.Exec(`INSERT INTO notification_watches (user_id, pubkey, created_at) VALUES (?, ?, ?)`, userID, pk, now); err != nil { + if _, err := tx.Exec(`INSERT INTO notification_watches (user_id, pubkey, created_at) VALUES (`+s.p(1)+`, `+s.p(2)+`, `+s.p(3)+`)`, userID, pk, now); err != nil { return 0, 0, 0, err } have[pk] = true @@ -326,14 +346,19 @@ func (s *Store) RemoveWatch(userID int64, pubkey string) error { return err } defer tx.Rollback() - if _, err := tx.Exec(`DELETE FROM notification_watches WHERE user_id = ? AND pubkey = ?`, userID, pubkey); err != nil { + if err := s.lockUser(tx, userID); errors.Is(err, ErrNotFound) { + return nil + } else if err != nil { return err } - if _, err := tx.Exec(`DELETE FROM notification_state WHERE user_id = ? AND subject = ? AND event IN (?, ?)`, + if _, err := tx.Exec(`DELETE FROM notification_watches WHERE user_id = `+s.p(1)+` AND pubkey = `+s.p(2), userID, pubkey); err != nil { + return err + } + if _, err := tx.Exec(`DELETE FROM notification_state WHERE user_id = `+s.p(1)+` AND subject = `+s.p(2)+` AND event IN (`+s.p(3)+`, `+s.p(4)+`)`, userID, pubkey, NotifyNodeOffline, NotifyNodeBattery); err != nil { return err } - if _, err := tx.Exec(`DELETE FROM notification_state WHERE user_id = ? AND event = ? AND substr(subject, 1, ?) = ?`, + if _, err := tx.Exec(`DELETE FROM notification_state WHERE user_id = `+s.p(1)+` AND event = `+s.p(2)+` AND substr(subject, 1, `+s.p(3)+`) = `+s.p(4)+``, userID, NotifyNodeExternal, len(pubkey)+1, pubkey+"/"); err != nil { return err } @@ -361,7 +386,7 @@ func (s *Store) queryWatches(q string, args ...any) ([]NotifyWatch, error) { // WatchesFor returns the user's watches, oldest first. func (s *Store) WatchesFor(userID int64) ([]NotifyWatch, error) { - return s.queryWatches(`SELECT user_id, pubkey, created_at FROM notification_watches WHERE user_id = ? ORDER BY created_at, pubkey`, userID) + return s.queryWatches(`SELECT user_id, pubkey, created_at FROM notification_watches WHERE user_id = `+s.p(1)+` ORDER BY created_at, pubkey`, userID) } // AllWatches returns every watch, by user and pubkey. @@ -398,15 +423,22 @@ func (s *Store) WriteNotifyStates(list []NotifyState) error { return err } defer tx.Rollback() + lock := "" + if s.backend == dbconfig.Postgres { + lock = " FOR KEY SHARE" + } stmt, err := tx.Prepare(`INSERT INTO notification_state (user_id, event, subject, state, changed_at) - SELECT ?, ?, ?, ?, ? WHERE EXISTS (SELECT 1 FROM users WHERE id = ?) + SELECT id, ` + s.p(2) + `, ` + s.p(3) + `, ` + s.p(4) + `, ` + s.p(5) + ` FROM users WHERE id=` + s.p(1) + lock + ` ON CONFLICT (user_id, event, subject) DO UPDATE SET state = excluded.state, changed_at = excluded.changed_at`) if err != nil { return err } defer stmt.Close() - for _, st := range list { - if _, err := stmt.Exec(st.UserID, st.Event, st.Subject, st.State, unix(st.ChangedAt), st.UserID); err != nil { + // Match the janitor's ascending account lock order to avoid deadlocks. + ordered := append([]NotifyState(nil), list...) + sort.SliceStable(ordered, func(i, j int) bool { return ordered[i].UserID < ordered[j].UserID }) + for _, st := range ordered { + if _, err := stmt.Exec(st.UserID, st.Event, st.Subject, st.State, unix(st.ChangedAt)); err != nil { return err } } @@ -424,7 +456,7 @@ func (s *Store) DeleteNotifyStates(keys []NotifyKey) error { return err } defer tx.Rollback() - stmt, err := tx.Prepare(`DELETE FROM notification_state WHERE user_id = ? AND event = ? AND subject = ?`) + stmt, err := tx.Prepare(`DELETE FROM notification_state WHERE user_id = ` + s.p(1) + ` AND event = ` + s.p(2) + ` AND subject = ` + s.p(3)) if err != nil { return err } @@ -441,7 +473,7 @@ func (s *Store) DeleteNotifyStates(keys []NotifyKey) error { // total (deleted accounts included) and per existing user. func (s *Store) NotifyMailCounts(since time.Time) (total int, perUser map[int64]int, err error) { perUser = map[int64]int{} - rows, err := s.db.Query(`SELECT user_id, COUNT(*) FROM mail_log WHERE purpose = ? AND sent_at >= ? GROUP BY user_id`, + rows, err := s.db.Query(`SELECT user_id, COUNT(*) FROM mail_log WHERE purpose = `+s.p(1)+` AND sent_at >= `+s.p(2)+` GROUP BY user_id`, NotifyMailPurpose, unix(since)) if err != nil { return 0, nil, err diff --git a/internal/users/notify_test.go b/internal/users/notify_test.go index 70846eb44..5315995a6 100644 --- a/internal/users/notify_test.go +++ b/internal/users/notify_test.go @@ -1,9 +1,7 @@ package users import ( - "database/sql" "errors" - "path/filepath" "reflect" "strings" "testing" @@ -42,32 +40,11 @@ func statesByKey(t *testing.T, st *Store) map[NotifyKey]NotifyState { return out } -// A users.db written by a v4 binary gains the notification tables and keeps its rows. +// A native account schema at v4 binary gains the notification tables and keeps its rows. func TestMigrateV4DatabaseToV5(t *testing.T) { - path := filepath.Join(t.TempDir(), "users.db") - db, err := sql.Open("sqlite", path) - if err != nil { - t.Fatal(err) - } - stmts := []string{`CREATE TABLE schema_version (version INTEGER NOT NULL)`, `INSERT INTO schema_version (version) VALUES (4)`} - for _, m := range migrations[:4] { - stmts = append(stmts, m...) - } - stmts = append(stmts, `INSERT INTO users (email, display_name, password_hash, created_at) VALUES ('old@example.org', 'Old', 'x', 1)`) - for _, s := range stmts { - if _, err := db.Exec(s); err != nil { - t.Fatalf("build v4 db: %v", err) - } - } - db.Close() - - st, err := Open(path) - if err != nil { - t.Fatalf("Open v4 db: %v", err) - } - defer st.Close() - if v, err := st.SchemaVersion(); err != nil || v != len(migrations) { - t.Fatalf("SchemaVersion = %d, %v; want %d", v, err, len(migrations)) + st := migratedTestStore(t, 4, `INSERT INTO users (email,display_name,password_hash,created_at) VALUES ('old@example.org','Old','x',1)`) + if v, err := st.SchemaVersion(); err != nil || v != schemaVersion(t) { + t.Fatalf("SchemaVersion = %d, %v; want %d", v, err, schemaVersion(t)) } if len(migrations) < 5 { t.Fatal("migration v5 missing") diff --git a/internal/users/onetime.go b/internal/users/onetime.go index 89d3556ba..2e0f06c22 100644 --- a/internal/users/onetime.go +++ b/internal/users/onetime.go @@ -29,7 +29,10 @@ func (s *Store) IssueToken(userID int64, p Purpose, ttl time.Duration, newEmail return "", err } defer tx.Rollback() - if _, err := tx.Exec(`UPDATE tokens SET used_at = ? WHERE user_id = ? AND purpose = ? AND used_at IS NULL`, + if err := s.lockUser(tx, userID); err != nil { + return "", err + } + if _, err := tx.Exec(`UPDATE tokens SET used_at = `+s.p(1)+` WHERE user_id = `+s.p(2)+` AND purpose = `+s.p(3)+` AND used_at IS NULL`, unix(now), userID, string(p)); err != nil { return "", err } @@ -37,7 +40,7 @@ func (s *Store) IssueToken(userID int64, p Purpose, ttl time.Duration, newEmail if newEmail != "" { ne = newEmail } - if _, err := tx.Exec(`INSERT INTO tokens (token_hash, user_id, purpose, new_email, expires_at) VALUES (?, ?, ?, ?, ?)`, + if _, err := tx.Exec(`INSERT INTO tokens (token_hash, user_id, purpose, new_email, expires_at) VALUES (`+s.p(1)+`, `+s.p(2)+`, `+s.p(3)+`, `+s.p(4)+`, `+s.p(5)+`)`, hash, userID, string(p), ne, unix(now.Add(ttl))); err != nil { return "", err } @@ -56,7 +59,7 @@ func (s *Store) checkToken(q tokenQuerier, hash string, p Purpose) (userID int64 var expires int64 var ne sql.NullString var used sql.NullInt64 - err = q.QueryRow(`SELECT user_id, purpose, new_email, expires_at, used_at FROM tokens WHERE token_hash = ?`, hash). + err = q.QueryRow(`SELECT user_id, purpose, new_email, expires_at, used_at FROM tokens WHERE token_hash = `+s.p(1), hash). Scan(&userID, &purpose, &ne, &expires, &used) if errors.Is(err, sql.ErrNoRows) { return 0, "", ErrTokenInvalid @@ -93,7 +96,14 @@ func (s *Store) ConsumeToken(raw string, p Purpose) (userID int64, newEmail stri if err != nil { return 0, "", err } - if err := expectOne(tx.Exec(`UPDATE tokens SET used_at = ? WHERE token_hash = ? AND used_at IS NULL`, unix(s.now()), hash)); err != nil { + if err := s.lockUser(tx, userID); err != nil { + return 0, "", ErrTokenInvalid + } + userID, newEmail, err = s.checkToken(tx, hash, p) + if err != nil { + return 0, "", err + } + if err := expectOne(tx.Exec(`UPDATE tokens SET used_at = `+s.p(1)+` WHERE token_hash = `+s.p(2)+` AND used_at IS NULL`, unix(s.now()), hash)); err != nil { if errors.Is(err, ErrNotFound) { return 0, "", ErrTokenInvalid } @@ -117,6 +127,9 @@ func (s *Store) ActivateWithToken(raw string, id int64, role Role, verifiedHash return err } defer tx.Rollback() + if err := s.lockUser(tx, id); err != nil { + return ErrAccountChanged + } uid, _, err := s.checkToken(tx, hash, PurposeActivate) if err != nil { return err @@ -125,14 +138,14 @@ func (s *Store) ActivateWithToken(raw string, id int64, role Role, verifiedHash return ErrTokenInvalid } now := unix(s.now()) - if err := expectOne(tx.Exec(`UPDATE tokens SET used_at = ? WHERE token_hash = ? AND used_at IS NULL`, now, hash)); err != nil { + if err := expectOne(tx.Exec(`UPDATE tokens SET used_at = `+s.p(1)+` WHERE token_hash = `+s.p(2)+` AND used_at IS NULL`, now, hash)); err != nil { if errors.Is(err, ErrNotFound) { return ErrTokenInvalid } return err } - err = expectOne(tx.Exec(`UPDATE users SET status = 'active', role = ?, activated_at = ?, activated_by = NULL - WHERE id = ? AND status = 'pending' AND password_hash = ?`, string(role), now, id, verifiedHash)) + err = expectOne(tx.Exec(`UPDATE users SET status = 'active', role = `+s.p(1)+`, activated_at = `+s.p(2)+`, activated_by = NULL + WHERE id = `+s.p(3)+` AND status = 'pending' AND password_hash = `+s.p(4), string(role), now, id, verifiedHash)) if errors.Is(err, ErrNotFound) { return ErrAccountChanged } @@ -144,9 +157,20 @@ func (s *Store) ActivateWithToken(raw string, id int64, role Role, verifiedHash // InvalidateTokens burns all of a user's unused tokens for purpose p. func (s *Store) InvalidateTokens(userID int64, p Purpose) error { - _, err := s.db.Exec(`UPDATE tokens SET used_at = ? WHERE user_id = ? AND purpose = ? AND used_at IS NULL`, - unix(s.now()), userID, string(p)) - return err + tx, err := s.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + if err := s.lockUser(tx, userID); errors.Is(err, ErrNotFound) { + return nil + } else if err != nil { + return err + } + if _, err = tx.Exec(`UPDATE tokens SET used_at=`+s.p(1)+` WHERE user_id=`+s.p(2)+` AND purpose=`+s.p(3)+` AND used_at IS NULL`, unix(s.now()), userID, string(p)); err != nil { + return err + } + return tx.Commit() } // PendingEmailChange returns the new address of the user's unused, @@ -155,7 +179,7 @@ func (s *Store) InvalidateTokens(userID int64, p Purpose) error { func (s *Store) PendingEmailChange(userID int64) (string, error) { var ne sql.NullString err := s.db.QueryRow(`SELECT new_email FROM tokens - WHERE user_id = ? AND purpose = ? AND used_at IS NULL AND expires_at > ? + WHERE user_id = `+s.p(1)+` AND purpose = `+s.p(2)+` AND used_at IS NULL AND expires_at > `+s.p(3)+` ORDER BY expires_at DESC LIMIT 1`, userID, string(PurposeEmailChange), unix(s.now())).Scan(&ne) if errors.Is(err, sql.ErrNoRows) { return "", nil @@ -165,7 +189,7 @@ func (s *Store) PendingEmailChange(userID int64) (string, error) { // PruneTokens deletes tokens that expired more than keep ago. func (s *Store) PruneTokens(keep time.Duration) (int64, error) { - res, err := s.db.Exec(`DELETE FROM tokens WHERE expires_at < ?`, unix(s.now())-int64(keep/time.Second)) + res, err := s.db.Exec(`DELETE FROM tokens WHERE expires_at < `+s.p(1), unix(s.now())-int64(keep/time.Second)) if err != nil { return 0, err } diff --git a/internal/users/pending_registration_test.go b/internal/users/pending_registration_test.go index 41294f8c0..18cc80bf7 100644 --- a/internal/users/pending_registration_test.go +++ b/internal/users/pending_registration_test.go @@ -1,7 +1,6 @@ package users import ( - "database/sql" "errors" "testing" "time" @@ -61,20 +60,14 @@ func TestUpdatePendingRegistrationRejectsChangedAccount(t *testing.T) { } } -// SQLite's writer lock holds activation uncommitted while registration starts. +// A separate writer holds activation uncommitted while registration starts. // Observe the store's only connection in use, then commit the winning activation. // The UPDATE must test the newly committed status rather than the stale snapshot. func TestUpdatePendingRegistrationCannotOverwriteConcurrentActivation(t *testing.T) { - st, _ := newTestStore(t) + target := testTarget(t, false) + st, _ := newTestStoreAt(t, target) u := mustCreate(t, st, "activation@example.org", "Original") - var file string - if err := st.db.QueryRow(`PRAGMA database_list`).Scan(new(int), new(string), &file); err != nil { - t.Fatal(err) - } - writer, err := sql.Open("sqlite", file) - if err != nil { - t.Fatal(err) - } + writer := testOwner(t, target) defer writer.Close() tx, err := writer.Begin() if err != nil { @@ -82,7 +75,7 @@ func TestUpdatePendingRegistrationCannotOverwriteConcurrentActivation(t *testing } defer tx.Rollback() activated := time.Now().Unix() - if _, err := tx.Exec(`UPDATE users SET status='active', role='admin', activated_at=? WHERE id=?`, activated, u.ID); err != nil { + if _, err := tx.Exec(`UPDATE users SET status='active', role='admin', activated_at=`+st.p(1)+` WHERE id=`+st.p(2), activated, u.ID); err != nil { t.Fatal(err) } result := make(chan error, 1) diff --git a/internal/users/postgres_test.go b/internal/users/postgres_test.go new file mode 100644 index 000000000..82c6499ac --- /dev/null +++ b/internal/users/postgres_test.go @@ -0,0 +1,398 @@ +package users + +import ( + "errors" + "github.com/meshcore-analyzer/dbconfig" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/meshcore-analyzer/pgutil" + "github.com/meshcore-analyzer/pgutil/pgtest" +) + +func TestOpenRejectsCredentialsInErrors(t *testing.T) { + _, err := Open("postgresql://private-user:private-password@localhost:invalid/accounts") + if err == nil { + t.Fatal("invalid database URL accepted") + } + for _, secret := range []string{"private-user", "private-password"} { + if strings.Contains(err.Error(), secret) { + t.Errorf("connection error exposed credentials") + } + } +} + +func TestOpenRefusesSchemaOwner(t *testing.T) { + requirePostgres(t) + dsn := pgtest.NewSchema(t) + db, err := pgutil.Open(dsn, false) + if err != nil { + t.Fatal(err) + } + defer db.Close() + if err := Apply(db); err != nil { + t.Fatal(err) + } + st, err := Open(dsn) + if st != nil { + st.Close() + } + if err == nil { + t.Fatal("runtime account connection accepted the migration owner") + } +} + +func TestSettingsCASAcrossPools(t *testing.T) { + st, _ := newTestStore(t) + other, err := Open(st.target) + if err != nil { + t.Fatal(err) + } + defer other.Close() + u := mustCreate(t, st, "cas@example.org", "CAS") + start := make(chan struct{}) + var done sync.WaitGroup + var won atomic.Int32 + for i := 0; i < 20; i++ { + done.Add(1) + go func(i int) { + defer done.Done() + <-start + s := st + if i%2 == 1 { + s = other + } + _, err := s.PutSettings(u.ID, SettingsVersion{}, `{"device":true}`) + if err == nil { + won.Add(1) + } else if !errors.Is(err, ErrSettingsConflict) { + t.Errorf("settings: %v", err) + } + }(i) + } + close(start) + done.Wait() + if got := won.Load(); got != 1 { + t.Fatalf("initial settings writers succeeded=%d; want 1", got) + } +} + +func TestProposalQuotaAcrossPools(t *testing.T) { + st, _ := newTestStore(t) + other, err := Open(st.target) + if err != nil { + t.Fatal(err) + } + defer other.Close() + u := mustCreate(t, st, "quota@example.org", "Quota") + start := make(chan struct{}) + var done sync.WaitGroup + var won atomic.Int32 + for i := 0; i < 20; i++ { + done.Add(1) + go func(i int) { + defer done.Done() + <-start + s := st + if i%2 == 1 { + s = other + } + _, err := s.Propose(KindHashtagChannel, strings.Repeat("x", i+1), u.ID, ProposalLimits{MaxPending: 1}) + if err == nil { + won.Add(1) + } else if !errors.Is(err, ErrProposalPendingLimit) { + t.Errorf("proposal: %v", err) + } + }(i) + } + close(start) + done.Wait() + if got := won.Load(); got != 1 { + t.Fatalf("pending proposals accepted=%d; want 1", got) + } +} + +func TestOpenRequiresBootstrap(t *testing.T) { + requirePostgres(t) + dsn := pgtest.NewSchema(t) + st, err := Open(dsn) + if st != nil { + st.Close() + } + if err == nil || !strings.Contains(err.Error(), "bootstrap") { + t.Fatal("runtime Open must report that the PostgreSQL schema needs bootstrap") + } +} + +func TestSingleUseAndLatestTokenAcrossPools(t *testing.T) { + st, _ := newTestStore(t) + other, err := Open(st.target) + if err != nil { + t.Fatal(err) + } + defer other.Close() + other.SetClock(st.now) + u := mustCreate(t, st, "token-race@example.org", "Token") + raw, err := st.IssueToken(u.ID, PurposeReset, time.Hour, "") + if err != nil { + t.Fatal(err) + } + var won atomic.Int32 + raceStores(st, other, func(s *Store, i int) { + _, _, err := s.ConsumeToken(raw, PurposeReset) + if err == nil { + won.Add(1) + } else if !errors.Is(err, ErrTokenInvalid) { + t.Errorf("consume: %v", err) + } + }) + if won.Load() != 1 { + t.Fatalf("token consumed %d times", won.Load()) + } + tokens := make([]string, 20) + raceStores(st, other, func(s *Store, i int) { + var err error + tokens[i], err = s.IssueToken(u.ID, PurposeReset, time.Hour, "") + if err != nil { + t.Errorf("issue: %v", err) + } + }) + var valid int + for _, raw := range tokens { + if _, err := st.TokenUser(raw, PurposeReset); err == nil { + valid++ + } else if !errors.Is(err, ErrTokenInvalid) { + t.Fatal(err) + } + } + if valid != 1 { + t.Fatalf("live reset tokens=%d; want 1", valid) + } +} + +func TestWatchAndApprovalQuotasAcrossPools(t *testing.T) { + st, _ := newTestStore(t) + other, err := Open(st.target) + if err != nil { + t.Fatal(err) + } + defer other.Close() + other.SetClock(st.now) + u := mustCreate(t, st, "watch-race@example.org", "Watch") + var watched atomic.Int32 + raceStores(st, other, func(s *Store, i int) { + err := s.AddWatch(u.ID, strings.Repeat("a", i+1), 1) + if err == nil { + watched.Add(1) + } else if !errors.Is(err, ErrWatchLimit) { + t.Errorf("watch: %v", err) + } + }) + if watched.Load() != 1 { + t.Fatalf("watches=%d; want 1", watched.Load()) + } + var proposals []*Proposal + for i := 0; i < 20; i++ { + p, err := st.Propose(KindHashtagChannel, strings.Repeat("x", i+1), u.ID, ProposalLimits{}) + if err != nil { + t.Fatal(err) + } + proposals = append(proposals, p) + } + var approved atomic.Int32 + raceStores(st, other, func(s *Store, i int) { + _, err := s.Decide(proposals[i].ID, ProposalApprove, u.ID, "", 1) + if err == nil { + approved.Add(1) + } else if !errors.Is(err, ErrProposalApprovedLimit) { + t.Errorf("approve: %v", err) + } + }) + if approved.Load() != 1 { + t.Fatalf("approved=%d; want 1", approved.Load()) + } +} + +func raceStores(first, second *Store, fn func(*Store, int)) { + start := make(chan struct{}) + var wg sync.WaitGroup + for i := 0; i < 20; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + <-start + s := first + if i%2 == 1 { + s = second + } + fn(s, i) + }(i) + } + close(start) + wg.Wait() +} + +func TestApplyForImportCannotAppearReady(t *testing.T) { + requirePostgres(t) + dsn := pgtest.NewSchema(t) + db := testOwner(t, dsn) + if err := ApplyForImport(db); err != nil { + t.Fatal(err) + } + if err := AssertReady(db); err == nil { + t.Fatal("unverified import is runtime-ready") + } + if err := Apply(db); err == nil { + t.Fatal("normal bootstrap bypassed import verification") + } + if err := ApplyForImport(db); err == nil { + t.Fatal("restarted bootstrap accepted partial import") + } + if _, err := db.Exec(`UPDATE corescope_schema SET ready=true WHERE kind='accounts'`); err != nil { + t.Fatal(err) + } + if err := AssertReady(db); err != nil { + t.Fatal(err) + } +} + +func TestAccountRuntimeCannotAlterSchemaOrReadiness(t *testing.T) { + requirePostgres(t) + st, _ := newTestStore(t) + for _, stmt := range []string{`CREATE TABLE forbidden(id int)`, `ALTER TABLE users ADD COLUMN forbidden int`, `UPDATE corescope_schema SET ready=false`, `DELETE FROM schema_version`} { + if _, err := st.db.Exec(stmt); err == nil { + t.Fatalf("runtime accepted privileged statement: %s", stmt) + } + } +} + +func TestNativeAccountTextUsesBinaryCollation(t *testing.T) { + st, _ := newTestStore(t) + if testBackend(t) == dbconfig.Postgres { + var nonBinary int + if err := st.db.QueryRow(`SELECT count(*) FROM information_schema.columns WHERE table_schema=current_schema() AND data_type='text' AND collation_name IS DISTINCT FROM 'C'`).Scan(&nonBinary); err != nil { + t.Fatal(err) + } + if nonBinary != 0 { + t.Fatalf("%d account text columns inherit locale-dependent collation", nonBinary) + } + } + for _, email := range []string{"a@example.org", "B@example.org", "ä@example.org"} { + mustCreate(t, st, email, "Case") + } + rows, err := st.db.Query(`SELECT email FROM users ORDER BY email`) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + var got []string + for rows.Next() { + var email string + if err := rows.Scan(&email); err != nil { + t.Fatal(err) + } + got = append(got, email) + } + if strings.Join(got, ",") != "B@example.org,a@example.org,ä@example.org" { + t.Fatal("account byte ordering changed") + } +} + +func TestPendingPruneRechecksTokenAfterConcurrentIssue(t *testing.T) { + requirePostgres(t) + st, clk := newTestStore(t) + u := mustCreate(t, st, "pending-race@example.org", "Pending") + clk.Advance(8 * 24 * time.Hour) + other, err := pgutil.Open(st.target, false) + if err != nil { + t.Fatal(err) + } + defer other.Close() + tx, err := other.Begin() + if err != nil { + t.Fatal(err) + } + defer tx.Rollback() + if err := st.lockUser(tx, u.ID); err != nil { + t.Fatal(err) + } + type result struct { + n int64 + err error + } + finished := make(chan result, 1) + go func() { n, err := st.PruneStalePending(7 * 24 * time.Hour); finished <- result{n, err} }() + deadline := time.Now().Add(5 * time.Second) + for { + var waiting int + if err := other.QueryRow(`SELECT COUNT(*) FROM pg_stat_activity WHERE usename=current_user AND wait_event_type='Lock'`).Scan(&waiting); err != nil { + t.Fatal(err) + } + if waiting > 0 { + break + } + if time.Now().After(deadline) { + t.Fatal("prune did not reach the account row lock") + } + time.Sleep(10 * time.Millisecond) + } + if _, err := tx.Exec(`INSERT INTO tokens(token_hash,user_id,purpose,expires_at) VALUES('prune-race',$1,'activate',$2)`, u.ID, unix(clk.Now().Add(time.Hour))); err != nil { + t.Fatal(err) + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + got := <-finished + if got.err != nil || got.n != 0 { + t.Fatalf("pruned account after new activation token=%d,%v; want 0", got.n, got.err) + } + if _, err := st.GetByID(u.ID); err != nil { + t.Fatal("newly issued activation account was deleted") + } +} + +func TestNotifyStatesSkipConcurrentAccountDeletion(t *testing.T) { + requirePostgres(t) + st, clk := newTestStore(t) + u := mustCreate(t, st, "delete-race@example.org", "Deleted") + other, err := pgutil.Open(st.target, false) + if err != nil { + t.Fatal(err) + } + defer other.Close() + tx, err := other.Begin() + if err != nil { + t.Fatal(err) + } + defer tx.Rollback() + if _, err := tx.Exec(`DELETE FROM users WHERE id=$1`, u.ID); err != nil { + t.Fatal(err) + } + finished := make(chan error, 1) + go func() { + finished <- st.WriteNotifyStates([]NotifyState{nState(u.ID, NotifyNodeOffline, "node", NotifyBad, clk.Now())}) + }() + deadline := time.Now().Add(5 * time.Second) + for { + var waiting int + if err := other.QueryRow(`SELECT COUNT(*) FROM pg_stat_activity WHERE usename=current_user AND wait_event_type='Lock'`).Scan(&waiting); err != nil { + t.Fatal(err) + } + if waiting > 0 { + break + } + if time.Now().After(deadline) { + t.Fatal("notification insert did not reach the account lock") + } + time.Sleep(10 * time.Millisecond) + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + if err := <-finished; err != nil { + t.Fatalf("deleted account was not skipped: %v", err) + } +} diff --git a/internal/users/proposals.go b/internal/users/proposals.go index ba3d5bba2..8a10bcff8 100644 --- a/internal/users/proposals.go +++ b/internal/users/proposals.go @@ -3,6 +3,8 @@ package users import ( "database/sql" "errors" + "fmt" + "github.com/meshcore-analyzer/dbconfig" "time" ) @@ -116,8 +118,13 @@ func (s *Store) Propose(kind, subject string, userID int64, lim ProposalLimits) return nil, err } defer tx.Rollback() + // ponytail: serialize the small proposal queue across processes; replace + // with per-kind locks only if measured proposal throughput requires it. + if err := s.lockProposals(tx); err != nil { + return nil, err + } now := unix(s.now()) - cur, err := scanProposal(tx.QueryRow(`SELECT `+proposalCols+` FROM proposals WHERE kind = ? AND subject = ?`, kind, subject)) + cur, err := scanProposal(tx.QueryRow(`SELECT `+proposalCols+` FROM proposals WHERE kind = `+s.p(1)+` AND subject = `+s.p(2), kind, subject)) switch { case errors.Is(err, ErrNotFound): cur = nil @@ -132,7 +139,7 @@ func (s *Store) Propose(kind, subject string, userID int64, lim ProposalLimits) } if lim.PerUserPerDay > 0 { var n int - if err := tx.QueryRow(`SELECT COUNT(*) FROM proposals WHERE proposer_id = ? AND created_at > ?`, + if err := tx.QueryRow(`SELECT COUNT(*) FROM proposals WHERE proposer_id = `+s.p(1)+` AND created_at > `+s.p(2), userID, now-secs(24*time.Hour)).Scan(&n); err != nil { return nil, err } @@ -151,22 +158,19 @@ func (s *Store) Propose(kind, subject string, userID int64, lim ProposalLimits) } var id int64 if cur == nil { - res, err := tx.Exec(`INSERT INTO proposals (kind, subject, status, proposer_id, created_at) VALUES (?, ?, 'pending', ?, ?)`, - kind, subject, userID, now) + err := tx.QueryRow(`INSERT INTO proposals (kind, subject, status, proposer_id, created_at) VALUES (`+s.p(1)+`, `+s.p(2)+`, 'pending', `+s.p(3)+`, `+s.p(4)+`) RETURNING id`, + kind, subject, userID, now).Scan(&id) if err != nil { return nil, err } - if id, err = res.LastInsertId(); err != nil { - return nil, err - } } else { id = cur.ID - if _, err := tx.Exec(`UPDATE proposals SET status = 'pending', proposer_id = ?, reviewer_id = NULL, note = '', - created_at = ?, decided_at = NULL WHERE id = ?`, userID, now, id); err != nil { + if _, err := tx.Exec(`UPDATE proposals SET status = 'pending', proposer_id = `+s.p(1)+`, reviewer_id = NULL, note = '', + created_at = `+s.p(2)+`, decided_at = NULL WHERE id = `+s.p(3), userID, now, id); err != nil { return nil, err } } - p, err := scanProposal(tx.QueryRow(`SELECT `+proposalCols+` FROM proposals WHERE id = ?`, id)) + p, err := scanProposal(tx.QueryRow(`SELECT `+proposalCols+` FROM proposals WHERE id = `+s.p(1), id)) if err != nil { return nil, err } @@ -183,7 +187,12 @@ func (s *Store) Decide(id int64, action ProposalAction, reviewerID int64, note s return nil, err } defer tx.Rollback() - p, err := scanProposal(tx.QueryRow(`SELECT `+proposalCols+` FROM proposals WHERE id = ?`, id)) + // ponytail: serialize the small proposal queue across processes; replace + // with per-kind locks only if measured proposal throughput requires it. + if err := s.lockProposals(tx); err != nil { + return nil, err + } + p, err := scanProposal(tx.QueryRow(`SELECT `+proposalCols+` FROM proposals WHERE id = `+s.p(1), id)) if err != nil { return nil, err } @@ -200,7 +209,7 @@ func (s *Store) Decide(id int64, action ProposalAction, reviewerID int64, note s } if to == ProposalApproved && maxApproved > 0 { var n int - if err := tx.QueryRow(`SELECT COUNT(*) FROM proposals WHERE kind = ? AND status = 'approved'`, p.Kind).Scan(&n); err != nil { + if err := tx.QueryRow(`SELECT COUNT(*) FROM proposals WHERE kind = `+s.p(1)+` AND status = 'approved'`, p.Kind).Scan(&n); err != nil { return nil, err } if n >= maxApproved { @@ -208,7 +217,7 @@ func (s *Store) Decide(id int64, action ProposalAction, reviewerID int64, note s } } now := s.now() - if _, err := tx.Exec(`UPDATE proposals SET status = ?, reviewer_id = ?, note = ?, decided_at = ? WHERE id = ?`, + if _, err := tx.Exec(`UPDATE proposals SET status = `+s.p(1)+`, reviewer_id = `+s.p(2)+`, note = `+s.p(3)+`, decided_at = `+s.p(4)+` WHERE id = `+s.p(5), string(to), reviewerID, note, unix(now), id); err != nil { return nil, err } @@ -225,14 +234,14 @@ func (s *Store) ListProposals(f ProposalFilter) ([]Proposal, error) { q := `SELECT ` + proposalCols + ` FROM proposals WHERE 1=1` var args []any if f.Status != "" { - q += ` AND status = ?` + q += fmt.Sprintf(` AND status = %s`, s.p(len(args)+1)) args = append(args, string(f.Status)) } if f.Kind != "" { - q += ` AND kind = ?` + q += fmt.Sprintf(` AND kind = %s`, s.p(len(args)+1)) args = append(args, f.Kind) } - q += ` ORDER BY created_at DESC, id DESC LIMIT ?` + q += fmt.Sprintf(` ORDER BY created_at DESC, id DESC LIMIT %s`, s.p(len(args)+1)) args = append(args, ProposalListMax) return s.queryProposals(q, args...) } @@ -245,12 +254,12 @@ func (s *Store) ProposalsByUser(userID int64) ([]Proposal, error) { // AllProposalsByUser is ProposalsByUser without the cap (the account export). func (s *Store) AllProposalsByUser(userID int64) ([]Proposal, error) { - return s.proposalsByUser(userID, noLimit) + return s.proposalsByUser(userID, s.noLimit()) } -func (s *Store) proposalsByUser(userID int64, limit int) ([]Proposal, error) { - return s.queryProposals(`SELECT `+proposalCols+` FROM proposals WHERE proposer_id = ? - ORDER BY created_at DESC, id DESC LIMIT ?`, userID, limit) +func (s *Store) proposalsByUser(userID int64, limit any) ([]Proposal, error) { + return s.queryProposals(`SELECT `+proposalCols+` FROM proposals WHERE proposer_id = `+s.p(1)+` + ORDER BY created_at DESC, id DESC LIMIT `+s.p(2), userID, limit) } func (s *Store) queryProposals(q string, args ...any) ([]Proposal, error) { @@ -274,10 +283,10 @@ func (s *Store) queryProposals(q string, args ...any) ([]Proposal, error) { // first, at most limit (0 = all). Never nil. cmd/ingestor runs the same // query as raw SQL (see ingestorApprovedQuery in proposals_test.go). func (s *Store) ApprovedSubjects(kind string, limit int) ([]string, error) { - q := `SELECT subject FROM proposals WHERE kind = ? AND status = 'approved' ORDER BY decided_at, id` + q := `SELECT subject FROM proposals WHERE kind = ` + s.p(1) + ` AND status = 'approved' ORDER BY decided_at, id` args := []any{kind} if limit > 0 { - q += ` LIMIT ?` + q += ` LIMIT ` + s.p(2) args = append(args, limit) } rows, err := s.db.Query(q, args...) @@ -299,10 +308,20 @@ func (s *Store) ApprovedSubjects(kind string, limit int) ([]string, error) { // PruneProposals deletes rejected and revoked proposals decided more than // maxAge ago. Pending and approved rows are kept however old they are. func (s *Store) PruneProposals(maxAge time.Duration) (int64, error) { - res, err := s.db.Exec(`DELETE FROM proposals WHERE status IN ('rejected','revoked') AND decided_at < ?`, + res, err := s.db.Exec(`DELETE FROM proposals WHERE status IN ('rejected','revoked') AND decided_at < `+s.p(1), unix(s.now())-secs(maxAge)) if err != nil { return 0, err } return res.RowsAffected() } + +// SQLite begins write transactions IMMEDIATE. PostgreSQL needs an explicit +// queue lock because several processes can check the same quota concurrently. +func (s *Store) lockProposals(tx *sql.Tx) error { + if s.backend == dbconfig.SQLite { + return nil + } + _, err := tx.Exec(`SELECT pg_advisory_xact_lock(hashtextextended(current_schema(),7289041403))`) + return err +} diff --git a/internal/users/proposals_test.go b/internal/users/proposals_test.go index 1154965d9..596a9929b 100644 --- a/internal/users/proposals_test.go +++ b/internal/users/proposals_test.go @@ -1,9 +1,8 @@ package users import ( - "database/sql" "errors" - "path/filepath" + "github.com/meshcore-analyzer/dbconfig" "strings" "testing" "time" @@ -13,7 +12,7 @@ import ( // literal (it does not import this package). Its test checks the literal is // still in this file; TestApprovedSubjects checks it still agrees with // ApprovedSubjects. -const ingestorApprovedQuery = `SELECT subject FROM proposals WHERE kind = 'hashtag_channel' AND status = 'approved' ORDER BY decided_at, id LIMIT ?` +const ingestorApprovedQuery = `SELECT subject FROM approved_channels WHERE kind = 'hashtag_channel' ORDER BY decided_at, id LIMIT $1` func mustPropose(t *testing.T, st *Store, subject string, uid int64) *Proposal { t.Helper() @@ -274,7 +273,11 @@ func TestApprovedSubjects(t *testing.T) { if empty, err := st.ApprovedSubjects("nothing", 0); err != nil || empty == nil || len(empty) != 0 { t.Fatalf("no rows = %#v, %v; want a non-nil empty slice", empty, err) } - rows, err := st.db.Query(ingestorApprovedQuery, 2) + ingestorQuery := ingestorApprovedQuery + if st.backend == dbconfig.SQLite { + ingestorQuery = `SELECT subject FROM proposals WHERE kind = 'hashtag_channel' AND status = 'approved' ORDER BY decided_at, id LIMIT ?1` + } + rows, err := st.db.Query(ingestorQuery, 2) if err != nil { t.Fatalf("ingestor query: %v", err) } @@ -333,35 +336,11 @@ func TestProposalSurvivesProposerDeletion(t *testing.T) { } } -// A users.db written by a v3 binary gains the proposals table and keeps its rows. +// A native account schema at v3 binary gains the proposals table and keeps its rows. func TestMigrateV3DatabaseToV4(t *testing.T) { - path := filepath.Join(t.TempDir(), "users.db") - db, err := sql.Open("sqlite", path) - if err != nil { - t.Fatal(err) - } - stmts := []string{ - `CREATE TABLE schema_version (version INTEGER NOT NULL)`, - `INSERT INTO schema_version (version) VALUES (3)`, - } - for _, m := range migrations[:3] { - stmts = append(stmts, m...) - } - stmts = append(stmts, `INSERT INTO users (email, display_name, password_hash, created_at) VALUES ('old@example.org', 'Old', 'x', 1)`) - for _, s := range stmts { - if _, err := db.Exec(s); err != nil { - t.Fatalf("build v3 db: %v", err) - } - } - db.Close() - - st, err := Open(path) - if err != nil { - t.Fatalf("Open v3 db: %v", err) - } - defer st.Close() - if v, err := st.SchemaVersion(); err != nil || v != len(migrations) { - t.Fatalf("SchemaVersion = %d, %v; want %d", v, err, len(migrations)) + st := migratedTestStore(t, 3, `INSERT INTO users (email,display_name,password_hash,created_at) VALUES ('old@example.org','Old','x',1)`) + if v, err := st.SchemaVersion(); err != nil || v != schemaVersion(t) { + t.Fatalf("SchemaVersion = %d, %v; want %d", v, err, schemaVersion(t)) } if !hasIndex(t, st, "proposals_kind_subject") || !hasIndex(t, st, "proposals_status") { t.Fatal("proposals indexes missing after migration") diff --git a/internal/users/schema.go b/internal/users/schema.go index 06b0c9e60..fae85a9ba 100644 --- a/internal/users/schema.go +++ b/internal/users/schema.go @@ -6,81 +6,80 @@ import ( "fmt" ) -// migrations[i] upgrades the schema from version i to i+1. Forward-only: -// never edit a shipped entry, append a new one. var migrations = [][]string{ { // v1: sub-project A `CREATE TABLE users ( - id INTEGER PRIMARY KEY, - email TEXT NOT NULL UNIQUE, - display_name TEXT NOT NULL, - password_hash TEXT NOT NULL, - role TEXT NOT NULL DEFAULT 'user' CHECK (role IN ('user','admin')), - status TEXT NOT NULL DEFAULT 'pending' CHECK (status IN ('pending','active','disabled')), - created_at INTEGER NOT NULL, - activated_at INTEGER, - activated_by INTEGER, - last_login_at INTEGER, - email_bouncing INTEGER NOT NULL DEFAULT 0 + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + email TEXT COLLATE "C" NOT NULL UNIQUE, + display_name TEXT COLLATE "C" NOT NULL, + password_hash TEXT COLLATE "C" NOT NULL, + role TEXT COLLATE "C" NOT NULL DEFAULT 'user' CHECK (role IN ('user','admin')), + status TEXT COLLATE "C" NOT NULL DEFAULT 'pending' CHECK (status IN ('pending','active','disabled')), + created_at BIGINT NOT NULL, + activated_at BIGINT, + activated_by BIGINT, + last_login_at BIGINT, + email_bouncing BIGINT NOT NULL DEFAULT 0 CHECK (email_bouncing IN (0,1)) )`, `CREATE TABLE sessions ( - id INTEGER PRIMARY KEY, - token_hash TEXT NOT NULL UNIQUE, - user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE, - csrf_token TEXT NOT NULL, - created_at INTEGER NOT NULL, - expires_at INTEGER NOT NULL, - last_seen_at INTEGER NOT NULL, - user_agent TEXT NOT NULL DEFAULT '' + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + token_hash TEXT COLLATE "C" NOT NULL UNIQUE, + user_id BIGINT NOT NULL REFERENCES users(id) ON DELETE CASCADE, + csrf_token TEXT COLLATE "C" NOT NULL, + created_at BIGINT NOT NULL, + expires_at BIGINT NOT NULL, + last_seen_at BIGINT NOT NULL, + user_agent TEXT COLLATE "C" NOT NULL DEFAULT '' )`, `CREATE INDEX sessions_user ON sessions(user_id)`, `CREATE TABLE tokens ( - token_hash TEXT PRIMARY KEY, - user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE, - purpose TEXT NOT NULL CHECK (purpose IN ('activate','reset','email_change')), - new_email TEXT, - expires_at INTEGER NOT NULL, - used_at INTEGER + token_hash TEXT COLLATE "C" PRIMARY KEY, + user_id BIGINT NOT NULL REFERENCES users(id) ON DELETE CASCADE, + purpose TEXT COLLATE "C" NOT NULL CHECK (purpose IN ('activate','reset','email_change')), + new_email TEXT COLLATE "C", + expires_at BIGINT NOT NULL, + used_at BIGINT )`, `CREATE INDEX tokens_user ON tokens(user_id, purpose)`, `CREATE TABLE audit_log ( - id INTEGER PRIMARY KEY, - at INTEGER NOT NULL, - actor_user_id INTEGER, - action TEXT NOT NULL, - target_user_id INTEGER, - detail TEXT NOT NULL DEFAULT '{}' + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + at BIGINT NOT NULL, + actor_user_id BIGINT, + action TEXT COLLATE "C" NOT NULL, + target_user_id BIGINT, + detail TEXT COLLATE "C" NOT NULL DEFAULT '{}' )`, `CREATE INDEX audit_target ON audit_log(target_user_id)`, `CREATE INDEX audit_actor ON audit_log(actor_user_id)`, `CREATE TABLE mail_log ( - id INTEGER PRIMARY KEY, - user_id INTEGER REFERENCES users(id) ON DELETE SET NULL, - to_email TEXT NOT NULL, - purpose TEXT NOT NULL, - provider_message_id TEXT, - sent_at INTEGER NOT NULL, - last_event TEXT NOT NULL DEFAULT 'sent', - last_event_at INTEGER NOT NULL, - last_reason TEXT NOT NULL DEFAULT '' + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + user_id BIGINT REFERENCES users(id) ON DELETE SET NULL, + to_email TEXT COLLATE "C" NOT NULL, + purpose TEXT COLLATE "C" NOT NULL, + provider_message_id TEXT COLLATE "C", + sent_at BIGINT NOT NULL, + last_event TEXT COLLATE "C" NOT NULL DEFAULT 'sent', + last_event_at BIGINT NOT NULL, + last_reason TEXT COLLATE "C" NOT NULL DEFAULT '' )`, `CREATE INDEX mail_user ON mail_log(user_id)`, `CREATE UNIQUE INDEX mail_msgid ON mail_log(provider_message_id) WHERE provider_message_id IS NOT NULL`, `CREATE TABLE mail_events ( - mail_id INTEGER NOT NULL REFERENCES mail_log(id) ON DELETE CASCADE, - event TEXT NOT NULL, - at INTEGER NOT NULL, - reason TEXT NOT NULL DEFAULT '', + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + mail_id BIGINT NOT NULL REFERENCES mail_log(id) ON DELETE CASCADE, + event TEXT COLLATE "C" NOT NULL, + at BIGINT NOT NULL, + reason TEXT COLLATE "C" NOT NULL DEFAULT '', UNIQUE (mail_id, event, at) )`, }, { // v2: sub-project B, settings sync (one document per user) `CREATE TABLE user_settings ( - user_id INTEGER PRIMARY KEY REFERENCES users(id) ON DELETE CASCADE, - doc TEXT NOT NULL, - revision INTEGER NOT NULL, - generation TEXT NOT NULL, - updated_at INTEGER NOT NULL + user_id BIGINT PRIMARY KEY REFERENCES users(id) ON DELETE CASCADE, + doc TEXT COLLATE "C" NOT NULL, + revision BIGINT NOT NULL, + generation TEXT COLLATE "C" NOT NULL, + updated_at BIGINT NOT NULL )`, }, { // v3: sub-project C, global audit list ordered and filtered by time @@ -88,103 +87,188 @@ var migrations = [][]string{ }, { // v4: sub-project D, proposals (one generic table; this version ships the kind 'hashtag_channel') `CREATE TABLE proposals ( - id INTEGER PRIMARY KEY, - kind TEXT NOT NULL, - subject TEXT NOT NULL, - status TEXT NOT NULL CHECK (status IN ('pending','approved','rejected','revoked')), - proposer_id INTEGER REFERENCES users(id) ON DELETE SET NULL, - reviewer_id INTEGER REFERENCES users(id) ON DELETE SET NULL, - note TEXT NOT NULL DEFAULT '', - created_at INTEGER NOT NULL, - decided_at INTEGER + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + kind TEXT COLLATE "C" NOT NULL, + subject TEXT COLLATE "C" NOT NULL, + status TEXT COLLATE "C" NOT NULL CHECK (status IN ('pending','approved','rejected','revoked')), + proposer_id BIGINT REFERENCES users(id) ON DELETE SET NULL, + reviewer_id BIGINT REFERENCES users(id) ON DELETE SET NULL, + note TEXT COLLATE "C" NOT NULL DEFAULT '', + created_at BIGINT NOT NULL, + decided_at BIGINT )`, `CREATE UNIQUE INDEX proposals_kind_subject ON proposals(kind, subject)`, `CREATE INDEX proposals_status ON proposals(status)`, }, { // v5: sub-project E, node notifications `CREATE TABLE notification_prefs ( - user_id INTEGER PRIMARY KEY REFERENCES users(id) ON DELETE CASCADE, - enabled INTEGER NOT NULL DEFAULT 1, - events TEXT NOT NULL DEFAULT '', - unsub_token TEXT NOT NULL UNIQUE, - updated_at INTEGER NOT NULL + user_id BIGINT PRIMARY KEY REFERENCES users(id) ON DELETE CASCADE, + enabled BIGINT NOT NULL DEFAULT 1 CHECK (enabled IN (0,1)), + events TEXT COLLATE "C" NOT NULL DEFAULT '', + unsub_token TEXT COLLATE "C" NOT NULL UNIQUE, + updated_at BIGINT NOT NULL )`, `CREATE TABLE notification_watches ( - user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE, - pubkey TEXT NOT NULL, - created_at INTEGER NOT NULL, + user_id BIGINT NOT NULL REFERENCES users(id) ON DELETE CASCADE, + pubkey TEXT COLLATE "C" NOT NULL, + created_at BIGINT NOT NULL, PRIMARY KEY (user_id, pubkey) )`, `CREATE TABLE notification_state ( - user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE, - event TEXT NOT NULL, - subject TEXT NOT NULL, - state TEXT NOT NULL, - changed_at INTEGER NOT NULL, + user_id BIGINT NOT NULL REFERENCES users(id) ON DELETE CASCADE, + event TEXT COLLATE "C" NOT NULL, + subject TEXT COLLATE "C" NOT NULL, + state TEXT COLLATE "C" NOT NULL, + changed_at BIGINT NOT NULL, PRIMARY KEY (user_id, event, subject) )`, }, { // v6: sub-project F, companion linking (device sessions, linked companions) - `ALTER TABLE sessions ADD COLUMN kind TEXT NOT NULL DEFAULT 'web' + `ALTER TABLE sessions ADD COLUMN kind TEXT COLLATE "C" NOT NULL DEFAULT 'web' CHECK (kind IN ('web','device'))`, - `ALTER TABLE sessions ADD COLUMN label TEXT NOT NULL DEFAULT ''`, - `ALTER TABLE sessions ADD COLUMN scopes TEXT NOT NULL DEFAULT ''`, + `ALTER TABLE sessions ADD COLUMN label TEXT COLLATE "C" NOT NULL DEFAULT ''`, + `ALTER TABLE sessions ADD COLUMN scopes TEXT COLLATE "C" NOT NULL DEFAULT ''`, `CREATE TABLE companion_links ( - pubkey TEXT PRIMARY KEY, - user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE, - name TEXT NOT NULL DEFAULT '', - linked_at INTEGER NOT NULL + pubkey TEXT COLLATE "C" PRIMARY KEY, + user_id BIGINT NOT NULL REFERENCES users(id) ON DELETE CASCADE, + name TEXT COLLATE "C" NOT NULL DEFAULT '', + linked_at BIGINT NOT NULL )`, `CREATE INDEX companion_links_user ON companion_links(user_id)`, `CREATE TABLE link_challenges ( - challenge_hash TEXT PRIMARY KEY, - user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE, - pubkey TEXT NOT NULL, - expires_at INTEGER NOT NULL + challenge_hash TEXT COLLATE "C" PRIMARY KEY, + user_id BIGINT NOT NULL REFERENCES users(id) ON DELETE CASCADE, + pubkey TEXT COLLATE "C" NOT NULL, + expires_at BIGINT NOT NULL )`, }, } -func (s *Store) migrate() error { - if _, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS schema_version (version INTEGER NOT NULL)`); err != nil { - return fmt.Errorf("users: schema_version: %w", err) - } - v, err := s.SchemaVersion() - if errors.Is(err, sql.ErrNoRows) { - if _, err := s.db.Exec(`INSERT INTO schema_version (version) VALUES (0)`); err != nil { - return fmt.Errorf("users: init schema_version: %w", err) - } - v = 0 - } else if err != nil { - return fmt.Errorf("users: read schema_version: %w", err) +// SchemaVersion is the account data layout version; PostgreSQL readiness is +// recorded separately so the legacy importer can preserve its source history. +const CurrentSchemaVersion = 6 +const PostgresSchemaVersion = 1 + +// Tables is the dependency-ordered account data manifest (metadata excluded). +func Tables() []string { + return []string{"users", "sessions", "tokens", "audit_log", "mail_log", "mail_events", "user_settings", "proposals", "notification_prefs", "notification_watches", "notification_state", "companion_links", "link_challenges"} +} + +// Apply installs PostgreSQL accounts using the migration owner's connection. +// PostgreSQL runtime Open never runs DDL. An advisory transaction lock serializes simultaneous +// bootstraps, and all DDL plus the readiness marker commit atomically. +func Apply(db *sql.DB) error { + return apply(db, true) +} + +func ApplyPostgres(db *sql.DB) error { return Apply(db) } +func AssertPostgresReady(db *sql.DB) error { return AssertReady(db) } + +// ApplyForImport creates the schema with readiness false in the same commit. +// Only the offline importer's verified completion may change it to true. +func ApplyForImport(db *sql.DB) error { return apply(db, false) } + +func apply(db *sql.DB, markReady bool) error { + tx, err := db.Begin() + if err != nil { + return err } - if v > len(migrations) { - return fmt.Errorf("users: database schema version %d is newer than this binary supports (%d)", v, len(migrations)) + defer tx.Rollback() + if err := applyTx(tx, markReady); err != nil { + return err } - for i := v; i < len(migrations); i++ { - tx, err := s.db.Begin() - if err != nil { + return tx.Commit() +} + +// ApplyForImportTx lets the offline importer atomically create schema and its +// source-bound progress marker, with readiness false throughout. +func ApplyForImportTx(tx *sql.Tx) error { return applyTx(tx, false) } + +func applyTx(tx *sql.Tx, markReady bool) error { + var err error + if _, err = tx.Exec(`SELECT pg_advisory_xact_lock(hashtextextended(current_schema(),7289041402))`); err != nil { + return err + } + if _, err = tx.Exec(`CREATE TABLE IF NOT EXISTS corescope_schema (version INTEGER NOT NULL, kind TEXT COLLATE "C" NOT NULL PRIMARY KEY CHECK (kind='accounts'), ready BOOLEAN NOT NULL)`); err != nil { + return err + } + var kind string + var version int + var ready bool + err = tx.QueryRow(`SELECT kind,version,ready FROM corescope_schema`).Scan(&kind, &version, &ready) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return err + } + if err == nil && !markReady { + return errors.New("users: import bootstrap requires a fresh account database") + } + if err == nil && (kind != "accounts" || version != PostgresSchemaVersion || !ready) { + return errors.New("users: incompatible or incomplete PostgreSQL account schema") + } + if _, err = tx.Exec(`CREATE TABLE IF NOT EXISTS schema_version (version INTEGER NOT NULL CHECK (version>=0))`); err != nil { + return err + } + var v, count int + if err = tx.QueryRow(`SELECT COUNT(*), COALESCE(MAX(version),0) FROM schema_version`).Scan(&count, &v); err != nil { + return err + } + if count > 1 { + return errors.New("users: invalid account schema version metadata") + } + if v > CurrentSchemaVersion { + return fmt.Errorf("users: database schema version %d is newer than supported %d", v, CurrentSchemaVersion) + } + if count == 0 { + if _, err = tx.Exec(`INSERT INTO schema_version VALUES (0)`); err != nil { return err } + } + for i := v; i < len(migrations); i++ { for _, stmt := range migrations[i] { - if _, err := tx.Exec(stmt); err != nil { - tx.Rollback() + if _, err = tx.Exec(stmt); err != nil { return fmt.Errorf("users: migration %d: %w", i+1, err) } } - if _, err := tx.Exec(`UPDATE schema_version SET version = ?`, i+1); err != nil { - tx.Rollback() - return fmt.Errorf("users: migration %d: %w", i+1, err) - } - if err := tx.Commit(); err != nil { - return fmt.Errorf("users: migration %d: %w", i+1, err) + if _, err = tx.Exec(`UPDATE schema_version SET version=$1`, i+1); err != nil { + return err } } + if _, err = tx.Exec(`CREATE OR REPLACE VIEW approved_channels AS SELECT kind,subject,decided_at,id FROM proposals WHERE status='approved'`); err != nil { + return err + } + if _, err = tx.Exec(`CREATE OR REPLACE VIEW linked_companions AS SELECT pubkey FROM companion_links`); err != nil { + return err + } + if _, err = tx.Exec(`INSERT INTO corescope_schema(version,kind,ready) VALUES (1,'accounts',$1) ON CONFLICT(kind) DO NOTHING`, markReady); err != nil { + return err + } + return nil +} + +// AssertReady rejects missing, unsupported or partially imported account state. +func AssertReady(db *sql.DB) error { + var kind string + var version int + var ready bool + if err := db.QueryRow(`SELECT kind,version,ready FROM corescope_schema`).Scan(&kind, &version, &ready); err != nil { + return errors.New("users: account database needs bootstrap before runtime startup") + } + if kind != "accounts" || version != PostgresSchemaVersion || !ready { + return errors.New("users: account database is incompatible or import is incomplete") + } + var n, v int + if err := db.QueryRow(`SELECT COUNT(*),COALESCE(MAX(version),0) FROM schema_version`).Scan(&n, &v); err != nil { + return errors.New("users: account database needs bootstrap") + } + if v > CurrentSchemaVersion { + return fmt.Errorf("users: database schema version %d is newer than supported %d", v, CurrentSchemaVersion) + } + if n != 1 || v != CurrentSchemaVersion { + return errors.New("users: account database needs bootstrap") + } return nil } -// SchemaVersion returns the applied schema version (sql.ErrNoRows on a -// database that has never been migrated). func (s *Store) SchemaVersion() (int, error) { var v int err := s.db.QueryRow(`SELECT version FROM schema_version`).Scan(&v) diff --git a/internal/users/schema_postgres_upgrade_test.go b/internal/users/schema_postgres_upgrade_test.go new file mode 100644 index 000000000..b713fc46d --- /dev/null +++ b/internal/users/schema_postgres_upgrade_test.go @@ -0,0 +1,48 @@ +package users + +import "testing" + +func TestPostgresAccountsV5UpgradeKeepsAuthentication(t *testing.T) { + requirePostgres(t) + db := testOwner(t, testTarget(t, false)) + for _, migration := range migrations[:5] { + for _, q := range migration { + if _, err := db.Exec(q); err != nil { + t.Fatal(err) + } + } + } + for _, q := range []string{ + `CREATE TABLE schema_version(version INTEGER NOT NULL); INSERT INTO schema_version VALUES(5)`, + `CREATE TABLE corescope_schema(version INTEGER NOT NULL,kind TEXT PRIMARY KEY,ready BOOLEAN NOT NULL); INSERT INTO corescope_schema VALUES(1,'accounts',true)`, + `INSERT INTO users(id,email,display_name,password_hash,created_at) VALUES(4000000000,'upgrade@example.invalid','Retained','opaque-hash',1)`, + `INSERT INTO sessions(id,token_hash,user_id,csrf_token,created_at,expires_at,last_seen_at) VALUES(42,'opaque-session',4000000000,'opaque-csrf',1,9999999999,2)`, + } { + if _, err := db.Exec(q); err != nil { + t.Fatal(err) + } + } + if err := AssertPostgresReady(db); err == nil { + t.Fatal("v5 accepted before owner upgrade") + } + if err := ApplyPostgres(db); err != nil { + t.Fatal(err) + } + if err := AssertPostgresReady(db); err != nil { + t.Fatal(err) + } + var token, csrf, kind, label, scopes string + var user int64 + if err := db.QueryRow(`SELECT token_hash,csrf_token,user_id,kind,label,scopes FROM sessions WHERE id=42`).Scan(&token, &csrf, &user, &kind, &label, &scopes); err != nil { + t.Fatal(err) + } + if token != "opaque-session" || csrf != "opaque-csrf" || user != 4000000000 || kind != "web" || label != "" || scopes != "" { + t.Fatal("existing web authentication changed") + } + if _, err := db.Exec(`INSERT INTO companion_links(pubkey,user_id,linked_at) VALUES('fixture-key',4000000000,9999999999)`); err != nil { + t.Fatal("new account references narrowed to 32 bits", err) + } + if err := ApplyPostgres(db); err != nil { + t.Fatal("repeated owner upgrade", err) + } +} diff --git a/internal/users/schema_v6_test.go b/internal/users/schema_v6_test.go index 7e43e2d56..315e7d806 100644 --- a/internal/users/schema_v6_test.go +++ b/internal/users/schema_v6_test.go @@ -15,8 +15,9 @@ func TestMigrateV5DatabaseToV6(t *testing.T) { if err != nil { t.Fatal(err) } + defer db.Close() stmts := []string{`CREATE TABLE schema_version (version INTEGER NOT NULL)`, `INSERT INTO schema_version (version) VALUES (5)`} - for _, m := range migrations[:5] { + for _, m := range LegacyMigrations()[:5] { stmts = append(stmts, m...) } stmts = append(stmts, @@ -37,8 +38,8 @@ func TestMigrateV5DatabaseToV6(t *testing.T) { if len(migrations) < 6 { t.Fatal("migration v6 missing") } - if v, err := st.SchemaVersion(); err != nil || v != len(migrations) { - t.Fatalf("SchemaVersion = %d, %v; want %d", v, err, len(migrations)) + if v, err := st.SchemaVersion(); err != nil || v != SQLiteSchemaVersion { + t.Fatalf("SchemaVersion = %d, %v; want %d", v, err, SQLiteSchemaVersion) } var kind, label, scopes string diff --git a/internal/users/sessions.go b/internal/users/sessions.go index fb9c90b6b..a31471d57 100644 --- a/internal/users/sessions.go +++ b/internal/users/sessions.go @@ -134,13 +134,10 @@ func (s *Store) createSession(userID int64, ttl time.Duration, userAgent, kind, userAgent = strings.ToValidUTF8(userAgent[:maxUserAgent], "") } now := s.now() - res, err := s.db.Exec(`INSERT INTO sessions (token_hash, user_id, csrf_token, created_at, expires_at, last_seen_at, user_agent, kind, label, scopes) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, hash, userID, csrf, unix(now), unix(now.Add(ttl)), unix(now), userAgent, - kind, label, strings.Join(scopes, ",")) - if err != nil { - return "", nil, err - } - id, err := res.LastInsertId() + var id int64 + err = s.db.QueryRow(`INSERT INTO sessions (token_hash, user_id, csrf_token, created_at, expires_at, last_seen_at, user_agent, kind, label, scopes) + VALUES (`+s.p(1)+`, `+s.p(2)+`, `+s.p(3)+`, `+s.p(4)+`, `+s.p(5)+`, `+s.p(6)+`, `+s.p(7)+`, `+s.p(8)+`, `+s.p(9)+`, `+s.p(10)+`) RETURNING id`, hash, userID, csrf, unix(now), unix(now.Add(ttl)), unix(now), userAgent, + kind, label, strings.Join(scopes, ",")).Scan(&id) if err != nil { return "", nil, err } @@ -152,7 +149,7 @@ func (s *Store) createSession(userID int64, ttl time.Duration, userAgent, kind, // LookupSession resolves a raw cookie token. Unknown and expired sessions // return ErrNotFound; expired ones are deleted on the way. func (s *Store) LookupSession(raw string) (*Session, error) { - sess, err := scanSession(s.db.QueryRow(`SELECT `+sessionCols+` FROM sessions WHERE token_hash = ?`, HashToken(raw))) + sess, err := scanSession(s.db.QueryRow(`SELECT `+sessionCols+` FROM sessions WHERE token_hash = `+s.p(1), HashToken(raw))) if errors.Is(err, sql.ErrNoRows) { return nil, ErrNotFound } @@ -160,7 +157,7 @@ func (s *Store) LookupSession(raw string) (*Session, error) { return nil, fmt.Errorf("users: lookup session: %w", err) } if !s.now().Before(sess.ExpiresAt) { - _, _ = s.db.Exec(`DELETE FROM sessions WHERE id = ?`, sess.ID) + _, _ = s.db.Exec(`DELETE FROM sessions WHERE id = `+s.p(1), sess.ID) return nil, ErrNotFound } return sess, nil @@ -169,29 +166,29 @@ func (s *Store) LookupSession(raw string) (*Session, error) { // ExtendSession marks the session seen now and moves its expiry to now+ttl. func (s *Store) ExtendSession(id int64, ttl time.Duration) error { now := s.now() - return expectOne(s.db.Exec(`UPDATE sessions SET last_seen_at = ?, expires_at = ? WHERE id = ?`, + return expectOne(s.db.Exec(`UPDATE sessions SET last_seen_at = `+s.p(1)+`, expires_at = `+s.p(2)+` WHERE id = `+s.p(3), unix(now), unix(now.Add(ttl)), id)) } func (s *Store) DeleteSessionByToken(raw string) error { - _, err := s.db.Exec(`DELETE FROM sessions WHERE token_hash = ?`, HashToken(raw)) + _, err := s.db.Exec(`DELETE FROM sessions WHERE token_hash = `+s.p(1), HashToken(raw)) return err } // DeleteSession revokes one session, only if it belongs to userID. func (s *Store) DeleteSession(userID, sessionID int64) error { - return expectOne(s.db.Exec(`DELETE FROM sessions WHERE id = ? AND user_id = ?`, sessionID, userID)) + return expectOne(s.db.Exec(`DELETE FROM sessions WHERE id = `+s.p(1)+` AND user_id = `+s.p(2), sessionID, userID)) } // DeleteUserSessions revokes all of a user's sessions except exceptID (0 = none kept). func (s *Store) DeleteUserSessions(userID, exceptID int64) error { - _, err := s.db.Exec(`DELETE FROM sessions WHERE user_id = ? AND id != ?`, userID, exceptID) + _, err := s.db.Exec(`DELETE FROM sessions WHERE user_id = `+s.p(1)+` AND id != `+s.p(2), userID, exceptID) return err } // ListSessions returns a user's sessions, most recently seen first. func (s *Store) ListSessions(userID int64) ([]Session, error) { - rows, err := s.db.Query(`SELECT `+sessionCols+` FROM sessions WHERE user_id = ? ORDER BY last_seen_at DESC, id DESC`, userID) + rows, err := s.db.Query(`SELECT `+sessionCols+` FROM sessions WHERE user_id = `+s.p(1)+` ORDER BY last_seen_at DESC, id DESC`, userID) if err != nil { return nil, err } @@ -208,7 +205,7 @@ func (s *Store) ListSessions(userID int64) ([]Session, error) { } func (s *Store) PruneExpiredSessions() (int64, error) { - res, err := s.db.Exec(`DELETE FROM sessions WHERE expires_at <= ?`, unix(s.now())) + res, err := s.db.Exec(`DELETE FROM sessions WHERE expires_at <= `+s.p(1), unix(s.now())) if err != nil { return 0, err } diff --git a/internal/users/settings.go b/internal/users/settings.go index 148298abb..27da0c821 100644 --- a/internal/users/settings.go +++ b/internal/users/settings.go @@ -36,7 +36,7 @@ func newGeneration() (string, error) { func (s *Store) GetSettings(userID int64) (string, SettingsVersion, error) { var doc string var v SettingsVersion - err := s.db.QueryRow(`SELECT doc, revision, generation FROM user_settings WHERE user_id = ?`, userID).Scan(&doc, &v.Revision, &v.Generation) + err := s.db.QueryRow(`SELECT doc, revision, generation FROM user_settings WHERE user_id = `+s.p(1), userID).Scan(&doc, &v.Revision, &v.Generation) if errors.Is(err, sql.ErrNoRows) { return "", SettingsVersion{}, nil } @@ -59,7 +59,7 @@ type SettingsRecord struct { func (s *Store) SettingsRecordFor(userID int64) (*SettingsRecord, error) { var r SettingsRecord var updated int64 - err := s.db.QueryRow(`SELECT doc, revision, generation, updated_at FROM user_settings WHERE user_id = ?`, userID). + err := s.db.QueryRow(`SELECT doc, revision, generation, updated_at FROM user_settings WHERE user_id = `+s.p(1), userID). Scan(&r.Doc, &r.Version.Revision, &r.Version.Generation, &updated) if errors.Is(err, sql.ErrNoRows) { return nil, nil @@ -82,8 +82,11 @@ func (s *Store) PutSettings(userID int64, base SettingsVersion, doc string) (Set return SettingsVersion{}, err } defer tx.Rollback() + if err := s.lockUser(tx, userID); err != nil { + return SettingsVersion{}, err + } var cur SettingsVersion - err = tx.QueryRow(`SELECT revision, generation FROM user_settings WHERE user_id = ?`, userID).Scan(&cur.Revision, &cur.Generation) + err = tx.QueryRow(`SELECT revision, generation FROM user_settings WHERE user_id = `+s.p(1), userID).Scan(&cur.Revision, &cur.Generation) if err != nil && !errors.Is(err, sql.ErrNoRows) { return SettingsVersion{}, err } @@ -99,7 +102,7 @@ func (s *Store) PutSettings(userID int64, base SettingsVersion, doc string) (Set return cur, ErrSettingsConflict } next.Revision = cur.Revision + 1 - if _, err := tx.Exec(`INSERT INTO user_settings (user_id, doc, revision, generation, updated_at) VALUES (?, ?, ?, ?, ?) + if _, err := tx.Exec(`INSERT INTO user_settings (user_id, doc, revision, generation, updated_at) VALUES (`+s.p(1)+`, `+s.p(2)+`, `+s.p(3)+`, `+s.p(4)+`, `+s.p(5)+`) ON CONFLICT(user_id) DO UPDATE SET doc = excluded.doc, revision = excluded.revision, generation = excluded.generation, updated_at = excluded.updated_at`, userID, doc, next.Revision, next.Generation, unix(s.now())); err != nil { return SettingsVersion{}, err @@ -110,6 +113,18 @@ func (s *Store) PutSettings(userID int64, base SettingsVersion, doc string) (Set // DeleteSettings removes the user's synced settings; the next write starts // a new generation. No row is not an error. func (s *Store) DeleteSettings(userID int64) error { - _, err := s.db.Exec(`DELETE FROM user_settings WHERE user_id = ?`, userID) - return err + tx, err := s.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + if err = s.lockUser(tx, userID); errors.Is(err, ErrNotFound) { + return nil + } else if err != nil { + return err + } + if _, err = tx.Exec(`DELETE FROM user_settings WHERE user_id=`+s.p(1), userID); err != nil { + return err + } + return tx.Commit() } diff --git a/internal/users/settings_test.go b/internal/users/settings_test.go index bf25a10ad..ff28fa061 100644 --- a/internal/users/settings_test.go +++ b/internal/users/settings_test.go @@ -1,9 +1,7 @@ package users import ( - "database/sql" "errors" - "path/filepath" "testing" "time" ) @@ -44,7 +42,7 @@ func TestSettingsPutRevisions(t *testing.T) { t.Fatalf("GetSettings = %q, %+v, %v", doc, got, err) } var at int64 - if err := st.db.QueryRow(`SELECT updated_at FROM user_settings WHERE user_id = ?`, u.ID).Scan(&at); err != nil || at != unix(clk.Now()) { + if err := st.db.QueryRow(`SELECT updated_at FROM user_settings WHERE user_id = $1`, u.ID).Scan(&at); err != nil || at != unix(clk.Now()) { t.Fatalf("updated_at = %d, %v; want %d", at, err, unix(clk.Now())) } } @@ -134,32 +132,11 @@ func TestSettingsSurviveDisable(t *testing.T) { } } -// A users.db written by a v1 binary gains user_settings and keeps its rows. +// A native account schema at v1 binary gains user_settings and keeps its rows. func TestMigrateV1DatabaseToV2(t *testing.T) { - path := filepath.Join(t.TempDir(), "users.db") - db, err := sql.Open("sqlite", path) - if err != nil { - t.Fatal(err) - } - stmts := append([]string{ - `CREATE TABLE schema_version (version INTEGER NOT NULL)`, - `INSERT INTO schema_version (version) VALUES (1)`, - }, migrations[0]...) - stmts = append(stmts, `INSERT INTO users (email, display_name, password_hash, created_at) VALUES ('old@example.org', 'Old', 'x', 1)`) - for _, s := range stmts { - if _, err := db.Exec(s); err != nil { - t.Fatalf("build v1 db: %v", err) - } - } - db.Close() - - st, err := Open(path) - if err != nil { - t.Fatalf("Open v1 db: %v", err) - } - defer st.Close() - if v, err := st.SchemaVersion(); err != nil || v != len(migrations) { - t.Fatalf("SchemaVersion = %d, %v; want %d", v, err, len(migrations)) + st := migratedTestStore(t, 1, `INSERT INTO users (email,display_name,password_hash,created_at) VALUES ('old@example.org','Old','x',1)`) + if v, err := st.SchemaVersion(); err != nil || v != schemaVersion(t) { + t.Fatalf("SchemaVersion = %d, %v; want %d", v, err, schemaVersion(t)) } u, err := st.GetByEmail("old@example.org") if err != nil { diff --git a/internal/users/snapshot.go b/internal/users/snapshot.go index a4c2a0f4f..4f059de24 100644 --- a/internal/users/snapshot.go +++ b/internal/users/snapshot.go @@ -1,30 +1,57 @@ package users import ( + "context" "errors" - "fmt" - "io/fs" + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/pgutil" "os" + "path/filepath" ) -// Snapshot writes a consistent copy of users.db to path with VACUUM INTO -// (the technique of GET /api/backup) and makes it readable by the owner -// only: it holds password hashes and addresses. path must not exist; an -// empty file there is refused too, although VACUUM INTO would accept it. -// On failure no file is left at path. +// Snapshot writes a consistent, owner-readable native backup. SQLite produces +// a standalone database; PostgreSQL produces a pg_restore archive. The target +// must not exist, including an existing empty file. func (s *Store) Snapshot(path string) error { + return s.SnapshotContext(context.Background(), path) +} + +// SnapshotContext cancels the native backup when the caller disconnects or times out. +func (s *Store) SnapshotContext(ctx context.Context, path string) error { + if err := s.db.PingContext(ctx); err != nil { + return errors.New("users: account database is not available for snapshot") + } + if s.backend == dbconfig.Postgres { + return pgutil.Dump(ctx, s.target, path) + } if _, err := os.Lstat(path); err == nil { - return fmt.Errorf("users: snapshot target %s already exists", path) - } else if !errors.Is(err, fs.ErrNotExist) { - return fmt.Errorf("users: snapshot target %s: %w", path, err) + return errors.New("users: snapshot target already exists") + } else if !errors.Is(err, os.ErrNotExist) { + return err + } + // VACUUM INTO accepts a pre-created empty file, so private mode applies before + // any account data is written. Publish with a hard link to avoid overwriting a + // target created by another caller while the snapshot was running. + f, err := os.CreateTemp(filepath.Dir(path), ".accounts-snapshot-*") + if err != nil { + return err + } + temporary := f.Name() + defer os.Remove(temporary) + if err := f.Close(); err != nil { + return err + } + if _, err := s.db.ExecContext(ctx, `VACUUM INTO ?1`, temporary); err != nil { + return err } - if _, err := s.db.Exec(`VACUUM INTO ?`, path); err != nil { - os.Remove(path) - return fmt.Errorf("users: snapshot: %w", err) + if err := ctx.Err(); err != nil { + return err } - if err := os.Chmod(path, 0o600); err != nil { - os.Remove(path) - return fmt.Errorf("users: snapshot mode: %w", err) + if err := os.Link(temporary, path); err != nil { + if errors.Is(err, os.ErrExist) { + return errors.New("users: snapshot target already exists") + } + return err } return nil } diff --git a/internal/users/snapshot_test.go b/internal/users/snapshot_test.go index fdf282248..5f3fad482 100644 --- a/internal/users/snapshot_test.go +++ b/internal/users/snapshot_test.go @@ -1,47 +1,103 @@ package users import ( + "context" + "errors" "os" + "os/exec" "path/filepath" "runtime" "strings" "testing" + "time" + + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/pgutil" ) func TestSnapshotCopiesRows(t *testing.T) { - st, _ := newTestStore(t) + st, clk := newTestStoreAt(t, testTarget(t, true)) u := mustCreate(t, st, "a@example.org", "Aaa") + if err := st.Activate(u.ID, RoleUser, nil); err != nil { + t.Fatal(err) + } + rawSession, session, err := st.CreateSession(u.ID, 24*time.Hour, "backup-test") + if err != nil { + t.Fatal(err) + } + usedToken, err := st.IssueToken(u.ID, PurposeReset, time.Hour, "") + if err != nil { + t.Fatal(err) + } + if _, _, err := st.ConsumeToken(usedToken, PurposeReset); err != nil { + t.Fatal(err) + } + liveToken, err := st.IssueToken(u.ID, PurposeEmailChange, time.Hour, "new@example.org") + if err != nil { + t.Fatal(err) + } uid := u.ID if err := st.Audit(&uid, "user.register", &uid, nil); err != nil { t.Fatal(err) } - path := filepath.Join(t.TempDir(), "snap.db") + path := filepath.Join(t.TempDir(), "snap.dump") if err := st.Snapshot(path); err != nil { t.Fatal(err) } - cp, err := Open(path) + target := path + if st.backend == dbconfig.Postgres { + target = testTarget(t, true) + env, err := pgutil.CommandEnv(target) + if err != nil { + t.Fatal(err) + } + // --dbname is needed for restore mode; use a credential-free database name. + cfg, err := pgutil.ParseConfig(target) + if err != nil { + t.Fatal(err) + } + cmd := exec.Command("pg_restore", "--no-password", "--exit-on-error", "--no-owner", "--no-privileges", "--dbname="+cfg.Database, path) + cmd.Env = env + if err := cmd.Run(); err != nil { + t.Fatal("native account restore failed") + } + } + cp, err := Open(testRuntimeURL(t, target)) if err != nil { t.Fatal(err) } defer cp.Close() + cp.SetClock(clk.Now) got, err := cp.GetByEmail("a@example.org") if err != nil || got.ID != u.ID || got.DisplayName != "Aaa" { t.Fatalf("user in snapshot = %+v, %v", got, err) } - if v, err := cp.SchemaVersion(); err != nil || v != len(migrations) { - t.Fatalf("snapshot schema version = %d, %v; want %d", v, err, len(migrations)) + if v, err := cp.SchemaVersion(); err != nil || v != schemaVersion(t) { + t.Fatalf("snapshot schema version = %d, %v; want %d", v, err, schemaVersion(t)) } if e, err := cp.AuditAllFor(u.ID); err != nil || len(e) != 1 { t.Fatalf("audit in snapshot = %d, %v", len(e), err) } + if got.PasswordHash != u.PasswordHash || got.Status != StatusActive { + t.Fatal("password hash or account status changed") + } + if restored, err := cp.LookupSession(rawSession); err != nil || restored.ID != session.ID || restored.CSRFToken != session.CSRFToken { + t.Fatal("restored session/CSRF changed") + } + if _, err := cp.TokenUser(usedToken, PurposeReset); !errors.Is(err, ErrTokenInvalid) { + t.Fatal("used token revived after restore") + } + if id, email, err := cp.ConsumeToken(liveToken, PurposeEmailChange); err != nil || id != u.ID || email != "new@example.org" { + t.Fatal("unconsumed token did not survive restore") + } } func TestSnapshotFileMode(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("Unix file modes") } - st, _ := newTestStore(t) - path := filepath.Join(t.TempDir(), "snap.db") + st, _ := newTestStoreAt(t, testTarget(t, true)) + path := filepath.Join(t.TempDir(), "snap.dump") if err := st.Snapshot(path); err != nil { t.Fatal(err) } @@ -55,9 +111,9 @@ func TestSnapshotFileMode(t *testing.T) { } func TestSnapshotRefusesExistingTarget(t *testing.T) { - st, _ := newTestStore(t) - path := filepath.Join(t.TempDir(), "snap.db") - // An empty file: VACUUM INTO alone would accept and fill it. + st, _ := newTestStoreAt(t, testTarget(t, true)) + path := filepath.Join(t.TempDir(), "snap.dump") + // Existing empty files must also be preserved. if err := os.WriteFile(path, nil, 0o600); err != nil { t.Fatal(err) } @@ -71,9 +127,9 @@ func TestSnapshotRefusesExistingTarget(t *testing.T) { } func TestSnapshotFailureLeavesNoFile(t *testing.T) { - st, _ := newTestStore(t) + st, _ := newTestStoreAt(t, testTarget(t, true)) st.Close() - path := filepath.Join(t.TempDir(), "snap.db") + path := filepath.Join(t.TempDir(), "snap.dump") if err := st.Snapshot(path); err == nil { t.Fatal("snapshot of a closed store succeeded") } @@ -81,3 +137,16 @@ func TestSnapshotFailureLeavesNoFile(t *testing.T) { t.Fatalf("partial snapshot left behind: %v", err) } } + +func TestSnapshotCanceledLeavesNoFile(t *testing.T) { + st, _ := newTestStoreAt(t, testTarget(t, true)) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + path := filepath.Join(t.TempDir(), "cancel.dump") + if err := st.SnapshotContext(ctx, path); err == nil { + t.Fatal("canceled snapshot succeeded") + } + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Fatal("canceled snapshot left a file") + } +} diff --git a/internal/users/sqlite_backend_test.go b/internal/users/sqlite_backend_test.go new file mode 100644 index 000000000..1de5579e8 --- /dev/null +++ b/internal/users/sqlite_backend_test.go @@ -0,0 +1,124 @@ +package users + +import ( + "database/sql" + "github.com/meshcore-analyzer/dbconfig" + "os" + "path/filepath" + "runtime" + "testing" + "time" + + _ "modernc.org/sqlite" +) + +func TestSQLitePathOpensWithoutPostgres(t *testing.T) { + t.Setenv("CORESCOPE_TEST_POSTGRES_URL", "") + st, err := Open(filepath.Join(t.TempDir(), "accounts.db")) + if err != nil { + t.Fatalf("open default local account storage without PostgreSQL: %v", err) + } + defer st.Close() + u := mustCreate(t, st, "local@example.invalid", "Local account") + want := NotifyState{NotifyKey: NotifyKey{UserID: u.ID, Event: NotifyNodeBattery, Subject: "node-local"}, State: NotifyBad, ChangedAt: time.Unix(1234567890, 0).UTC()} + if err := st.WriteNotifyStates([]NotifyState{want}); err != nil { + t.Fatal(err) + } + got, err := st.AllNotifyStates() + if err != nil || len(got) != 1 || got[0] != want { + t.Fatalf("out-of-order notification parameters changed: %+v, %v", got, err) + } +} + +func TestModerncNumberedParametersPreservePositions(t *testing.T) { + db, err := sql.Open("sqlite", ":memory:") + if err != nil { + t.Fatal(err) + } + defer db.Close() + var second, first, repeated int + var null sql.NullString + if err := db.QueryRow(`SELECT ?2,?1,?2,?3`, 11, 22, nil).Scan(&second, &first, &repeated, &null); err != nil { + t.Fatal(err) + } + if second != 22 || first != 11 || repeated != 22 || null.Valid { + t.Fatal("SQLite numbered parameters did not retain logical argument positions") + } +} + +func TestOpenStorageUsesOnlySelectedAccountTarget(t *testing.T) { + target := filepath.Join(t.TempDir(), "accounts.db") + setup, err := Open(target) + if err != nil { + t.Fatal(err) + } + setup.Close() + st, err := OpenStorage(dbconfig.Storage{Backend: dbconfig.SQLite, UsersDBPath: target, UsersDatabaseURL: "postgres://inactive.invalid/accounts", WriterDatabaseURL: "postgres://inactive.invalid/accounts"}) + if err != nil { + t.Fatal(err) + } + if st.Backend() != dbconfig.SQLite { + t.Fatal("inactive URL changed selected backend") + } + st.Close() + before, err := os.ReadFile(target) + if err != nil { + t.Fatal(err) + } + for _, backend := range []dbconfig.Backend{dbconfig.Postgres, ""} { + if st, err := OpenStorage(dbconfig.Storage{Backend: backend, UsersDBPath: target}); err == nil { + st.Close() + t.Fatal("missing explicit account backend/URL fell back to SQLite") + } + } + after, err := os.ReadFile(target) + if err != nil || string(before) != string(after) { + t.Fatal("invalid selection mutated existing account file") + } + if st, err := OpenStorage(dbconfig.Storage{Backend: dbconfig.SQLite, UsersDBPath: target, DBPath: target}); err == nil { + st.Close() + t.Fatal("selected telemetry file accepted as account storage") + } +} + +func TestSQLiteAccountLiteralFilenameAndAliasIsolation(t *testing.T) { + names := []string{"accounts # % ü.db"} + if runtime.GOOS != "windows" { + names = append(names, "accounts ?mode=memory.db") + } + for _, name := range names { + t.Run(name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), name) + setup, err := Open(path) + if err != nil { + t.Fatal(err) + } + setup.Close() + st, err := OpenStorage(dbconfig.Storage{Backend: dbconfig.SQLite, UsersDBPath: path}) + if err != nil { + t.Fatal(err) + } + defer st.Close() + mustCreate(t, st, "literal@example.invalid", "Literal") + if _, err := os.Stat(path); err != nil { + t.Fatal("literal account file was not created:", err) + } + var actual string + var n int + if err := st.db.QueryRow(`PRAGMA database_list`).Scan(&n, &actual, &actual); err != nil { + t.Fatal(err) + } + if !samePath(path, actual) { + t.Fatal("URI opened another filesystem target") + } + alias := filepath.Join(filepath.Dir(path), "alias.db") + if err := os.Link(path, alias); err != nil { + t.Fatal(err) + } + if bad, err := OpenStorage(dbconfig.Storage{Backend: dbconfig.SQLite, UsersDBPath: alias, DBPath: path}); err == nil { + bad.Close() + t.Fatal("hardlink alias bypassed telemetry isolation") + } + }) + } +} diff --git a/internal/users/sqlite_bootstrap_test.go b/internal/users/sqlite_bootstrap_test.go new file mode 100644 index 000000000..be4bf15fd --- /dev/null +++ b/internal/users/sqlite_bootstrap_test.go @@ -0,0 +1,51 @@ +package users + +import ( + "context" + "database/sql" + "errors" + "net/url" + "path/filepath" + "testing" + "time" + + "github.com/meshcore-analyzer/dbconfig" +) + +func TestSQLiteWALBootstrapWaitIsBounded(t *testing.T) { + path := filepath.Join(t.TempDir(), "journal.db") + dsn, err := dbconfig.SQLiteURI(path, url.Values{"_pragma": {"busy_timeout(0)"}}) + if err != nil { + t.Fatal(err) + } + owner, err := sql.Open("sqlite", dsn) + if err != nil { + t.Fatal(err) + } + defer owner.Close() + owner.SetMaxOpenConns(1) + mustSQL(t, owner, `CREATE TABLE preserved(value TEXT);INSERT INTO preserved VALUES('keep');BEGIN IMMEDIATE`) + defer owner.Exec(`ROLLBACK`) + other, err := sql.Open("sqlite", dsn) + if err != nil { + t.Fatal(err) + } + defer other.Close() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond) + defer cancel() + if err = enableSQLiteWAL(ctx, other); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("locked journal wait did not honor its bound: %v", err) + } + if _, err = owner.Exec(`COMMIT`); err != nil { + t.Fatal(err) + } + ctx2, cancel2 := context.WithTimeout(context.Background(), time.Second) + defer cancel2() + if err = enableSQLiteWAL(ctx2, other); err != nil { + t.Fatal("journal did not become usable after lock release", err) + } + var value string + if err = other.QueryRow(`SELECT value FROM preserved`).Scan(&value); err != nil || value != "keep" { + t.Fatal("bootstrap changed data", err) + } +} diff --git a/internal/users/sqlite_companion_upgrade_test.go b/internal/users/sqlite_companion_upgrade_test.go new file mode 100644 index 000000000..e2d05ffa1 --- /dev/null +++ b/internal/users/sqlite_companion_upgrade_test.go @@ -0,0 +1,148 @@ +package users + +import ( + "database/sql" + "strings" + "testing" +) + +// Reproduce this PR's previously shipped identity-only v6, not upstream's +// companion-linking v6. Both declared the same version before this merge. +func identityV6Fixture(t *testing.T) *sql.DB { + t.Helper() + _, db := legacySQLiteFixture(t, 5, false) + for _, table := range sqliteIdentities { + body := table.body + if table.name == "sessions" { + body = body[:strings.Index(body, ", kind TEXT")] + ")" + } + rows, err := db.Query(`SELECT sql FROM sqlite_schema WHERE type='index' AND tbl_name=? AND sql IS NOT NULL`, table.name) + if err != nil { + t.Fatal(err) + } + var indexes []string + for rows.Next() { + var ddl string + if err := rows.Scan(&ddl); err != nil { + t.Fatal(err) + } + indexes = append(indexes, ddl) + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + rows.Close() + mustSQL(t, db, `DROP TABLE `+sqliteQuote(table.name), `CREATE TABLE `+sqliteQuote(table.name)+body) + for _, ddl := range indexes { + mustSQL(t, db, ddl) + } + } + mustSQL(t, db, `UPDATE schema_version SET version=6`) + return db +} + +func TestSQLiteBothV6LayoutsUpgradeWithoutLosingSessions(t *testing.T) { + for _, native := range []bool{false, true} { + name := "upstream-companions" + if native { + name = "fork-identities" + } + t.Run(name, func(t *testing.T) { + var db *sql.DB + if native { + db = identityV6Fixture(t) + } else { + _, db = legacySQLiteFixture(t, 6, false) + } + mustSQL(t, db, + `INSERT INTO users(id,email,display_name,password_hash,created_at) VALUES(4000000000,'upgrade@example.invalid','Retained','hash',1)`, + `INSERT INTO sessions(id,token_hash,user_id,csrf_token,created_at,expires_at,last_seen_at) VALUES(42,'session',4000000000,'csrf',1,9999999999,2)`) + if native { + mustSQL(t, db, `UPDATE sqlite_sequence SET seq=4000000100 WHERE name='users'`) + } else { + mustSQL(t, db, + `UPDATE sessions SET kind='device',label='Radio',scopes='account:read' WHERE id=42`, + `INSERT INTO companion_links VALUES('pubkey',4000000000,'Companion',3)`, + `INSERT INTO link_challenges VALUES('challenge',4000000000,'pubkey',9999999999)`) + } + if err := ValidateSQLiteSourceLayout(db); err != nil { + t.Fatal(err) + } + if err := ApplySQLite(db); err != nil { + t.Fatal(err) + } + if err := AssertSQLiteReady(db); err != nil { + t.Fatal(err) + } + var token, kind, label, scopes string + var user int64 + if err := db.QueryRow(`SELECT token_hash,user_id,kind,label,scopes FROM sessions WHERE id=42`).Scan(&token, &user, &kind, &label, &scopes); err != nil { + t.Fatal(err) + } + if token != "session" || user != 4000000000 { + t.Fatal("session identity or token changed") + } + if native && (kind != "web" || label != "" || scopes != "") { + t.Fatal("old web session was not preserved") + } + if !native && (kind != "device" || label != "Radio" || scopes != "account:read") { + t.Fatal("device session fields changed") + } + var next int64 + if err := db.QueryRow(`INSERT INTO users(email,display_name,password_hash,created_at) VALUES('next@example.invalid','','hash',2) RETURNING id`).Scan(&next); err != nil { + t.Fatal(err) + } + floor := int64(4000000000) + if native { + floor = 4000000100 + } + if next != floor+1 { + t.Fatalf("identity high water lost: got %d want %d", next, floor+1) + } + for _, table := range []string{"companion_links", "link_challenges"} { + var count int + if err := db.QueryRow(`SELECT count(*) FROM ` + table).Scan(&count); err != nil { + t.Fatal(err) + } + want := 1 + if native { + want = 0 + } + if count != want { + t.Fatalf("%s count %d want %d", table, count, want) + } + } + if err := ApplySQLite(db); err != nil { + t.Fatal("repeated upgrade:", err) + } + }) + } +} + +func TestSQLiteV6UnknownOrPartialLayoutsRefused(t *testing.T) { + for _, alter := range []string{ + `ALTER TABLE sessions ADD COLUMN unexpected TEXT`, + `CREATE TABLE companion_links(pubkey TEXT PRIMARY KEY)`, + `CREATE TRIGGER unknown_session AFTER INSERT ON sessions BEGIN SELECT 1; END`, + } { + t.Run(alter, func(t *testing.T) { + db := identityV6Fixture(t) + mustSQL(t, db, alter) + before := sqliteRows(t, db, `SELECT type,name,sql FROM sqlite_schema ORDER BY type,name`) + if err := ApplySQLite(db); err == nil { + t.Fatal("unknown schema accepted") + } + after := sqliteRows(t, db, `SELECT type,name,sql FROM sqlite_schema ORDER BY type,name`) + if len(before) != len(after) { + t.Fatal("rejected source changed") + } + for i := range before { + for j := range before[i] { + if before[i][j] != after[i][j] { + t.Fatal("rejected schema changed") + } + } + } + }) + } +} diff --git a/internal/users/sqlite_identity_test.go b/internal/users/sqlite_identity_test.go new file mode 100644 index 000000000..e72cd3707 --- /dev/null +++ b/internal/users/sqlite_identity_test.go @@ -0,0 +1,388 @@ +package users + +import ( + "context" + "database/sql" + "fmt" + "path/filepath" + "reflect" + "strings" + "sync" + "testing" +) + +func legacySQLiteFixture(t *testing.T, version int, auto bool) (string, *sql.DB) { + t.Helper() + path := filepath.Join(t.TempDir(), "legacy.db") + db, err := sql.Open("sqlite", path) + if err != nil { + t.Fatal(err) + } + db.SetMaxOpenConns(1) + t.Cleanup(func() { db.Close() }) + mustSQL(t, db, `CREATE TABLE schema_version(version INTEGER NOT NULL)`, fmt.Sprintf(`INSERT INTO schema_version VALUES(%d)`, version)) + for _, migration := range LegacyMigrations()[:version] { + for _, statement := range migration { + if auto { + statement = strings.Replace(statement, "id INTEGER PRIMARY KEY,", "id INTEGER PRIMARY KEY AUTOINCREMENT,", 1) + } + mustSQL(t, db, statement) + } + } + return path, db +} +func mustSQL(t *testing.T, db *sql.DB, statements ...string) { + t.Helper() + for _, stmt := range statements { + if _, err := db.Exec(stmt); err != nil { + t.Fatalf("fixture SQL: %v", err) + } + } +} +func sqliteRows(t *testing.T, db *sql.DB, query string) [][]any { + t.Helper() + rows, err := db.Query(query) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + cols, err := rows.Columns() + if err != nil { + t.Fatal(err) + } + var result [][]any + for rows.Next() { + values := make([]any, len(cols)) + dest := make([]any, len(cols)) + for i := range values { + dest[i] = &values[i] + } + if err := rows.Scan(dest...); err != nil { + t.Fatal(err) + } + result = append(result, values) + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + return result +} +func seedSQLiteAccounts(t *testing.T, db *sql.DB) { + mustSQL(t, db, + `INSERT INTO users VALUES(0,'zero@example.invalid','Zero','opaque-hash','admin','active',1,NULL,123,NULL,0)`, + `INSERT INTO users VALUES(-7,'negative@example.invalid','Negative','opaque-hash-2','user','pending',-1,NULL,NULL,2,1)`, + `INSERT INTO sessions VALUES(0,'session-hash',0,'csrf',1,9999999999,2,'agent')`, + `INSERT INTO tokens VALUES('live-token',0,'reset',NULL,9999999999,NULL),('used-token',0,'email_change','next@example.invalid',9999999999,3)`, + `INSERT INTO audit_log VALUES(-1,1,77,'user.old',89,' { "raw": true } ')`, + `INSERT INTO mail_log VALUES(0,0,'zero@example.invalid','reset','provider-id',1,'sent',2,'')`, + `INSERT INTO mail_events(rowid,mail_id,event,at,reason) VALUES(0,0,'sent',2,''),(14,0,'delivered',3,'delivery')`, + `INSERT INTO user_settings VALUES(0,' { "saved": 1 } ',2,'generation',3)`, + `INSERT INTO proposals VALUES(0,'hashtag_channel','#test','approved',0,NULL,'note',1,3)`, + `INSERT INTO notification_prefs VALUES(0,1,'node.offline','unsubscribe',3)`, + `INSERT INTO notification_watches VALUES(0,'public-key',3)`, + `INSERT INTO notification_state VALUES(0,'node.offline','public-key','bad',3)`) +} + +func TestSQLiteIdentityUpgradePreservesAllAccountRows(t *testing.T) { + path, db := legacySQLiteFixture(t, 5, false) + seedSQLiteAccounts(t, db) + tables := []string{"users", "sessions", "tokens", "audit_log", "mail_log", "mail_events", "user_settings", "proposals", "notification_prefs", "notification_watches", "notification_state"} + before := make(map[string][][]any) + for _, table := range tables { + projection := "*" + if table == "mail_events" { + projection = "rowid,*" + } + before[table] = sqliteRows(t, db, `SELECT `+projection+` FROM `+sqliteQuote(table)+` ORDER BY 1`) + } + mustSQL(t, db, `PRAGMA foreign_keys=ON`) + if err := ApplySQLite(db); err != nil { + t.Fatal(err) + } + for _, table := range tables { + projection := "*" + if table == "sessions" { + projection = "id,token_hash,user_id,csrf_token,created_at,expires_at,last_seen_at,user_agent" + } + if got := sqliteRows(t, db, `SELECT `+projection+` FROM `+sqliteQuote(table)+` ORDER BY 1`); !reflect.DeepEqual(got, before[table]) { + t.Fatalf("%s rows changed: got=%v want=%v", table, got, before[table]) + } + } + var fk int + if err := db.QueryRow(`PRAGMA foreign_keys`).Scan(&fk); err != nil || fk != 1 { + t.Fatalf("foreign_keys=%d,%v", fk, err) + } + if err := AssertSQLiteReady(db); err != nil { + t.Fatal(err) + } + // Historical references to deleted users remain meaningful after the upgrade. + st, err := Open(path) + if err != nil { + t.Fatal(err) + } + defer st.Close() + next := mustCreate(t, st, "next@example.invalid", "Next") + if next.ID != 124 { + t.Fatalf("next user id=%d; want123+1", next.ID) + } + // Existing rowid=0 becomes an explicit ID; next mail-event allocation uses14+1. + mustSQL(t, db, `INSERT INTO mail_events(mail_id,event,at) VALUES(0,'opened',4)`) + var eventID int64 + if err := db.QueryRow(`SELECT id FROM mail_events WHERE event='opened'`).Scan(&eventID); err != nil || eventID != 15 { + t.Fatalf("mail event id=%d,%v", eventID, err) + } + // Original FK actions and uniqueness survive the rebuild. + mustSQL(t, db, `DELETE FROM users WHERE id=0`) + for _, table := range []string{"sessions", "tokens", "user_settings", "notification_prefs", "notification_watches", "notification_state"} { + var n int + if err := db.QueryRow(`SELECT COUNT(*) FROM ` + sqliteQuote(table)).Scan(&n); err != nil || n != 0 { + t.Fatalf("%s cascade count=%d,%v", table, n, err) + } + } + var mailUser, proposer sql.NullInt64 + if err := db.QueryRow(`SELECT user_id FROM mail_log WHERE id=0`).Scan(&mailUser); err != nil || mailUser.Valid { + t.Fatal("mail SET NULL failed") + } + if err := db.QueryRow(`SELECT proposer_id FROM proposals WHERE id=0`).Scan(&proposer); err != nil || proposer.Valid { + t.Fatal("proposal SET NULL failed") + } + if _, err := db.Exec(`INSERT INTO mail_events(mail_id,event,at) VALUES(0,'opened',4)`); err == nil { + t.Fatal("mail event uniqueness lost") + } + if _, err := db.Exec(`INSERT INTO users(email,display_name,password_hash,role,created_at) VALUES('invalid','x','x','invalid',1)`); err == nil { + t.Fatal("role constraint lost") + } + mustSQL(t, db, `DELETE FROM mail_log WHERE id=0`) + var n int + db.QueryRow(`SELECT count(*) FROM mail_events`).Scan(&n) + if n != 0 { + t.Fatal("mail-event cascade lost") + } +} + +func TestSQLiteIdentityMigrationEveryLegacyVersion(t *testing.T) { + for version := 1; version <= 5; version++ { + t.Run(fmt.Sprint(version), func(t *testing.T) { + _, db := legacySQLiteFixture(t, version, false) + mustSQL(t, db, `INSERT INTO users(id,email,display_name,password_hash,created_at) VALUES(0,'legacy','Legacy','hash',1)`) + if err := ApplySQLite(db); err != nil { + t.Fatal(err) + } + if err := ApplySQLite(db); err != nil { + t.Fatal("idempotence:", err) + } + var id int64 + if err := db.QueryRow(`SELECT id FROM users WHERE email='legacy'`).Scan(&id); err != nil || id != 0 { + t.Fatal("legacy zero ID lost") + } + }) + } +} + +func TestSQLiteIdentityMigrationPreservesDeletedSequenceHighWater(t *testing.T) { + path, db := legacySQLiteFixture(t, 5, true) + seedSQLiteAccounts(t, db) + for _, table := range sqliteIdentities { + if table.name == "mail_events" { + continue + } + mustSQL(t, db, `UPDATE sqlite_sequence SET seq=900 WHERE name='`+table.name+`'`) + } + if err := ApplySQLite(db); err != nil { + t.Fatal(err) + } + for _, table := range sqliteIdentities { + if table.name == "mail_events" { + continue + } + var seq int64 + if err := db.QueryRow(`SELECT seq FROM sqlite_sequence WHERE name=?1`, table.name).Scan(&seq); err != nil || seq != 900 { + t.Fatalf("%s seq=%d,%v", table.name, seq, err) + } + } + st, err := Open(path) + if err != nil { + t.Fatal(err) + } + defer st.Close() + u := mustCreate(t, st, "allocate@example.invalid", "Allocate") + if u.ID != 901 { + t.Fatalf("next id=%d; want901", u.ID) + } + if err := st.Delete(u.ID); err != nil { + t.Fatal(err) + } + snapshot := filepath.Join(t.TempDir(), "copy.db") + if err := st.Snapshot(snapshot); err != nil { + t.Fatal(err) + } + restored, err := Open(snapshot) + if err != nil { + t.Fatal(err) + } + defer restored.Close() + if got := mustCreate(t, restored, "later@example.invalid", "Later"); got.ID != 902 { + t.Fatalf("snapshot reused deleted id: %d", got.ID) + } +} + +func TestSQLiteIdentityMigrationRefusesExtensionsWithoutMutation(t *testing.T) { + for _, ddl := range []string{`CREATE TRIGGER extension AFTER INSERT ON users BEGIN UPDATE users SET display_name='changed' WHERE id=NEW.id; END`, `ALTER TABLE users ADD COLUMN generated TEXT GENERATED ALWAYS AS (email) VIRTUAL`, `CREATE UNIQUE INDEX custom_constraint ON users(display_name)`} { + t.Run(strings.Fields(ddl)[1], func(t *testing.T) { + _, db := legacySQLiteFixture(t, 5, false) + mustSQL(t, db, ddl) + before := sqliteRows(t, db, `SELECT type,name,sql FROM sqlite_schema ORDER BY type,name`) + if err := ApplySQLite(db); err == nil { + t.Fatal("unsupported extension accepted") + } + after := sqliteRows(t, db, `SELECT type,name,sql FROM sqlite_schema ORDER BY type,name`) + if !reflect.DeepEqual(before, after) { + t.Fatal("refused migration changed source schema") + } + var version int + db.QueryRow(`SELECT version FROM schema_version`).Scan(&version) + if version != 5 { + t.Fatal("refused migration changed version") + } + }) + } +} + +func TestSQLiteIdentityMigrationFailureRollsBackAndRestoresPragma(t *testing.T) { + _, db := legacySQLiteFixture(t, 5, false) + mustSQL(t, db, `INSERT INTO tokens VALUES('orphan',999,'reset',NULL,1,NULL)`, `PRAGMA foreign_keys=ON`) + before := sqliteRows(t, db, `SELECT type,name,sql FROM sqlite_schema ORDER BY type,name`) + if err := ApplySQLite(db); err == nil || !strings.Contains(err.Error(), "foreign keys") { + t.Fatalf("late migration error=%v", err) + } + if after := sqliteRows(t, db, `SELECT type,name,sql FROM sqlite_schema ORDER BY type,name`); !reflect.DeepEqual(before, after) { + t.Fatal("failed table rebuild was not rolled back") + } + var fk, version int + db.QueryRow(`PRAGMA foreign_keys`).Scan(&fk) + db.QueryRow(`SELECT version FROM schema_version`).Scan(&version) + if fk != 1 || version != 5 { + t.Fatalf("rollback state FK=%d version=%d", fk, version) + } + if rows := sqliteRows(t, db, `SELECT token_hash,user_id FROM tokens`); len(rows) != 1 || rows[0][0] != "orphan" { + t.Fatal("source row changed on rollback") + } +} + +func TestSQLiteReadyRejectsMissingAccountObjects(t *testing.T) { + for _, statement := range []string{`DROP TABLE tokens`, `DROP INDEX proposals_kind_subject`, `DROP TABLE notification_state`} { + t.Run(statement, func(t *testing.T) { + st, err := Open(filepath.Join(t.TempDir(), "accounts.db")) + if err != nil { + t.Fatal(err) + } + defer st.Close() + mustSQL(t, st.db, statement) + if err := AssertSQLiteReady(st.db); err == nil { + t.Fatal("incomplete current account schema accepted") + } + }) + } +} + +func TestSQLiteImmediateTransactionsAcrossIndependentPools(t *testing.T) { + path := filepath.Join(t.TempDir(), "accounts.db") + first, err := Open(path) + if err != nil { + t.Fatal(err) + } + defer first.Close() + second, err := Open(path) + if err != nil { + t.Fatal(err) + } + defer second.Close() + // Begin itself must reserve the writer before either caller reads a quota. + tx, err := first.db.Begin() + if err != nil { + t.Fatal(err) + } + defer tx.Rollback() + // A native NOWAIT connection deterministically detects the reserved writer; + // no goroutine sleeps or scheduling order are needed to prove IMMEDIATE. + conn, err := second.db.Conn(context.Background()) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + if _, err := conn.ExecContext(context.Background(), `PRAGMA busy_timeout=0`); err != nil { + t.Fatal(err) + } + if _, err := conn.BeginTx(context.Background(), nil); err == nil { + t.Fatal("second write transaction began before first committed") + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + next, err := conn.BeginTx(context.Background(), nil) + if err != nil { + t.Fatal(err) + } + next.Rollback() + conn.Close() + // Explicit read-only snapshot transactions are not promoted to writers. + read, err := first.db.BeginTx(context.Background(), &sql.TxOptions{ReadOnly: true}) + if err != nil { + t.Fatal(err) + } + defer read.Rollback() + if _, err := second.CreatePending("writer@example.invalid", "Writer", "hash"); err != nil { + t.Fatal(err) + } + if first.db.Stats().MaxOpenConnections != 1 || second.db.Stats().MaxOpenConnections != 1 { + t.Fatal("account pool grew") + } + +} + +func TestSQLiteEmptyVersionTableFromInterruptedLegacyInitialization(t *testing.T) { + _, db := legacySQLiteFixture(t, 0, false) + mustSQL(t, db, `DELETE FROM schema_version`) + if err := ApplySQLite(db); err != nil { + t.Fatal("legacy initialization could stop between CREATE and INSERT:", err) + } + if err := AssertSQLiteReady(db); err != nil { + t.Fatal(err) + } +} + +func TestSQLiteAccountBootstrapAcrossPools(t *testing.T) { + path := filepath.Join(t.TempDir(), "accounts.db") + start := make(chan struct{}) + var wg sync.WaitGroup + for i := 0; i < 4; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + <-start + st, err := Open(path) + if err != nil { + t.Errorf("bootstrap: %v", err) + return + } + defer st.Close() + if _, err := st.CreatePending(fmt.Sprintf("worker%d@example.invalid", i), "Concurrent", "hash"); err != nil { + t.Error(err) + } + }(i) + } + close(start) + wg.Wait() + st, err := Open(path) + if err != nil { + t.Fatal(err) + } + defer st.Close() + var n int + if err := st.db.QueryRow(`SELECT count(*) FROM users`).Scan(&n); err != nil || n != 4 { + t.Fatalf("accounts=%d,%v", n, err) + } + if err := AssertSQLiteReady(st.db); err != nil { + t.Fatal(err) + } +} diff --git a/internal/users/sqlite_import_test.go b/internal/users/sqlite_import_test.go new file mode 100644 index 000000000..e0fc6e291 --- /dev/null +++ b/internal/users/sqlite_import_test.go @@ -0,0 +1,32 @@ +package users + +import ( + "database/sql" + "errors" + "path/filepath" + "testing" + + "github.com/meshcore-analyzer/dbconfig" +) + +func TestSQLitePendingImportRefusesBeforeAccountSchemaChanges(t *testing.T) { + for name, check := range map[string]func(*sql.DB) error{"apply": ApplySQLite, "ready": AssertSQLiteReady} { + t.Run(name, func(t *testing.T) { + db, err := sql.Open("sqlite", filepath.Join(t.TempDir(), "partial.db")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + if _, err := db.Exec(`CREATE TABLE corescope_reverse_progress (table_name TEXT)`); err != nil { + t.Fatal(err) + } + if err := check(db); !errors.Is(err, dbconfig.ErrSQLiteImportIncomplete) { + t.Errorf("pending account import was not identified: %v", err) + } + var objects int + if err := db.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE name <> 'corescope_reverse_progress'`).Scan(&objects); err != nil || objects != 0 { + t.Errorf("pending source changed before refusal: objects=%d, err=%v", objects, err) + } + }) + } +} diff --git a/internal/users/sqlite_open.go b/internal/users/sqlite_open.go new file mode 100644 index 000000000..359a91c9e --- /dev/null +++ b/internal/users/sqlite_open.go @@ -0,0 +1,134 @@ +package users + +import ( + "context" + "database/sql" + "errors" + "net/url" + "os" + "path/filepath" + "strings" + "time" + + "github.com/meshcore-analyzer/dbconfig" + "modernc.org/sqlite" +) + +func openSQLite(path string, forbidden ...string) (*Store, error) { + return openSQLiteMode(path, true, forbidden...) +} +func openSQLiteMode(path string, create bool, forbidden ...string) (*Store, error) { + if !create && strings.TrimSpace(path) == "" { + return nil, errors.New("users: accounts are not initialized; run explicit account setup") + } + if strings.TrimSpace(path) == "" || strings.Contains(path, "://") || strings.HasPrefix(strings.ToLower(path), "file:") { + return nil, errors.New("users: SQLite account storage requires a local filesystem path") + } + abs, err := filepath.Abs(path) + if err != nil { + return nil, errors.New("users: cannot resolve account database path") + } + for _, other := range forbidden { + if strings.TrimSpace(other) != "" && samePath(abs, other) { + return nil, errors.New("users: refusing to open the measurement database") + } + } + if create { + if err := os.MkdirAll(filepath.Dir(abs), 0755); err != nil { + return nil, err + } + } else { + info, err := os.Stat(abs) + if err != nil || !info.Mode().IsRegular() || info.Size() == 0 { + return nil, errors.New("users: initialized SQLite accounts are missing or empty; restore the original store") + } + } + // Immediate write transactions serialize read-before-write across processes. + // The pinned modernc driver leaves explicit read-only transactions deferred. + dsn, err := sqliteAccountDSN(abs, create) + if err != nil { + return nil, err + } + db, err := sql.Open("sqlite", dsn) + if err != nil { + return nil, errors.New("users: cannot open SQLite account storage") + } + db.SetMaxOpenConns(1) + db.SetMaxIdleConns(1) + if !create { + if err := validateExistingSQLiteAccounts(db); err != nil { + db.Close() + return nil, err + } + } + + if err := ApplySQLite(db); err != nil { + db.Close() + return nil, err + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := enableSQLiteWAL(ctx, db); err != nil { + db.Close() + return nil, err + } + + return &Store{db: db, target: abs, backend: dbconfig.SQLite, now: time.Now}, nil +} + +func samePath(abs, other string) bool { + otherAbs, err := filepath.Abs(other) + if err != nil { + return false + } + a, ea := os.Stat(abs) + b, eb := os.Stat(otherAbs) + if ea == nil && eb == nil { + return os.SameFile(a, b) + } + return strings.EqualFold(filepath.Clean(abs), filepath.Clean(otherAbs)) +} + +func sqliteAccountDSN(path string, create bool) (string, error) { + q := url.Values{"_pragma": {"busy_timeout(5000)", "foreign_keys(1)"}, "_txlock": {"immediate"}} + if !create { + q.Set("mode", "rw") + } + return dbconfig.SQLiteURI(path, q) +} + +func validateExistingSQLiteAccounts(db *sql.DB) error { + if err := dbconfig.AssertSQLiteImportComplete(db); err != nil { + return err + } + var count, version int + if err := db.QueryRow(`SELECT count(*),COALESCE(MAX(version),0) FROM schema_version`).Scan(&count, &version); err != nil || count != 1 || version < 1 || version > SQLiteSchemaVersion { + return errors.New("users: initialized SQLite accounts have no supported schema; restore or explicitly upgrade the original store") + } + return validateSQLiteLayout(db, version) +} + +// journal_mode may return SQLITE_BUSY immediately while another first opener +// changes the journal or creates its schema, even with busy_timeout enabled. +// Retry only that transient native result, bounded by the startup deadline. +func enableSQLiteWAL(ctx context.Context, db *sql.DB) error { + for { + var mode string + err := db.QueryRowContext(ctx, `PRAGMA journal_mode=WAL`).Scan(&mode) + if err == nil { + if mode != "wal" { + return errors.New("users: SQLite accounts require WAL mode") + } + return nil + } + var busy *sqlite.Error + if !errors.As(err, &busy) || busy.Code()&0xff != 5 { + return err + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(10 * time.Millisecond): + } + } +} diff --git a/internal/users/sqlite_runtime_open_test.go b/internal/users/sqlite_runtime_open_test.go new file mode 100644 index 000000000..2cb6e6e46 --- /dev/null +++ b/internal/users/sqlite_runtime_open_test.go @@ -0,0 +1,216 @@ +package users + +import ( + "crypto/sha256" + "database/sql" + "errors" + "fmt" + "os" + "path/filepath" + "testing" + "time" + + "github.com/meshcore-analyzer/dbconfig" +) + +func TestSelectedSQLiteAccountsCannotInitializeReplacement(t *testing.T) { + for _, kind := range []string{"missing", "empty", "header only", "wrong schema"} { + t.Run(kind, func(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "accounts.db") + original, err := Open(path) // Explicit setup/fixture creation. + if err != nil { + t.Fatal(err) + } + user := mustCreate(t, original, "preserved@example.invalid", "Preserved") + token, session, err := original.CreateSession(user.ID, time.Hour, "test") + if err != nil { + t.Fatal(err) + } + mustSQL(t, original.db, `INSERT INTO audit_log(id,at,action,detail) VALUES(900,1,'seed','{}'); DELETE FROM audit_log WHERE id=900`) + original.Close() + selected, err := dbconfig.NewSelection(dbconfig.Storage{Backend: dbconfig.SQLite, StateDir: dir, DBPath: filepath.Join(dir, "telemetry.db"), UsersDBPath: path}) + if err != nil { + t.Fatal(err) + } + preserved := path + ".preserved" + if err := os.Rename(path, preserved); err != nil { + t.Fatal(err) + } + before, err := os.ReadFile(preserved) + if err != nil { + t.Fatal(err) + } + if kind == "empty" { + if err := os.WriteFile(path, nil, 0600); err != nil { + t.Fatal(err) + } + } else if kind == "header only" || kind == "wrong schema" { + db, err := sql.Open("sqlite", path) + if err != nil { + t.Fatal(err) + } + query := `PRAGMA user_version=1` + if kind == "wrong schema" { + query = `CREATE TABLE unrelated(value TEXT); INSERT INTO unrelated VALUES('keep')` + } + if _, err := db.Exec(query); err != nil { + t.Fatal(err) + } + db.Close() + } + badBefore, _ := os.ReadFile(path) + raw, err := selected.ApplyTo(dbconfig.StorageInputs{BaseDir: dir, UsersDBPath: filepath.Join(dir, "inactive-old-setting.db")}) + if err != nil { + t.Fatal(err) + } + storage, err := dbconfig.ResolveStorage(raw) + if err != nil { + t.Fatal(err) + } + if storage.UsersDBPath != path { + t.Fatal("recorded initialized account target disappeared from resolution") + } + if st, err := OpenStorage(storage); err == nil { + st.Close() + t.Fatal("selected account storage initialized a replacement") + } + badAfter, statErr := os.ReadFile(path) + if kind == "missing" { + if !errors.Is(statErr, os.ErrNotExist) { + t.Fatal("runtime created missing accounts", statErr) + } + } else { + if statErr != nil || sha256.Sum256(badBefore) != sha256.Sum256(badAfter) { + t.Fatal("runtime modified rejected account target", statErr) + } + if err := os.Remove(path); err != nil { + t.Fatal(err) + } + } + after, err := os.ReadFile(preserved) + if err != nil || sha256.Sum256(before) != sha256.Sum256(after) { + t.Fatal("preserved accounts changed", err) + } + if err := os.Rename(preserved, path); err != nil { + t.Fatal(err) + } + restored, err := OpenStorage(storage) + if err != nil { + t.Fatal(err) + } + defer restored.Close() + if got, err := restored.GetByID(user.ID); err != nil || got.Email != user.Email { + t.Fatal("user lost after restoring original store", err) + } + if got, err := restored.LookupSession(token); err != nil || got.ID != session.ID { + t.Fatal("session lost after restoring original store", err) + } + var sequence int + if err := restored.db.QueryRow(`SELECT seq FROM sqlite_sequence WHERE name='audit_log'`).Scan(&sequence); err != nil || sequence != 900 { + t.Fatal("allocator high-water lost", sequence, err) + } + }) + } +} + +func TestSQLiteAccountTargetRemovedAfterDSNValidation(t *testing.T) { + for _, replaceEmpty := range []bool{false, true} { + path := filepath.Join(t.TempDir(), "accounts.db") + setup, err := Open(path) + if err != nil { + t.Fatal(err) + } + setup.Close() + dsn, err := sqliteAccountDSN(path, false) + if err != nil { + t.Fatal(err) + } + if err = os.Rename(path, path+".preserved"); err != nil { + t.Fatal(err) + } + if replaceEmpty { + if err = os.WriteFile(path, nil, 0600); err != nil { + t.Fatal(err) + } + } + db, err := sql.Open("sqlite", dsn) + if err != nil { + t.Fatal(err) + } + if err = validateExistingSQLiteAccounts(db); err == nil { + t.Fatal("raced missing/empty target accepted") + } + db.Close() + info, err := os.Stat(path) + if replaceEmpty { + if err != nil || info.Size() != 0 { + t.Fatal("raced empty accounts were initialized", err) + } + } else if !errors.Is(err, os.ErrNotExist) { + t.Fatal("raced missing accounts were recreated", err) + } + } +} + +func TestSelectedSQLiteAccountsUpgradeSupportedInitializedVersions(t *testing.T) { + for version := 1; version < SQLiteSchemaVersion; version++ { + t.Run(fmt.Sprint(version), func(t *testing.T) { + path, db := legacySQLiteFixture(t, version, false) + mustSQL(t, db, `INSERT INTO users(id,email,display_name,password_hash,role,status,created_at) VALUES(91,'legacy@example.invalid','Legacy','hash','admin','active',1)`) + db.Close() + st, err := OpenStorage(dbconfig.Storage{Backend: dbconfig.SQLite, UsersDBPath: path}) + if err != nil { + t.Fatal(err) + } + defer st.Close() + if user, err := st.GetByID(91); err != nil || user.Email != "legacy@example.invalid" { + t.Fatal("supported initialized account row lost", err) + } + if err = AssertSQLiteReady(st.db); err != nil { + t.Fatal(err) + } + }) + } +} + +func TestSelectedSQLiteAccountsNeedExplicitInitialization(t *testing.T) { + dir := t.TempDir() + selected, err := dbconfig.NewSelection(dbconfig.Storage{Backend: dbconfig.SQLite, StateDir: dir, DBPath: filepath.Join(dir, "telemetry.db")}) + if err != nil { + t.Fatal(err) + } + raw, err := selected.ApplyTo(dbconfig.StorageInputs{BaseDir: dir, UsersDBPath: filepath.Join(dir, "unused.db")}) + if err != nil { + t.Fatal(err) + } + uninitialized, err := dbconfig.ResolveStorage(raw) + if err != nil { + t.Fatal(err) + } + if uninitialized.UsersDBPath != "" { + t.Fatal("uninitialized selected accounts received an inferred file") + } + if st, err := OpenStorage(uninitialized); err == nil { + st.Close() + t.Fatal("uninitialized accounts were opened") + } + path := filepath.Join(t.TempDir(), "new", "accounts.db") + if st, err := OpenStorage(dbconfig.Storage{Backend: dbconfig.SQLite, UsersDBPath: path}); err == nil { + st.Close() + t.Fatal("selected runtime performed first initialization") + } + if _, err := os.Stat(filepath.Dir(path)); !errors.Is(err, os.ErrNotExist) { + t.Fatal("runtime created account directory", err) + } + setup, err := Open(path) + if err != nil { + t.Fatal(err) + } + setup.Close() + runtime, err := OpenStorage(dbconfig.Storage{Backend: dbconfig.SQLite, UsersDBPath: path}) + if err != nil { + t.Fatal("explicitly initialized accounts failed to open", err) + } + runtime.Close() +} diff --git a/internal/users/sqlite_schema.go b/internal/users/sqlite_schema.go new file mode 100644 index 000000000..d21d16fd8 --- /dev/null +++ b/internal/users/sqlite_schema.go @@ -0,0 +1,452 @@ +package users + +import ( + "context" + "database/sql" + "database/sql/driver" + "errors" + "fmt" + "strings" + "unicode" + + "github.com/meshcore-analyzer/dbconfig" +) + +// Version 7 combines upstream companion linking (upstream v6) with this +// branch's non-reusing identities (its earlier v6). Exact layout validation +// distinguishes those two historical versions before upgrading either one. +const SQLiteSchemaVersion = 7 + +var sqliteIdentities = []struct{ name, columns, body string }{ + {"users", "id,email,display_name,password_hash,role,status,created_at,activated_at,activated_by,last_login_at,email_bouncing", `( + id INTEGER PRIMARY KEY AUTOINCREMENT, email TEXT NOT NULL UNIQUE, display_name TEXT NOT NULL, password_hash TEXT NOT NULL, + role TEXT NOT NULL DEFAULT 'user' CHECK (role IN ('user','admin')), + status TEXT NOT NULL DEFAULT 'pending' CHECK (status IN ('pending','active','disabled')), + created_at INTEGER NOT NULL, activated_at INTEGER, activated_by INTEGER, last_login_at INTEGER, + email_bouncing INTEGER NOT NULL DEFAULT 0)`}, + {"sessions", "id,token_hash,user_id,csrf_token,created_at,expires_at,last_seen_at,user_agent,kind,label,scopes", `( + id INTEGER PRIMARY KEY AUTOINCREMENT, token_hash TEXT NOT NULL UNIQUE, + user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE, csrf_token TEXT NOT NULL, + created_at INTEGER NOT NULL, expires_at INTEGER NOT NULL, last_seen_at INTEGER NOT NULL, + user_agent TEXT NOT NULL DEFAULT '', kind TEXT NOT NULL DEFAULT 'web' CHECK (kind IN ('web','device')), + label TEXT NOT NULL DEFAULT '', scopes TEXT NOT NULL DEFAULT '')`}, + {"audit_log", "id,at,actor_user_id,action,target_user_id,detail", `( + id INTEGER PRIMARY KEY AUTOINCREMENT, at INTEGER NOT NULL, actor_user_id INTEGER, + action TEXT NOT NULL, target_user_id INTEGER, detail TEXT NOT NULL DEFAULT '{}')`}, + {"mail_log", "id,user_id,to_email,purpose,provider_message_id,sent_at,last_event,last_event_at,last_reason", `( + id INTEGER PRIMARY KEY AUTOINCREMENT, user_id INTEGER REFERENCES users(id) ON DELETE SET NULL, + to_email TEXT NOT NULL, purpose TEXT NOT NULL, provider_message_id TEXT, sent_at INTEGER NOT NULL, + last_event TEXT NOT NULL DEFAULT 'sent', last_event_at INTEGER NOT NULL, last_reason TEXT NOT NULL DEFAULT '')`}, + {"mail_events", "id,mail_id,event,at,reason", `( + id INTEGER PRIMARY KEY AUTOINCREMENT, mail_id INTEGER NOT NULL REFERENCES mail_log(id) ON DELETE CASCADE, + event TEXT NOT NULL, at INTEGER NOT NULL, reason TEXT NOT NULL DEFAULT '', UNIQUE (mail_id,event,at))`}, + {"proposals", "id,kind,subject,status,proposer_id,reviewer_id,note,created_at,decided_at", `( + id INTEGER PRIMARY KEY AUTOINCREMENT, kind TEXT NOT NULL, subject TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('pending','approved','rejected','revoked')), + proposer_id INTEGER REFERENCES users(id) ON DELETE SET NULL, reviewer_id INTEGER REFERENCES users(id) ON DELETE SET NULL, + note TEXT NOT NULL DEFAULT '', created_at INTEGER NOT NULL, decided_at INTEGER)`}, +} + +type sqliteSchemaQueryer interface { + QueryContext(context.Context, string, ...any) (*sql.Rows, error) + QueryRowContext(context.Context, string, ...any) *sql.Row +} + +func sqliteQuote(name string) string { return `"` + strings.ReplaceAll(name, `"`, `""`) + `"` } + +// Normalize only for schema comparison, never for execution or query rewriting. +// Keep quoted content intact so different defaults/check literals cannot match. +func normalizedSQLiteDDL(text string) string { + var out strings.Builder + var quote rune + for _, r := range text { + if quote != 0 { + out.WriteRune(r) + if r == quote { + quote = 0 + } + continue + } + if r == '\'' || r == '"' || r == '`' { + quote = r + out.WriteRune(r) + continue + } + if !unicode.IsSpace(r) { + out.WriteRune(unicode.ToUpper(r)) + } + } + return out.String() +} + +func sqliteObjects(q sqliteSchemaQueryer) (map[string]string, error) { + rows, err := q.QueryContext(context.Background(), `SELECT type,name,sql FROM sqlite_schema WHERE sql IS NOT NULL`) + if err != nil { + return nil, err + } + defer rows.Close() + out := map[string]string{} + for rows.Next() { + var kind, name, ddl string + if err := rows.Scan(&kind, &name, &ddl); err != nil { + return nil, err + } + if kind == "table" && (name == "schema_version" || name == "sqlite_sequence" || name == "sqlite_stat1" || name == "sqlite_stat2" || name == "sqlite_stat3" || name == "sqlite_stat4") { + continue + } + out[kind+":"+name] = normalizedSQLiteDDL(ddl) + } + return out, rows.Err() +} + +// Legacy account schemas are defined by the original append-only migrations. +// Refuse unsupported extensions rather than lose their columns or constraints +// during a table rebuild. Current versions never repeat the identity migration. +func validateSQLiteLayout(q sqliteSchemaQueryer, version int) error { + if version == 6 { + // This fork shipped identity v6 before upstream shipped companion v6. + // Accept only either complete known layout, never a partial mixture. + if err := validateSQLiteLayoutVariant(q, 5, 6); err == nil { + return nil + } + return validateSQLiteLayoutVariant(q, 6, 0) + } + identities := 0 + if version == SQLiteSchemaVersion { + identities = SQLiteSchemaVersion + } + return validateSQLiteLayoutVariant(q, version, identities) +} + +func validateSQLiteLayoutVariant(q sqliteSchemaQueryer, version, identities int) error { + expected, err := sql.Open("sqlite", ":memory:") + if err != nil { + return err + } + defer expected.Close() + expected.SetMaxOpenConns(1) + legacyVersion := version + if legacyVersion > len(legacyMigrations) { + legacyVersion = len(legacyMigrations) + } + for _, migration := range legacyMigrations[:legacyVersion] { + for _, statement := range migration { + if _, err := expected.Exec(statement); err != nil { + return err + } + } + } + want, err := sqliteObjects(expected) + if err != nil { + return err + } + if identities != 0 { + for _, table := range sqliteIdentities { + body := table.body + if identities == 6 && table.name == "sessions" { + body = body[:strings.Index(body, ", kind TEXT")] + ")" + } + want["table:"+table.name] = normalizedSQLiteDDL(`CREATE TABLE ` + table.name + body) + } + } + got, err := sqliteObjects(q) + if err != nil { + return err + } + if len(got) != len(want) { + return errors.New("users: unsupported SQLite account schema; preserve the database and review its extra or missing objects before upgrading") + } + for name, ddl := range want { + actual, ok := got[name] + if !ok { + return errors.New("users: incomplete SQLite account schema") + } + if identities != 0 && strings.HasPrefix(name, "table:") { + tableName := strings.TrimPrefix(name, "table:") + quoted := strings.Replace(ddl, "CREATETABLE"+strings.ToUpper(tableName)+"(", "CREATETABLE"+sqliteQuote(tableName)+"(", 1) + if actual == quoted { + continue + } + } + if actual == ddl { + continue + } + // Accept an already non-reusing allocator for a known INTEGER PK, and + // preserve its sequence high water. Hidden mail-event rowids are v1-v5. + safeIdentity := name == "table:users" || name == "table:sessions" || name == "table:audit_log" || name == "table:mail_log" || name == "table:proposals" + if safeIdentity && actual == strings.Replace(ddl, "IDINTEGERPRIMARYKEY", "IDINTEGERPRIMARYKEYAUTOINCREMENT", 1) { + continue + } + return errors.New("users: unsupported SQLite account columns or constraints; the original database was not changed") + } + return nil +} + +// ApplySQLite atomically upgrades known SQLite account layouts through version 7. +// A reserved connection keeps FK settings local; BEGIN IMMEDIATE serializes +// competing migration/writer processes before any read-modify-write work. +func ApplySQLite(db *sql.DB) (err error) { + if err := dbconfig.AssertSQLiteImportComplete(db); err != nil { + return err + } + var v int + if e := db.QueryRow(`SELECT version FROM schema_version`).Scan(&v); e == nil && v == SQLiteSchemaVersion { + return AssertSQLiteReady(db) + } + ctx := context.Background() + conn, err := db.Conn(ctx) + if err != nil { + return err + } + defer conn.Close() + var foreignKeys int + if err := conn.QueryRowContext(ctx, `PRAGMA foreign_keys`).Scan(&foreignKeys); err != nil { + return err + } + if _, err := conn.ExecContext(ctx, `PRAGMA foreign_keys=OFF`); err != nil { + return err + } + defer func() { + if _, restore := conn.ExecContext(ctx, fmt.Sprintf(`PRAGMA foreign_keys=%d`, foreignKeys)); restore != nil { + _ = conn.Raw(func(any) error { return driver.ErrBadConn }) + err = errors.Join(err, errors.New("users: cannot restore SQLite connection foreign-key enforcement")) + } + }() + if _, err := conn.ExecContext(ctx, `BEGIN IMMEDIATE`); err != nil { + return err + } + defer conn.ExecContext(ctx, `ROLLBACK`) + var hasVersion int + if err := conn.QueryRowContext(ctx, `SELECT count(*) FROM sqlite_schema WHERE type='table' AND name='schema_version'`).Scan(&hasVersion); err != nil { + return err + } + if hasVersion == 0 { + objects, err := sqliteObjects(conn) + if err != nil { + return err + } + if len(objects) != 0 { + return errors.New("users: refusing to initialize accounts in an existing unrelated SQLite database") + } + if _, err := conn.ExecContext(ctx, `CREATE TABLE schema_version(version INTEGER NOT NULL); INSERT INTO schema_version VALUES(0)`); err != nil { + return err + } + } + var count int + if err := conn.QueryRowContext(ctx, `SELECT count(*),COALESCE(MAX(version),0) FROM schema_version`).Scan(&count, &v); err != nil { + return err + } + // The historical initializer committed CREATE before the version-0 row. + // Recover only its empty layout, never reinterpret occupied data as fresh. + if count == 0 { + if err := validateSQLiteLayout(conn, 0); err != nil { + return err + } + if _, err := conn.ExecContext(ctx, `INSERT INTO schema_version(version) VALUES(0)`); err != nil { + return err + } + count = 1 + } + if count != 1 || v < 0 || v > SQLiteSchemaVersion { + return errors.New("users: invalid or unsupported SQLite account schema version") + } + if v == SQLiteSchemaVersion { + if err := assertSQLiteReady(conn); err != nil { + return err + } + _, err := conn.ExecContext(ctx, `COMMIT`) + return err + } + if err := validateSQLiteLayout(conn, v); err != nil { + return err + } + if v == 6 { + var linked int + if err := conn.QueryRowContext(ctx, `SELECT count(*) FROM sqlite_schema WHERE type='table' AND name='companion_links'`).Scan(&linked); err != nil { + return err + } + if linked == 0 { + v = 5 // exact native identity v6; it still needs upstream companion v6 + } + } + for i := v; i < len(legacyMigrations); i++ { + for _, statement := range legacyMigrations[i] { + if _, err := conn.ExecContext(ctx, statement); err != nil { + return fmt.Errorf("users: SQLite migration %d: %w", i+1, err) + } + } + } + high, err := sqliteAccountHighWater(conn) + if err != nil { + return err + } + indexRows, err := conn.QueryContext(ctx, `SELECT sql FROM sqlite_schema WHERE type='index' AND sql IS NOT NULL + AND tbl_name IN ('users','sessions','audit_log','mail_log','mail_events','proposals') ORDER BY name`) + if err != nil { + return err + } + var indexes []string + for indexRows.Next() { + var statement string + if err := indexRows.Scan(&statement); err != nil { + indexRows.Close() + return err + } + indexes = append(indexes, statement) + } + indexErr := indexRows.Err() + indexRows.Close() + if indexErr != nil { + return indexErr + } + for _, table := range sqliteIdentities { + temporary := "corescope_identity_" + table.name + if _, err := conn.ExecContext(ctx, `CREATE TABLE `+sqliteQuote(temporary)+table.body); err != nil { + return err + } + columns := table.columns + sourceColumns := columns + if table.name == "mail_events" { + sourceColumns = "rowid," + strings.TrimPrefix(columns, "id,") + } + if _, err := conn.ExecContext(ctx, `INSERT INTO `+sqliteQuote(temporary)+` (`+columns+`) SELECT `+sourceColumns+` FROM `+sqliteQuote(table.name)); err != nil { + return err + } + if _, err := conn.ExecContext(ctx, `DROP TABLE `+sqliteQuote(table.name)); err != nil { + return err + } + if _, err := conn.ExecContext(ctx, `ALTER TABLE `+sqliteQuote(temporary)+` RENAME TO `+sqliteQuote(table.name)); err != nil { + return err + } + res, err := conn.ExecContext(ctx, `UPDATE sqlite_sequence SET seq=?2 WHERE name=?1`, table.name, high[table.name]) + if err != nil { + return err + } + n, err := res.RowsAffected() + if err != nil { + return err + } + if n == 0 { + if _, err := conn.ExecContext(ctx, `INSERT INTO sqlite_sequence(name,seq) VALUES(?1,?2)`, table.name, high[table.name]); err != nil { + return err + } + } + } + // Recreate the exact declared indexes lost when their owning tables changed. + for _, statement := range indexes { + if _, err := conn.ExecContext(ctx, statement); err != nil { + return err + } + } + rows, err := conn.QueryContext(ctx, `PRAGMA foreign_key_check`) + if err != nil { + return err + } + invalid := rows.Next() + rowErr := rows.Err() + rows.Close() + if rowErr != nil { + return rowErr + } + if invalid { + return errors.New("users: SQLite account identity migration found invalid foreign keys; all changes were rolled back") + } + if _, err := conn.ExecContext(ctx, `UPDATE schema_version SET version=?1`, SQLiteSchemaVersion); err != nil { + return err + } + if err := assertSQLiteReady(conn); err != nil { + return err + } + _, err = conn.ExecContext(ctx, `COMMIT`) + return err +} + +func sqliteAccountHighWater(q sqliteSchemaQueryer) (map[string]int64, error) { + ctx := context.Background() + high := map[string]int64{} + var hasSequence int + if err := q.QueryRowContext(ctx, `SELECT count(*) FROM sqlite_schema WHERE name='sqlite_sequence' AND type='table'`).Scan(&hasSequence); err != nil { + return nil, err + } + if hasSequence != 0 { + rows, err := q.QueryContext(ctx, `SELECT name,MAX(seq),count(*) FROM sqlite_sequence GROUP BY name`) + if err != nil { + return nil, err + } + for rows.Next() { + var name string + var seq int64 + var count int + if err := rows.Scan(&name, &seq, &count); err != nil { + rows.Close() + return nil, err + } + if count != 1 || seq < 0 { + rows.Close() + return nil, errors.New("users: invalid SQLite account sequence state") + } + high[name] = seq + } + err = rows.Err() + rows.Close() + if err != nil { + return nil, err + } + } + for _, table := range sqliteIdentities { + column := "id" + if table.name == "mail_events" { + column = "rowid" + } + var n int64 + if err := q.QueryRowContext(ctx, `SELECT COALESCE(MAX(`+sqliteQuote(column)+`),0) FROM `+sqliteQuote(table.name)).Scan(&n); err != nil { + return nil, err + } + if n > high[table.name] { + high[table.name] = n + } + } + for _, ref := range []struct{ target, table, column string }{ + {"users", "audit_log", "actor_user_id"}, {"users", "audit_log", "target_user_id"}, {"users", "users", "activated_by"}, + {"users", "sessions", "user_id"}, {"users", "tokens", "user_id"}, {"users", "mail_log", "user_id"}, + {"users", "proposals", "proposer_id"}, {"users", "proposals", "reviewer_id"}, + {"users", "user_settings", "user_id"}, {"users", "notification_prefs", "user_id"}, + {"users", "notification_watches", "user_id"}, {"users", "notification_state", "user_id"}, {"mail_log", "mail_events", "mail_id"}, + {"users", "companion_links", "user_id"}, {"users", "link_challenges", "user_id"}, + } { + var n int64 + if err := q.QueryRowContext(ctx, `SELECT COALESCE(MAX(`+sqliteQuote(ref.column)+`),0) FROM `+sqliteQuote(ref.table)).Scan(&n); err != nil { + return nil, err + } + if n > high[ref.target] { + high[ref.target] = n + } + } + return high, nil +} + +func AssertSQLiteReady(db *sql.DB) error { + if err := dbconfig.AssertSQLiteImportComplete(db); err != nil { + return err + } + return assertSQLiteReady(db) +} + +func assertSQLiteReady(q sqliteSchemaQueryer) error { + ctx := context.Background() + var count, version int + if err := q.QueryRowContext(ctx, `SELECT count(*),COALESCE(MAX(version),0) FROM schema_version`).Scan(&count, &version); err != nil || count != 1 || version != SQLiteSchemaVersion { + return errors.New("users: SQLite account schema needs a supported upgrade") + } + return validateSQLiteLayout(q, SQLiteSchemaVersion) +} + +// ValidateSQLiteSourceLayout checks complete historical layouts without changing +// the source. The importer uses the same strict v6 disambiguation as runtime. +func ValidateSQLiteSourceLayout(db *sql.DB) error { + var count, version int + if err := db.QueryRow(`SELECT count(*),COALESCE(MAX(version),0) FROM schema_version`).Scan(&count, &version); err != nil || count != 1 || version < 0 || version > SQLiteSchemaVersion { + return errors.New("users: invalid or unsupported SQLite account schema version") + } + return validateSQLiteLayout(db, version) +} diff --git a/internal/users/stats.go b/internal/users/stats.go index d6b22731c..6093f62e9 100644 --- a/internal/users/stats.go +++ b/internal/users/stats.go @@ -1,6 +1,9 @@ package users -import "time" +import ( + "github.com/meshcore-analyzer/dbconfig" + "time" +) // Admin overview thresholds (docs/specs/2026-10-07-admin-dashboard-design.md). // Constants in this version; they belong in config later (AGENTS.md rule 8). @@ -61,18 +64,18 @@ func (s *Store) Stats(now time.Time) (Stats, error) { t := unix(now) var st Stats if err := s.db.QueryRow(`SELECT COUNT(*), - COALESCE(SUM(status = 'active'), 0), - COALESCE(SUM(status = 'pending'), 0), - COALESCE(SUM(status = 'disabled'), 0), - COALESCE(SUM(role = 'admin' AND status = 'active'), 0), - COALESCE(SUM(status = 'pending' AND created_at < ?), 0), - COALESCE(SUM(email_bouncing != 0), 0) + COUNT(*) FILTER (WHERE status = 'active'), + COUNT(*) FILTER (WHERE status = 'pending'), + COUNT(*) FILTER (WHERE status = 'disabled'), + COUNT(*) FILTER (WHERE role = 'admin' AND status = 'active'), + COUNT(*) FILTER (WHERE status = 'pending' AND created_at < `+s.p(1)+`), + COUNT(*) FILTER (WHERE email_bouncing != 0) FROM users`, t-secs(StuckPendingAfter)). Scan(&st.Total, &st.Active, &st.Pending, &st.Disabled, &st.Admins, &st.StuckPending, &st.Bouncing); err != nil { return Stats{}, err } - if err := s.db.QueryRow(`SELECT COALESCE(SUM(at >= ?), 0), COUNT(*) FROM audit_log - WHERE action = 'user.register' AND at >= ? AND at <= ?`, t-secs(7*day), t-secs(30*day), t). + if err := s.db.QueryRow(`SELECT COUNT(*) FILTER (WHERE at >= `+s.p(1)+`), COUNT(*) FROM audit_log + WHERE action = 'user.register' AND at >= `+s.p(2)+` AND at <= `+s.p(3), t-secs(7*day), t-secs(30*day), t). Scan(&st.New7d, &st.New30d); err != nil { return Stats{}, err } @@ -81,16 +84,20 @@ func (s *Store) Stats(now time.Time) (Stats, error) { return Stats{}, err } st.NewPerDay = perDay - if err := s.db.QueryRow(`SELECT COALESCE(SUM(last >= ?), 0), COUNT(*) FROM ( - SELECT MAX(COALESCE(u.last_login_at, 0), + greatest := "GREATEST" + if s.backend == dbconfig.SQLite { + greatest = "MAX" + } + if err := s.db.QueryRow(`SELECT COUNT(*) FILTER (WHERE last >= `+s.p(1)+`), COUNT(*) FROM ( + SELECT `+greatest+`(COALESCE(u.last_login_at, 0), COALESCE((SELECT MAX(last_seen_at) FROM sessions s WHERE s.user_id = u.id), 0)) AS last FROM users u - ) WHERE last >= ?`, t-secs(7*day), t-secs(30*day)). + ) activity WHERE last >= `+s.p(2), t-secs(7*day), t-secs(30*day)). Scan(&st.Active7d, &st.Active30d); err != nil { return Stats{}, err } - if err := s.db.QueryRow(`SELECT COALESCE(SUM(action = 'user.login'), 0), COALESCE(SUM(action = 'user.login.failed'), 0) - FROM audit_log WHERE action IN ('user.login', 'user.login.failed') AND at >= ?`, t-secs(day)). + if err := s.db.QueryRow(`SELECT COUNT(*) FILTER (WHERE action = 'user.login'), COUNT(*) FILTER (WHERE action = 'user.login.failed') + FROM audit_log WHERE action IN ('user.login', 'user.login.failed') AND at >= `+s.p(1), t-secs(day)). Scan(&st.Logins24h, &st.FailedLogins24h); err != nil { return Stats{}, err } @@ -107,8 +114,8 @@ func (s *Store) Stats(now time.Time) (Stats, error) { // statsPerDayDays days, today included. func (s *Store) registrationsPerDay(now time.Time) ([]DayCount, error) { first := now.UTC().Truncate(24*time.Hour).AddDate(0, 0, -(statsPerDayDays - 1)) - rows, err := s.db.Query(`SELECT (at - ?) / 86400, COUNT(*) FROM audit_log - WHERE action = 'user.register' AND at >= ? AND at <= ? GROUP BY 1`, unix(first), unix(first), unix(now)) + rows, err := s.db.Query(`SELECT (at - `+s.p(1)+`) / 86400, COUNT(*) FROM audit_log + WHERE action = 'user.register' AND at >= `+s.p(2)+` AND at <= `+s.p(3)+` GROUP BY 1`, unix(first), unix(first), unix(now)) if err != nil { return nil, err } @@ -134,7 +141,7 @@ func (s *Store) registrationsPerDay(now time.Time) ([]DayCount, error) { func (s *Store) mailCounts(since int64) (MailCounts, error) { var mc MailCounts - rows, err := s.db.Query(`SELECT last_event, COUNT(*) FROM mail_log WHERE sent_at >= ? GROUP BY last_event`, since) + rows, err := s.db.Query(`SELECT last_event, COUNT(*) FROM mail_log WHERE sent_at >= `+s.p(1)+` GROUP BY last_event`, since) if err != nil { return mc, err } @@ -166,8 +173,8 @@ func (s *Store) mailCounts(since int64) (MailCounts, error) { func (s *Store) guessedAccounts(since int64) ([]GuessedAccount, error) { rows, err := s.db.Query(`SELECT a.target_user_id, COALESCE(u.display_name, ''), COUNT(*) AS n FROM audit_log a LEFT JOIN users u ON u.id = a.target_user_id - WHERE a.action = 'user.login.failed' AND a.at >= ? AND a.target_user_id IS NOT NULL - GROUP BY a.target_user_id HAVING n >= ? ORDER BY n DESC, a.target_user_id LIMIT ?`, + WHERE a.action = 'user.login.failed' AND a.at >= `+s.p(1)+` AND a.target_user_id IS NOT NULL + GROUP BY a.target_user_id,u.display_name HAVING COUNT(*) >= `+s.p(2)+` ORDER BY n DESC, a.target_user_id LIMIT `+s.p(3), since, GuessingThreshold, statsGuessingLimit) if err != nil { return nil, err diff --git a/internal/users/store.go b/internal/users/store.go index a01ca3e7b..d64100392 100644 --- a/internal/users/store.go +++ b/internal/users/store.go @@ -1,20 +1,18 @@ -// Package users is CoreScope's optional account store (user management, -// docs/specs/2026-10-06-user-management-design.md). It owns users.db, a -// SQLite file separate from the measurement database. cmd/server never -// writes measurement data (#1283); this package is the single, opt-in -// exception, and Open refuses to touch the measurement database by path. +// Package users owns optional account state, separate from telemetry. One +// account implementation uses native SQLite or PostgreSQL storage (#1283). package users import ( + "context" "database/sql" "errors" - "fmt" - "os" - "path/filepath" "strings" "time" - _ "modernc.org/sqlite" + "github.com/jackc/pgx/v5/pgconn" + "github.com/meshcore-analyzer/dbconfig" + "github.com/meshcore-analyzer/pgutil" + "modernc.org/sqlite" ) var ( @@ -37,61 +35,121 @@ var ( ErrChallengeMismatch = errors.New("users: link challenge bound to another user or pubkey") ) -// noLimit is SQLite's LIMIT value for "every row" (the account export). -const noLimit = -1 - -// Store is the users.db handle. Safe for concurrent use. +// Store owns a separate account database. Safe for concurrent use. type Store struct { - db *sql.DB - now func() time.Time + db *sql.DB + target string + backend dbconfig.Backend + now func() time.Time } -// Open opens (creating if needed) the users database at path and applies -// pending migrations. forbidden lists paths Open must refuse; the server -// passes the measurement DB path so a misconfigured dbPath can never turn -// this package into a writer of measurement data. -func Open(path string, forbidden ...string) (*Store, error) { - if strings.ContainsAny(path, "?#") { - return nil, fmt.Errorf("users: database path %q must not contain '?' or '#'", path) +// OpenStorage consumes the resolved installation choice. Missing PostgreSQL +// settings never cause a SQLite fallback, and inactive settings are ignored. +func OpenStorage(storage dbconfig.Storage) (*Store, error) { + switch storage.Backend { + case dbconfig.SQLite: + return openSQLiteMode(storage.UsersDBPath, false, storage.DBPath) + case dbconfig.Postgres: + return openPostgres(storage.UsersDatabaseURL, storage.ReaderDatabaseURL, storage.WriterDatabaseURL) + default: + return nil, errors.New("users: account storage backend must be selected") } - abs, err := filepath.Abs(path) - if err != nil { - return nil, fmt.Errorf("users: resolve %s: %w", path, err) +} + +// Open retains compatibility for an unambiguous SQLite path or PostgreSQL URL. +// Application startup should pass its resolved choice to OpenStorage instead. +func Open(target string, forbidden ...string) (*Store, error) { + if strings.HasPrefix(strings.ToLower(target), "postgres:") || strings.HasPrefix(strings.ToLower(target), "postgresql:") { + return openPostgres(target, forbidden...) } - for _, f := range forbidden { - if strings.TrimSpace(f) != "" && samePath(abs, f) { - return nil, fmt.Errorf("users: refusing to open %s: it is the measurement database", path) - } + return openSQLite(target, forbidden...) +} + +// openPostgres connects using the account runtime role, never applying migrations. +// Every forbidden URL is checked against the effective PostgreSQL database, +// so alternate hostnames, credentials or search paths cannot bypass isolation. +func openPostgres(databaseURL string, forbidden ...string) (*Store, error) { + if _, err := pgutil.ParseConfig(databaseURL); err != nil { + return nil, err } - if err := os.MkdirAll(filepath.Dir(abs), 0o755); err != nil { - return nil, fmt.Errorf("users: create dir for %s: %w", path, err) + for _, other := range forbidden { + if strings.TrimSpace(other) == "" { + continue + } + same, err := pgutil.SameDatabase(context.Background(), databaseURL, other) + if err != nil { + return nil, err + } + if same { + return nil, errors.New("users: refusing to open the measurement database") + } } - db, err := sql.Open("sqlite", abs+"?_pragma=busy_timeout(5000)&_pragma=journal_mode(WAL)&_pragma=foreign_keys(1)") + db, err := pgutil.Open(databaseURL, false) if err != nil { - return nil, fmt.Errorf("users: open %s: %w", path, err) + return nil, err } - // Account traffic is tiny; one connection removes SQLITE_BUSY between - // our own writers and keeps the per-connection pragmas in force. + // Account traffic is small. Keep baseline per-process concurrency while + // database locks also protect transactions across independent processes. db.SetMaxOpenConns(1) - s := &Store{db: db, now: time.Now} - if err := s.migrate(); err != nil { + db.SetMaxIdleConns(1) + if err := AssertReady(db); err != nil { + db.Close() + return nil, err + } + if err := assertRuntimePrivileges(db); err != nil { db.Close() return nil, err } - return s, nil + return &Store{db: db, target: databaseURL, backend: dbconfig.Postgres, now: time.Now}, nil } -func samePath(abs, other string) bool { - oa, err := filepath.Abs(other) +// Backend identifies the native snapshot/query format without exposing targets. +func (s *Store) Backend() dbconfig.Backend { return s.backend } + +func (s *Store) p(position int) string { return s.backend.Parameter(position) } + +func (s *Store) noLimit() any { + if s.backend == dbconfig.SQLite { + return -1 + } + return nil +} + +func (s *Store) forUpdate() string { + if s.backend == dbconfig.Postgres { + return " FOR UPDATE" + } + return "" +} + +func assertRuntimePrivileges(db *sql.DB) error { + var elevated bool + err := db.QueryRow(`SELECT + EXISTS (SELECT 1 FROM pg_roles WHERE rolname=current_user AND (rolsuper OR rolcreaterole OR rolcreatedb OR rolreplication OR rolbypassrls)) + OR has_database_privilege(current_database(),'CREATE') + OR has_schema_privilege(current_schema(),'CREATE') + OR EXISTS (SELECT 1 FROM pg_class c JOIN pg_namespace n ON c.relnamespace=n.oid + WHERE n.nspname=current_schema() AND pg_has_role(c.relowner,'USAGE')) + OR has_table_privilege('corescope_schema','INSERT,UPDATE,DELETE,TRUNCATE,TRIGGER') + OR has_table_privilege('schema_version','INSERT,UPDATE,DELETE,TRUNCATE,TRIGGER')`).Scan(&elevated) if err != nil { - return false + return errors.New("users: cannot verify account runtime role privileges") } - a, errA := os.Stat(abs) - b, errB := os.Stat(oa) - if errA == nil && errB == nil { - return os.SameFile(a, b) + if elevated { + return errors.New("users: account runtime role must be separate from the migration owner and cannot modify schema metadata") } - return strings.EqualFold(filepath.Clean(abs), filepath.Clean(oa)) + return nil +} + +// lockUser serializes account-level read/modify/write operations, including +// the initial insert where no settings, tokens or watch rows exist yet. +func (s *Store) lockUser(tx *sql.Tx, id int64) error { + var found int64 + err := tx.QueryRow(`SELECT id FROM users WHERE id=`+s.p(1)+s.forUpdate(), id).Scan(&found) + if errors.Is(err, sql.ErrNoRows) { + return ErrNotFound + } + return err } // Close closes the database. @@ -135,5 +193,10 @@ func expectOne(res sql.Result, err error) error { } func isUniqueViolation(err error) bool { - return err != nil && strings.Contains(err.Error(), "UNIQUE constraint failed") + var pgerr *pgconn.PgError + if errors.As(err, &pgerr) { + return pgerr.Code == "23505" + } + var sqliteErr *sqlite.Error + return errors.As(err, &sqliteErr) && (sqliteErr.Code() == 2067 || sqliteErr.Code() == 1555) } diff --git a/internal/users/store_test.go b/internal/users/store_test.go index 0db2c9950..fcb130ebb 100644 --- a/internal/users/store_test.go +++ b/internal/users/store_test.go @@ -1,72 +1,110 @@ package users import ( - "os" + "github.com/meshcore-analyzer/dbconfig" + "net/url" "path/filepath" "strings" + "sync" "testing" ) func TestOpenCreatesSchemaAndIsIdempotent(t *testing.T) { - path := filepath.Join(t.TempDir(), "sub", "users.db") - st, err := Open(path) - if err != nil { - t.Fatalf("first Open: %v", err) - } - v, err := st.SchemaVersion() - if err != nil || v != len(migrations) { - t.Fatalf("SchemaVersion = %d, %v; want %d", v, err, len(migrations)) - } - st.Close() - - st2, err := Open(path) - if err != nil { - t.Fatalf("second Open: %v", err) + dsn := testTarget(t, false) + db := testOwner(t, dsn) + if err := applyTestSchema(t, db); err != nil { + t.Fatal(err) } - defer st2.Close() - if v2, _ := st2.SchemaVersion(); v2 != len(migrations) { - t.Fatalf("version after reopen = %d", v2) + runtimeDSN := testRuntimeURL(t, dsn) + for i := 0; i < 2; i++ { + st, err := Open(runtimeDSN) + if err != nil { + t.Fatal(err) + } + if v, err := st.SchemaVersion(); err != nil || v != schemaVersion(t) { + t.Fatalf("schema=%d,%v", v, err) + } + st.Close() + if err := applyTestSchema(t, db); err != nil { + t.Fatal(err) + } } } func TestOpenRefusesForbiddenPath(t *testing.T) { - dir := t.TempDir() - measurement := filepath.Join(dir, "meshcore.db") - if err := os.WriteFile(measurement, []byte{}, 0o644); err != nil { - t.Fatal(err) - } - _, err := Open(measurement, measurement) - if err == nil || !strings.Contains(err.Error(), "measurement database") { - t.Fatalf("Open(measurement) err = %v; want refusal", err) + dsn := testTarget(t, false) + alias, _ := url.Parse(dsn) + if testBackend(t) == dbconfig.SQLite { + if st, err := Open(dsn, filepath.Join(filepath.Dir(dsn), ".", filepath.Base(dsn))); err == nil { + st.Close() + t.Fatal("same SQLite target accepted") + } + return } - // A relative spelling of the same file is refused too. - wd, _ := os.Getwd() - defer os.Chdir(wd) - os.Chdir(dir) - if _, err := Open("meshcore.db", measurement); err == nil { - t.Fatal("relative path to the measurement DB was not refused") + q := alias.Query() + q.Set("application_name", "different-client") + q.Set("search_path", "public") + alias.RawQuery = q.Encode() + if st, err := Open(dsn, alias.String()); err == nil || !strings.Contains(err.Error(), "measurement database") { + if st != nil { + st.Close() + } + t.Fatal("the same effective database was not refused") } } func TestOpenRejectsNewerSchema(t *testing.T) { - path := filepath.Join(t.TempDir(), "users.db") - st, err := Open(path) - if err != nil { + dsn := testTarget(t, false) + db := testOwner(t, dsn) + if err := applyTestSchema(t, db); err != nil { t.Fatal(err) } - if _, err := st.db.Exec(`UPDATE schema_version SET version = 999`); err != nil { + runtimeDSN := testRuntimeURL(t, dsn) + if _, err := db.Exec(`UPDATE schema_version SET version=999`); err != nil { t.Fatal(err) } - st.Close() - if _, err := Open(path); err == nil || !strings.Contains(err.Error(), "newer") { - t.Fatalf("Open with newer schema err = %v", err) + if st, err := Open(runtimeDSN); err == nil { + if st != nil { + st.Close() + } + t.Fatalf("Open with newer schema error=%v", err) + } + if err := applyTestSchema(t, db); err == nil { + t.Fatal("bootstrap accepted a newer schema") } } func TestOpenRejectsDSNCharacters(t *testing.T) { - for _, p := range []string{"users?.db", "users#1.db"} { - if _, err := Open(filepath.Join(t.TempDir(), p)); err == nil { - t.Errorf("Open(%q) accepted", p) + for _, dsn := range []string{"", "file:users.db", "postgresql://u:p@host/a?host=elsewhere", "postgresql://u:p@host1,host2/a"} { + if st, err := Open(dsn); err == nil { + st.Close() + t.Fatal("unsupported database address accepted") } } } + +func TestApplyConcurrentAndIncomplete(t *testing.T) { + requirePostgres(t) + dsn := testTarget(t, false) + db := testOwner(t, dsn) + var wg sync.WaitGroup + for i := 0; i < 4; i++ { + wg.Add(1) + go func() { + defer wg.Done() + if err := applyTestSchema(t, db); err != nil { + t.Errorf("concurrent bootstrap: %v", err) + } + }() + } + wg.Wait() + if _, err := db.Exec(`UPDATE corescope_schema SET ready=false`); err != nil { + t.Fatal(err) + } + if err := AssertReady(db); err == nil { + t.Fatal("incomplete import accepted") + } + if err := applyTestSchema(t, db); err == nil { + t.Fatal("bootstrap completed an unverified import") + } +} diff --git a/internal/users/users.go b/internal/users/users.go index 46e20ee91..9d7864edf 100644 --- a/internal/users/users.go +++ b/internal/users/users.go @@ -3,6 +3,7 @@ package users import ( "database/sql" "errors" + "fmt" "strings" "time" ) @@ -69,18 +70,15 @@ func scanUser(row rowScanner) (*User, error) { // CreatePending inserts a new pending user. email must already be normalized. func (s *Store) CreatePending(email, displayName, passwordHash string) (*User, error) { - res, err := s.db.Exec(`INSERT INTO users (email, display_name, password_hash, role, status, created_at) - VALUES (?, ?, ?, 'user', 'pending', ?)`, email, displayName, passwordHash, unix(s.now())) + var id int64 + err := s.db.QueryRow(`INSERT INTO users (email, display_name, password_hash, role, status, created_at) + VALUES (`+s.p(1)+`, `+s.p(2)+`, `+s.p(3)+`, 'user', 'pending', `+s.p(4)+`) RETURNING id`, email, displayName, passwordHash, unix(s.now())).Scan(&id) if err != nil { if isUniqueViolation(err) { return nil, ErrEmailTaken } return nil, err } - id, err := res.LastInsertId() - if err != nil { - return nil, err - } return s.GetByID(id) } @@ -88,8 +86,8 @@ func (s *Store) CreatePending(email, displayName, passwordHash string) (*User, e // account is still pending. A caller's earlier lookup cannot authorize a write // after activation, disablement or deletion has committed. func (s *Store) UpdatePendingRegistration(id int64, passwordHash, displayName string) error { - err := expectOne(s.db.Exec(`UPDATE users SET password_hash = ?, display_name = ? - WHERE id = ? AND status = 'pending'`, passwordHash, displayName, id)) + err := expectOne(s.db.Exec(`UPDATE users SET password_hash = `+s.p(1)+`, display_name = `+s.p(2)+` + WHERE id = `+s.p(3)+` AND status = 'pending'`, passwordHash, displayName, id)) if errors.Is(err, ErrNotFound) { return ErrAccountChanged } @@ -97,40 +95,40 @@ func (s *Store) UpdatePendingRegistration(id int64, passwordHash, displayName st } func (s *Store) GetByID(id int64) (*User, error) { - return scanUser(s.db.QueryRow(`SELECT `+userCols+` FROM users WHERE id = ?`, id)) + return scanUser(s.db.QueryRow(`SELECT `+userCols+` FROM users WHERE id = `+s.p(1), id)) } // GetByEmail looks up a normalized address. func (s *Store) GetByEmail(email string) (*User, error) { - return scanUser(s.db.QueryRow(`SELECT `+userCols+` FROM users WHERE email = ?`, email)) + return scanUser(s.db.QueryRow(`SELECT `+userCols+` FROM users WHERE email = `+s.p(1), email)) } // Activate moves a pending user to active with role. by is the admin who // activated manually, nil for link activation. ErrNotFound if not pending. func (s *Store) Activate(id int64, role Role, by *int64) error { - return expectOne(s.db.Exec(`UPDATE users SET status = 'active', role = ?, activated_at = ?, activated_by = ? - WHERE id = ? AND status = 'pending'`, string(role), unix(s.now()), nullInt(by), id)) + return expectOne(s.db.Exec(`UPDATE users SET status = 'active', role = `+s.p(1)+`, activated_at = `+s.p(2)+`, activated_by = `+s.p(3)+` + WHERE id = `+s.p(4)+` AND status = 'pending'`, string(role), unix(s.now()), nullInt(by), id)) } func (s *Store) SetStatus(id int64, st Status) error { - return expectOne(s.db.Exec(`UPDATE users SET status = ? WHERE id = ?`, string(st), id)) + return expectOne(s.db.Exec(`UPDATE users SET status = `+s.p(1)+` WHERE id = `+s.p(2), string(st), id)) } func (s *Store) SetRole(id int64, r Role) error { - return expectOne(s.db.Exec(`UPDATE users SET role = ? WHERE id = ?`, string(r), id)) + return expectOne(s.db.Exec(`UPDATE users SET role = `+s.p(1)+` WHERE id = `+s.p(2), string(r), id)) } func (s *Store) SetPassword(id int64, hash string) error { - return expectOne(s.db.Exec(`UPDATE users SET password_hash = ? WHERE id = ?`, hash, id)) + return expectOne(s.db.Exec(`UPDATE users SET password_hash = `+s.p(1)+` WHERE id = `+s.p(2), hash, id)) } func (s *Store) SetDisplayName(id int64, name string) error { - return expectOne(s.db.Exec(`UPDATE users SET display_name = ? WHERE id = ?`, name, id)) + return expectOne(s.db.Exec(`UPDATE users SET display_name = `+s.p(1)+` WHERE id = `+s.p(2), name, id)) } // SetEmail changes the address (normalized) and clears the bounce flag. func (s *Store) SetEmail(id int64, email string) error { - err := expectOne(s.db.Exec(`UPDATE users SET email = ?, email_bouncing = 0 WHERE id = ?`, email, id)) + err := expectOne(s.db.Exec(`UPDATE users SET email = `+s.p(1)+`, email_bouncing = 0 WHERE id = `+s.p(2), email, id)) if isUniqueViolation(err) { return ErrEmailTaken } @@ -142,11 +140,11 @@ func (s *Store) SetEmailBouncing(id int64, v bool) error { if v { b = 1 } - return expectOne(s.db.Exec(`UPDATE users SET email_bouncing = ? WHERE id = ?`, b, id)) + return expectOne(s.db.Exec(`UPDATE users SET email_bouncing = `+s.p(1)+` WHERE id = `+s.p(2), b, id)) } func (s *Store) TouchLogin(id int64) error { - return expectOne(s.db.Exec(`UPDATE users SET last_login_at = ? WHERE id = ?`, unix(s.now()), id)) + return expectOne(s.db.Exec(`UPDATE users SET last_login_at = `+s.p(1)+` WHERE id = `+s.p(2), unix(s.now()), id)) } // Delete removes a user; sessions and tokens cascade. Mail-log rows survive @@ -158,16 +156,16 @@ func (s *Store) Delete(id int64) error { } defer tx.Rollback() var exists int - if err := tx.QueryRow(`SELECT 1 FROM users WHERE id = ?`, id).Scan(&exists); err != nil { + if err := tx.QueryRow(`SELECT 1 FROM users WHERE id = `+s.p(1)+s.forUpdate(), id).Scan(&exists); err != nil { if errors.Is(err, sql.ErrNoRows) { return ErrNotFound } return err } - if err := hashMailLogTx(tx, id); err != nil { + if err := s.hashMailLogTx(tx, id); err != nil { return err } - if _, err := tx.Exec(`DELETE FROM users WHERE id = ?`, id); err != nil { + if _, err := tx.Exec(`DELETE FROM users WHERE id = `+s.p(1), id); err != nil { return err } return tx.Commit() @@ -181,17 +179,16 @@ type ListFilter struct { Bouncing bool // only addresses whose mail bounces } -// List returns at most 1000 users, newest first. SQLite lower()/LIKE fold -// ASCII only, so a non-ASCII search is case-sensitive. +// List returns at most 1000 users, newest first, with case-insensitive search. func (s *Store) List(f ListFilter) ([]User, error) { q := `SELECT ` + userCols + ` FROM users WHERE 1=1` var args []any if f.Status != "" { - q += ` AND status = ?` + q += fmt.Sprintf(` AND status = %s`, s.p(len(args)+1)) args = append(args, string(f.Status)) } if f.Role != "" { - q += ` AND role = ?` + q += fmt.Sprintf(` AND role = %s`, s.p(len(args)+1)) args = append(args, string(f.Role)) } if f.Bouncing { @@ -199,7 +196,7 @@ func (s *Store) List(f ListFilter) ([]User, error) { } if t := strings.TrimSpace(f.Query); t != "" { like := "%" + escapeLike(strings.ToLower(t)) + "%" - q += ` AND (email LIKE ? ESCAPE '\' OR lower(display_name) LIKE ? ESCAPE '\')` + q += fmt.Sprintf(` AND (email LIKE %s ESCAPE '\' OR lower(display_name) LIKE %s ESCAPE '\')`, s.p(len(args)+1), s.p(len(args)+2)) args = append(args, like, like) } q += ` ORDER BY created_at DESC, id DESC LIMIT 1000` @@ -241,7 +238,7 @@ func (s *Store) UsersByID(ids []int64) (map[int64]User, error) { for i, id := range ids { args[i] = id } - ph := placeholders(len(ids)) + ph := s.placeholders(len(ids)) rows, err := s.db.Query(`SELECT `+userCols+` FROM users WHERE id IN (`+ph+`)`, args...) if err != nil { return nil, err @@ -266,9 +263,9 @@ func (s *Store) PruneStalePending(maxAge time.Duration) (int64, error) { return 0, err } defer tx.Rollback() - rows, err := tx.Query(`SELECT id FROM users WHERE status = 'pending' AND created_at < ? + rows, err := tx.Query(`SELECT id FROM users WHERE status = 'pending' AND created_at < `+s.p(1)+` AND NOT EXISTS (SELECT 1 FROM tokens t WHERE t.user_id = users.id AND t.purpose = 'activate' - AND t.used_at IS NULL AND t.expires_at > ?)`, now-int64(maxAge/time.Second), now) + AND t.used_at IS NULL AND t.expires_at > `+s.p(2)+`) ORDER BY id`+s.forUpdate(), now-int64(maxAge/time.Second), now) if err != nil { return 0, err } @@ -285,23 +282,36 @@ func (s *Store) PruneStalePending(maxAge time.Duration) (int64, error) { if err := rows.Err(); err != nil { return 0, err } + var removed int64 for _, id := range ids { - if err := hashMailLogTx(tx, id); err != nil { + // The locking SELECT may have waited for a concurrent IssueToken. + // Recheck with a fresh READ COMMITTED snapshot after owning the user + // lock; its original NOT EXISTS snapshot cannot see that new token. + var eligible bool + if err := tx.QueryRow(`SELECT EXISTS(SELECT 1 FROM users WHERE id=`+s.p(1)+` AND status='pending' AND created_at<`+s.p(2)+` + AND NOT EXISTS(SELECT 1 FROM tokens WHERE user_id=`+s.p(1)+` AND purpose='activate' AND used_at IS NULL AND expires_at>`+s.p(3)+`))`, id, now-int64(maxAge/time.Second), now).Scan(&eligible); err != nil { + return 0, err + } + if !eligible { + continue + } + if err := s.hashMailLogTx(tx, id); err != nil { return 0, err } - if _, err := tx.Exec(`DELETE FROM users WHERE id = ?`, id); err != nil { + if _, err := tx.Exec(`DELETE FROM users WHERE id = `+s.p(1), id); err != nil { return 0, err } + removed++ } - return int64(len(ids)), tx.Commit() + return removed, tx.Commit() } // hashMailLogTx replaces the plaintext address of every mail_log row of the // user with HashedEmail of that row's own address (rows may predate an email // change). Rows already hashed are skipped. The cursor is closed before any // UPDATE because the store uses a single connection. -func hashMailLogTx(tx *sql.Tx, userID int64) error { - rows, err := tx.Query(`SELECT id, to_email FROM mail_log WHERE user_id = ? AND to_email NOT LIKE 'sha256:%'`, userID) +func (s *Store) hashMailLogTx(tx *sql.Tx, userID int64) error { + rows, err := tx.Query(`SELECT id, to_email FROM mail_log WHERE user_id = `+s.p(1)+` AND to_email NOT LIKE 'sha256:%'`, userID) if err != nil { return err } @@ -323,7 +333,7 @@ func hashMailLogTx(tx *sql.Tx, userID int64) error { return err } for _, r := range list { - if _, err := tx.Exec(`UPDATE mail_log SET to_email = ? WHERE id = ?`, HashedEmail(r.email), r.id); err != nil { + if _, err := tx.Exec(`UPDATE mail_log SET to_email = `+s.p(1)+` WHERE id = `+s.p(2), HashedEmail(r.email), r.id); err != nil { return err } } diff --git a/manage.sh b/manage.sh index 099c0fb71..a89f88cff 100755 --- a/manage.sh +++ b/manage.sh @@ -7,6 +7,7 @@ # # Idempotent: safe to cancel and re-run at any point. # Each step checks what's already done and skips it. +set +x set -e IMAGE_NAME="corescope" @@ -64,7 +65,7 @@ NC='\033[0m' log() { printf '%b\n' "${GREEN}✓${NC} $1"; } warn() { printf '%b\n' "${YELLOW}⚠${NC} $1"; } -err() { printf '%b\n' "${RED}✗${NC} $1"; } +err() { printf '%b\n' "${RED}✗${NC} $1" >&2; } info() { printf '%b\n' "${CYAN}→${NC} $1"; } step() { printf '%b\n' "\n${BOLD}[$1/$TOTAL_STEPS] $2${NC}"; } @@ -75,20 +76,239 @@ is_true() { esac } +# The base files require no PostgreSQL credentials and are also the read-only +# discovery boundary before the recorded backend is known. +dc_prod_base() { + local file=docker-compose.yml + if is_true "${DISABLE_MOSQUITTO:-false}"; then file=docker-compose.no-mosquitto.yml; fi + $DC -f "$file" "$@" +} + +dc_staging_base() { + local file="$STAGING_COMPOSE_FILE" + if is_true "${DISABLE_MOSQUITTO:-false}"; then file=docker-compose.staging.no-mosquitto.yml; fi + $DC -f "$file" -p corescope-staging "$@" +} + +dc_postgres() { + local environment="$1" file override + shift + require_database_credentials || return 1 + case "$environment" in + prod) + file=docker-compose.yml; override=docker-compose.postgres.yml + if is_true "${DISABLE_MOSQUITTO:-false}"; then file=docker-compose.no-mosquitto.yml; fi + $DC -f "$file" -f "$override" "$@" + ;; + staging) + file="$STAGING_COMPOSE_FILE"; override=docker-compose.staging.postgres.yml + if is_true "${DISABLE_MOSQUITTO:-false}"; then file=docker-compose.staging.no-mosquitto.yml; fi + $DC -f "$file" -f "$override" -p corescope-staging "$@" + ;; + *) err "Unknown storage environment"; return 1 ;; + esac +} + +storage_at() { + local environment="$1" data_directory="$2" service + local volumes=() + shift 2 + if [ -n "$data_directory" ]; then volumes=(-v "$data_directory:/app/data"); fi + case "$environment" in prod) service=prod ;; staging) service=staging-go ;; *) err "Unknown storage environment"; return 1 ;; esac + "dc_${environment}_base" run --rm --no-deps "${volumes[@]}" --entrypoint /app/storage.sh "$service" "$@" +} + +storage_action() { + local environment="$1" + shift + storage_at "$environment" "" "$@" +} + +storage_field() { + storage_action "$1" field "$2" +} + +selected_backend() { + local environment="$1" state backend job + state=$(storage_field "$environment" state) || { err "Cannot read storage selection. Build the current image with setup; keep the existing data and selection intact."; return 1; } + case "$state" in + ready) backend=$(storage_field "$environment" backend) || return 1 ;; + unrecorded) + # The packaged entrypoint performs explicit guarded setup. This hint + # selects its Compose dependencies; it never authorizes fresh data itself. + backend=${CORESCOPE_DB_BACKEND:-sqlite} + ;; + pending) + job=$(storage_field "$environment" job_id) || return 1 + err "Storage switch $job is pending; keep services stopped and use storage resume or abort."; return 1 + ;; + *) err "Storage selection is invalid; preserve it and inspect storage status."; return 1 ;; + esac + case "$backend" in sqlite|postgres) printf '%s\n' "$backend" ;; *) err "Choose sqlite or postgres during setup."; return 1 ;; esac +} + dc_prod() { - if is_true "${DISABLE_MOSQUITTO:-false}"; then - $DC -f docker-compose.no-mosquitto.yml "$@" - else - $DC "$@" - fi + local backend + backend=$(selected_backend prod) || return 1 + case "$backend" in sqlite) dc_prod_base "$@" ;; postgres) dc_postgres prod "$@" ;; esac } dc_staging() { - if is_true "${DISABLE_MOSQUITTO:-false}"; then - $DC -f docker-compose.staging.no-mosquitto.yml -p corescope-staging "$@" - else - $DC -f "$STAGING_COMPOSE_FILE" -p corescope-staging "$@" + local backend + backend=$(selected_backend staging) || return 1 + case "$backend" in sqlite) dc_staging_base "$@" ;; postgres) dc_postgres staging "$@" ;; esac +} + +require_managed_postgres() { + storage_action "$1" managed-postgres +} + +sqlite_source_exists() { + local environment="$1" kind="$2" source service + if [ "$kind" = accounts ] && [ "$(storage_field "$environment" has_accounts)" = false ]; then return 3; fi + source=$(storage_field "$environment" "$kind.sqlite_path") || return 1 + service=prod; [ "$environment" != staging ] || service=staging-go + "dc_${environment}_base" run --rm --no-deps --entrypoint /bin/sh "$service" -eu -c ' + if [ -f "$1" ]; then exit 0; fi + # A nonnil recorded target is initialized. Missing it is data loss, not + # an intentionally absent optional store, even when no sidecar survives. + exit 1 + ' sh "$source" +} + +sqlite_dump_file() { + local environment="$1" kind="$2" destination="$3" temporary="$3.partial" source directory service + case "$kind" in telemetry|accounts) ;; *) err "Unknown SQLite store"; return 1 ;; esac + [ "$(basename "$destination")" = "$kind.db" ] || { err "Use the native store filename $kind.db"; return 1; } + source=$(storage_field "$environment" "$kind.sqlite_path") || return 1 + [ ! -e "$destination" ] || { err "Backup target already exists: $destination"; return 1; } + directory=$(cd "$(dirname "$destination")" && pwd -P) || return 1 + service=prod; [ "$environment" != staging ] || service=staging-go + ( + umask 077 + set -o noclobber + : > "$temporary" || exit 1 + trap 'rm -f -- "$temporary"' EXIT HUP INT TERM + # SQLite's native backup includes committed WAL frames and preserves rowids. + # The source is read-only; a fixed output name avoids dot-command quoting. + "dc_${environment}_base" run --rm --no-deps -v "$directory:/backup" --entrypoint sqlite3 "$service" \ + -readonly -cmd '.timeout 5000' "$source" ".backup '/backup/$kind.db.partial'" || exit 1 + sync -f "$temporary" || exit 1 + ln -- "$temporary" "$destination" || exit 1 + rm -- "$temporary" || exit 1 + trap - EXIT HUP INT TERM + ) +} + +# Read the owner credential inside the database container, never from argv. +pg_container_exec() { + local environment="$1" database="$2" + shift 2 + case "$database" in corescope_telemetry|corescope_accounts) ;; *) err "Unsupported managed database target"; return 1 ;; esac + dc_postgres "$environment" exec -T postgres sh -c 'export PGHOST=127.0.0.1 PGPORT=5432 PGUSER=corescope_owner PGPASSWORD="$CORESCOPE_OWNER_PASSWORD" PGDATABASE="$1"; shift; exec "$@"' sh "$database" "$@" +} + +pg_exec() { + require_managed_postgres "$1" || return 1 + if [ "$2" = corescope_accounts ] && [ "$(storage_field "$1" has_accounts)" != true ]; then + err "No initialized account database is selected; managed account operations were refused." + return 1 + fi + pg_container_exec "$@" +} + +pg_empty() { + local count execute=pg_exec + [ "${3:-}" != fresh-staging ] || execute=pg_container_exec + count=$("$execute" "$1" "$2" psql -X -A -t -v ON_ERROR_STOP=1 -c "SELECT COUNT(*) FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname NOT IN ('pg_catalog','information_schema') AND n.nspname NOT LIKE 'pg_toast%' AND c.relkind IN ('r','p','v','m','S','f')") || return 1 + [ "$count" = 0 ] +} + +pg_dump_file() { + local environment="$1" database="$2" destination="$3" temporary="$3.partial" + if [ -e "$destination" ]; then err "Backup target already exists: $destination"; return 1; fi + ( + umask 077 + set -o noclobber + exec 3>"$temporary" || exit 1 + trap 'rm -f -- "$temporary"' EXIT HUP INT TERM + pg_exec "$environment" "$database" pg_dump --format=custom --no-password --no-owner --no-privileges >&3 || exit 1 + exec 3>&- + sync -f "$temporary" || exit 1 + ln -- "$temporary" "$destination" || exit 1 + rm -- "$temporary" || exit 1 + trap - EXIT HUP INT TERM + ) +} + +pg_restore_file() { + local environment="$1" database="$2" archive="$3" scope="${4:-}" execute=pg_exec + [ "$scope" != fresh-staging ] || execute=pg_container_exec + if [ "$(head -c 5 "$archive")" != PGDMP ]; then err "SQLite files require docs/postgresql-upgrade.md; expected a PostgreSQL archive."; return 1; fi + pg_empty "$environment" "$database" "$scope" || { err "Refusing to overwrite a nonempty PostgreSQL destination."; return 1; } + "$execute" "$environment" "$database" pg_restore --no-password --exit-on-error --single-transaction --no-owner --no-privileges --dbname="$database" < "$archive" || return 1 + "$execute" "$environment" "$database" psql -X -v ON_ERROR_STOP=1 < docker/postgres-grants.sql >/dev/null || return 1 + "$execute" "$environment" "$database" psql -X -v ON_ERROR_STOP=1 -c ANALYZE >/dev/null || return 1 +} + +require_database_credentials() { + local key value + if [ -z "$POSTGRES_ADMIN_PASSWORD" ]; then err "Set PostgreSQL credentials in .env before starting; see .env.example."; return 1; fi + for key in CORESCOPE_OWNER_PASSWORD CORESCOPE_READER_PASSWORD CORESCOPE_WRITER_PASSWORD CORESCOPE_ACCOUNTS_PASSWORD CORESCOPE_CHANNELS_PASSWORD; do + value=$(printenv "$key" || true) + if ! [[ "$value" =~ ^[a-fA-F0-9]{32,}$ ]]; then err "Set $key in .env to at least 32 hexadecimal characters; see docs/postgresql-upgrade.md."; return 1; fi + done +} + +# Update only one private bootstrap setting, preserving unrelated .env content. +write_private_env() { + local key="$1" value="$2" temporary + umask 077 + temporary=$(mktemp .env.tmp.XXXXXX) || return 1 + if [ -f .env ]; then + awk -F= -v key="$key" '$1 != key { print }' .env > "$temporary" || { rm -f -- "$temporary"; return 1; } + fi + printf '%s=%s\n' "$key" "$value" >> "$temporary" + chmod 600 "$temporary" + mv -- "$temporary" .env + export "$key=$value" +} + +prepare_database_credentials() { + local directory="${PROD_POSTGRES_DATA_DIR:-$HOME/corescope-postgres}" key value occupied=false + if [ -d "$directory" ] && [ -n "$(find "$directory" -mindepth 1 -maxdepth 1 -print -quit)" ]; then occupied=true; fi + for key in POSTGRES_ADMIN_PASSWORD CORESCOPE_OWNER_PASSWORD CORESCOPE_READER_PASSWORD CORESCOPE_WRITER_PASSWORD CORESCOPE_ACCOUNTS_PASSWORD CORESCOPE_CHANNELS_PASSWORD; do + value=$(printenv "$key" || true) + if [ -z "$value" ]; then + if $occupied; then err "Existing PostgreSQL storage needs its matching private .env; changing passwords does not rotate existing roles. Keep the volume intact."; return 1; fi + value=$(openssl rand -hex 32) || return 1 + write_private_env "$key" "$value" || return 1 + fi + done + require_database_credentials +} + +choose_storage() { + local state backend reply + state=$(storage_field prod state) || return 1 + if [ "$state" = ready ]; then + backend=$(storage_field prod backend) || return 1 + info "Keeping recorded $backend storage. Use storage switch for an explicit offline backend change." + return 0 fi + if [ "$state" != unrecorded ]; then selected_backend prod >/dev/null; return 1; fi + echo "Storage for this installation:" + echo " 1) SQLite (default; no separate database service)" + echo " 2) PostgreSQL (separate local service and private role credentials)" + read -r -p "Choose [1]: " reply + case "${reply:-1}" in + 1|sqlite) backend=sqlite ;; + 2|postgres) backend=postgres ;; + *) err "Choose 1 (SQLite) or 2 (PostgreSQL)."; return 1 ;; + esac + write_private_env CORESCOPE_DB_BACKEND "$backend" || return 1 + if [ "$backend" = postgres ]; then prepare_database_credentials || return 1; fi + info "Setup will validate existing stores before adopting them, or initialize only a provably fresh $backend installation." } confirm() { @@ -287,7 +507,7 @@ write_env_managed_values() { local seen_data=0 local seen_disable_mosquitto=0 - : > "$tmp_file" + (umask 077; set -o noclobber; : > "$tmp_file") || return 1 while IFS= read -r line || [ -n "$line" ]; do case "$line" in PROD_HTTP_PORT=*) @@ -426,11 +646,11 @@ verify_health() { fi fi - # Wait for /api/stats response (Go backend loads packets into memory — may take 60s+) - info "Waiting for server to respond..." + # Wait for initialization, not just the HTTP listener (loading can take 60s+). + info "Waiting for server initialization..." local healthy=false for i in $(seq 1 45); do - if docker exec "$container" wget -qO- http://localhost:3000/api/stats &>/dev/null; then + if docker exec "$container" wget -qO- http://localhost:3000/api/healthz &>/dev/null; then healthy=true break fi @@ -438,11 +658,11 @@ verify_health() { done if ! $healthy; then - err "Server did not respond after 90 seconds." + err "Server did not become ready after 90 seconds." warn "Check logs: ./manage.sh logs" return 1 fi - log "Server is responding." + log "Server is ready." # Check for MQTT errors in recent logs local mqtt_errors @@ -456,7 +676,7 @@ verify_health() { if $use_https; then info "Checking HTTPS for ${caddyfile_domain}..." if command -v curl &>/dev/null; then - if curl -sf --connect-timeout 5 "https://${caddyfile_domain}/api/stats" &>/dev/null; then + if curl -sf --connect-timeout 5 "https://${caddyfile_domain}/api/healthz" &>/dev/null; then log "HTTPS is working: https://${caddyfile_domain}" else warn "HTTPS not reachable yet for ${caddyfile_domain}" @@ -793,16 +1013,18 @@ cmd_setup() { if [ -n "$IMAGE_EXISTS" ] && is_done "build"; then log "Image already built." if confirm "Rebuild? (only needed if you updated the code)"; then - dc_prod build prod + dc_prod_base build prod log "Image rebuilt." fi else info "This takes 1-2 minutes the first time..." - dc_prod build prod + dc_prod_base build prod log "Image built." fi mark_done "build" + choose_storage || return 1 + # ── Step 5: Start container ── step 5 "Starting container" @@ -814,18 +1036,18 @@ cmd_setup() { fi fi - # Detect existing data directories - if [ -d "$PROD_DATA" ] && [ -f "$PROD_DATA/meshcore.db" ]; then - info "Found existing data at $PROD_DATA/ — will use bind mount." - fi - - if docker ps --format '{{.Names}}' | grep -q "^corescope-prod$"; then - log "Container already running." + local backend + backend=$(selected_backend prod) || return 1 + dc_prod_base stop prod || return 1 + if [ "$backend" = postgres ]; then + require_database_credentials || return 1 + dc_postgres prod up -d --wait postgres || return 1 + dc_postgres prod run --rm --no-deps --entrypoint /app/storage.sh bootstrap setup -backend=postgres || return 1 else - mkdir -p "$PROD_DATA" - dc_prod up -d prod - log "Container started." + storage_action prod setup -backend=sqlite || return 1 fi + dc_prod up -d prod || return 1 + log "Validated storage and application started." mark_done "container" # ── Step 6: Verify ── @@ -878,16 +1100,54 @@ cmd_setup() { # ─── Staging Helpers ────────────────────────────────────────────────────── -# Copy production DB to staging data directory +# Copy telemetry into an empty staging database; accounts stay independent. prepare_staging_db() { + local backend state path destination copy_dir attempt contents mkdir -p "$STAGING_DATA" - if [ -f "$PROD_DATA/meshcore.db" ]; then - info "Copying production database to staging..." - cp "$PROD_DATA/meshcore.db" "$STAGING_DATA/meshcore.db" 2>/dev/null || true - log "Database snapshot copied to ${STAGING_DATA}/meshcore.db" + if container_running "$STAGING_CONTAINER"; then info "Keeping running staging data."; return 0; fi + state=$(storage_field staging state) || return 1 + if [ "$state" = ready ]; then info "Keeping recorded staging storage; automatic replacement is refused."; return 0; fi + [ "$state" = unrecorded ] || { err "Staging recovery must finish before cloning data."; return 1; } + contents=$(find "$STAGING_DATA" -mindepth 1 -maxdepth 1 ! -name config.json ! -name theme.json ! -name Caddyfile ! -name .env -print -quit) + if [ -n "$contents" ]; then info "Keeping existing staging files; packaged setup will validate them before startup."; return 0; fi + backend=$(selected_backend prod) || return 1 + prepare_staging_config + copy_dir=$(mktemp -d) + chmod 700 "$copy_dir" + if [ "$backend" = sqlite ]; then + path=$(storage_field prod telemetry.sqlite_path) || return 1 + case "$path" in /app/data/*) ;; *) err "Managed staging requires telemetry inside the persistent data mount."; return 1 ;; esac + destination=$(realpath -m -- "$STAGING_DATA/${path#/app/data/}") || return 1 + case "$destination" in "$(realpath -m -- "$STAGING_DATA")"/*) ;; *) err "Staging target escaped its data directory."; return 1 ;; esac + [ ! -e "$destination" ] || { err "Staging telemetry already exists."; return 1; } + sqlite_dump_file prod telemetry "$copy_dir/telemetry.db" || return 1 + mkdir -p "$(dirname "$destination")" + cp -- "$copy_dir/telemetry.db" "$destination" || return 1 + chmod 600 "$destination" + # Only telemetry is cloned. A new account target is independently initialized. + storage_action staging setup -backend=sqlite -sqlite-path "$path" -users-sqlite-path /app/data/users.db || return 1 else - warn "No production database found at ${PROD_DATA}/meshcore.db — staging starts empty." + # Explicit fresh staging only: start the local database, never bootstrap its + # schema before checking emptiness and restoring the native telemetry dump. + dc_postgres staging up -d postgres + for attempt in $(seq 1 30); do + if pg_container_exec staging corescope_telemetry psql -X -A -t -c 'SELECT 1' >/dev/null 2>&1; then break; fi + [ "$attempt" -lt 30 ] || { err "Staging PostgreSQL did not become ready"; return 1; } + sleep 1 + done + if ! pg_empty staging corescope_telemetry fresh-staging; then + info "Keeping existing staging PostgreSQL data; validating its adoption before startup." + rmdir "$copy_dir" + dc_postgres staging run --rm bootstrap + return $? + fi + pg_dump_file prod corescope_telemetry "$copy_dir/telemetry.dump" || return 1 + pg_restore_file staging corescope_telemetry "$copy_dir/telemetry.dump" fresh-staging || return 1 + dc_postgres staging run --rm bootstrap || return 1 fi + rm -f -- "$copy_dir/telemetry.db" "$copy_dir/telemetry.dump" + rmdir -- "$copy_dir" + log "Telemetry cloned into fresh staging storage. Staging accounts remain independent." } # Copy config.prod.json → config.staging.json with siteName change @@ -1046,14 +1306,16 @@ cmd_start() { # Always check prod config ensure_config "$PROD_DATA" + # Compose completes PostgreSQL/bootstrap dependencies before staging copies data. + info "Starting production container (corescope-prod) on ports ${PROD_HTTP_PORT:-80}/${PROD_HTTPS_PORT:-443}..." + dc_prod up -d prod + if $WITH_STAGING; then # Prepare staging data and config prepare_staging_db prepare_staging_config - info "Starting production container (corescope-prod) on ports ${PROD_HTTP_PORT:-80}/${PROD_HTTPS_PORT:-443}..." info "Starting staging container (${STAGING_CONTAINER}) on port ${STAGING_GO_HTTP_PORT:-82}..." - dc_prod up -d prod dc_staging up -d staging-go if is_true "${DISABLE_MOSQUITTO:-false}"; then log "Production started on ports ${PROD_HTTP_PORT:-80}/${PROD_HTTPS_PORT:-443} (MQTT disabled)" @@ -1063,8 +1325,6 @@ cmd_start() { log "Staging started on port ${STAGING_GO_HTTP_PORT:-82} (MQTT: ${STAGING_GO_MQTT_PORT:-1885})" fi else - info "Starting production container (corescope-prod) on ports ${PROD_HTTP_PORT:-80}/${PROD_HTTPS_PORT:-443}..." - dc_prod up -d prod log "Production started. Staging NOT running (use --with-staging to start both)." fi } @@ -1075,19 +1335,19 @@ cmd_stop() { case "$TARGET" in prod) info "Stopping production container (corescope-prod)..." - dc_prod stop prod + dc_prod_base stop prod log "Production stopped." ;; staging) info "Stopping staging container (${STAGING_CONTAINER})..." - dc_staging rm -sf staging-go 2>/dev/null || true + dc_staging_base rm -sf staging-go 2>/dev/null || true docker rm -f "$STAGING_CONTAINER" meshcore-staging-go corescope-staging meshcore-staging 2>/dev/null || true log "Staging stopped and cleaned up." ;; all) info "Stopping all containers..." - dc_prod stop prod - dc_staging rm -sf staging-go 2>/dev/null || true + dc_prod_base stop prod + dc_staging_base rm -sf staging-go 2>/dev/null || true docker rm -f "$STAGING_CONTAINER" meshcore-staging-go corescope-staging meshcore-staging 2>/dev/null || true log "All containers stopped." ;; @@ -1109,7 +1369,7 @@ cmd_restart() { staging) info "Restarting staging container (${STAGING_CONTAINER})..." # Stop and remove old container - dc_staging rm -sf staging-go 2>/dev/null || true + dc_staging_base rm -sf staging-go 2>/dev/null || true docker rm -f "$STAGING_CONTAINER" 2>/dev/null || true # Wait for container to be fully gone and memory to be reclaimed # This prevents OOM when old + new containers overlap on small VMs @@ -1132,7 +1392,7 @@ cmd_restart() { all) info "Restarting all containers..." dc_prod up -d --force-recreate prod - dc_staging rm -sf staging-go 2>/dev/null || true + dc_staging_base rm -sf staging-go 2>/dev/null || true docker rm -f "$STAGING_CONTAINER" 2>/dev/null || true dc_staging up -d staging-go log "All containers restarted." @@ -1199,19 +1459,21 @@ cmd_status() { fi echo "" - # Disk usage - if [ -d "$PROD_DATA" ] && [ -f "$PROD_DATA/meshcore.db" ]; then - local db_size - db_size=$(du -h "$PROD_DATA/meshcore.db" 2>/dev/null | cut -f1) - info "Production DB: ${db_size}" - fi - if [ -d "$STAGING_DATA" ] && [ -f "$STAGING_DATA/meshcore.db" ]; then - local staging_db_size - staging_db_size=$(du -h "$STAGING_DATA/meshcore.db" 2>/dev/null | cut -f1) - info "Staging DB: ${staging_db_size}" + local backend database size target + storage_action prod status || return 1 + backend=$(selected_backend prod) || return 1 + if [ "$backend" = postgres ]; then + for database in corescope_telemetry corescope_accounts; do + if [ "$database" = corescope_accounts ] && [ "$(storage_field prod has_accounts)" = false ]; then continue; fi + if size=$(pg_exec prod "$database" psql -X -A -t -c 'SELECT pg_size_pretty(pg_database_size(current_database()))' 2>/dev/null); then info "Production $database: $size"; else warn "PostgreSQL size unavailable for $database"; fi + done + else + for database in telemetry accounts; do + target=$(storage_field prod "$database.sqlite_path") || continue + if size=$(dc_prod_base run --rm --no-deps --entrypoint /bin/sh prod -c 'stat -c %s "$1"' sh "$target" 2>/dev/null); then info "Production SQLite $database: $size bytes (main file; WAL may be additional)"; fi + done fi - echo "" } # ─── Logs ───────────────────────────────────────────────────────────────── @@ -1269,18 +1531,8 @@ cmd_promote() { exit 0 fi - # Backup production DB - info "Backing up production database..." local BACKUP_DIR="./backups/pre-promotion-$(date +%Y%m%d-%H%M%S)" - mkdir -p "$BACKUP_DIR" - if [ -f "$PROD_DATA/meshcore.db" ]; then - cp "$PROD_DATA/meshcore.db" "$BACKUP_DIR/" - elif container_running "corescope-prod"; then - docker cp corescope-prod:/app/data/meshcore.db "$BACKUP_DIR/" - else - warn "Could not backup production database." - fi - log "Backup saved to ${BACKUP_DIR}/" + cmd_backup "$BACKUP_DIR" # Restart prod with latest image info "Restarting production with latest image..." @@ -1298,7 +1550,7 @@ cmd_promote() { if [ "$i" -eq 30 ]; then err "Production failed health check after 30s" warn "Check logs: ./manage.sh logs prod" - warn "Rollback: cp ${BACKUP_DIR}/meshcore.db ${PROD_DATA}/ && ./manage.sh restart prod" + warn "Native archives are in $BACKUP_DIR. Restore only into a fresh target; see docs/postgresql-upgrade.md." exit 1 fi sleep 1 @@ -1351,7 +1603,7 @@ cmd_update() { migrate_config auto info "Rebuilding image..." - dc_prod build prod + dc_prod_base build prod info "Restarting with new image..." dc_prod up -d --force-recreate prod @@ -1365,146 +1617,219 @@ cmd_update() { # ─── Backup ─────────────────────────────────────────────────────────────── -cmd_backup() { - TIMESTAMP=$(date +%Y%m%d-%H%M%S) - BACKUP_DIR="${1:-./backups/corescope-${TIMESTAMP}}" - mkdir -p "$BACKUP_DIR" - - info "Backing up to ${BACKUP_DIR}/" - - # Database - # Always use bind mount path (from .env or default) - DB_PATH="$PROD_DATA/meshcore.db" - if [ -f "$DB_PATH" ]; then - cp "$DB_PATH" "$BACKUP_DIR/meshcore.db" - log "Database ($(du -h "$BACKUP_DIR/meshcore.db" | cut -f1))" - elif container_running "corescope-prod"; then - docker cp corescope-prod:/app/data/meshcore.db "$BACKUP_DIR/meshcore.db" 2>/dev/null && \ - log "Database (via docker cp)" || warn "Could not backup database" +backup_state() { + local destination="$1" backend="$2" data_root target relative database_root + local excludes=() + data_root=$(cd "$PROD_DATA" && pwd -P) || return 1 + destination=$(realpath -m -- "$destination") || return 1 + case "$destination" in "$data_root"|"$data_root"/*) err "Choose a backup directory outside the live data directory."; return 1 ;; esac + if [ "$backend" = sqlite ]; then + for target in telemetry accounts; do + if [ "$target" = accounts ] && [ "$(storage_field prod has_accounts)" = false ]; then continue; fi + relative=$(storage_field prod "$target.sqlite_path") || return 1 + case "$relative" in + /app/data/*) + relative=${relative#/app/data/} + excludes+=("--exclude=./$relative" "--exclude=./$relative-wal" "--exclude=./$relative-shm" "--exclude=./$relative-journal") + ;; + esac + done else - warn "Database not found (container not running?)" - fi - - # Config (now lives in data dir) - if [ -f "$PROD_DATA/config.json" ]; then - cp "$PROD_DATA/config.json" "$BACKUP_DIR/config.json" - log "config.json" - elif [ -f config.json ]; then - cp config.json "$BACKUP_DIR/config.json" - log "config.json (legacy repo root)" - fi - - # Caddyfile - if [ -f caddy-config/Caddyfile ]; then - cp caddy-config/Caddyfile "$BACKUP_DIR/Caddyfile" - log "Caddyfile" - fi - - # Theme - # Always use bind mount path (from .env or default) - THEME_PATH="$PROD_DATA/theme.json" - if [ -f "$THEME_PATH" ]; then - cp "$THEME_PATH" "$BACKUP_DIR/theme.json" - log "theme.json" - elif [ -f theme.json ]; then - cp theme.json "$BACKUP_DIR/theme.json" - log "theme.json" + database_root=$(realpath -m -- "${PROD_POSTGRES_DATA_DIR:-$HOME/corescope-postgres}") || return 1 + case "$database_root" in + "$data_root") err "PostgreSQL storage and application state must use separate directories."; return 1 ;; + "$data_root"/*) excludes+=("--exclude=./${database_root#"$data_root"/}") ;; + esac fi + # Preserve queues, selection/history, settings and custom files as recovery + # material. The selected live databases are supplied only by native snapshots. + ( + umask 077 + set -o noclobber + exec 3>"$destination/state.tar.partial" || exit 1 + trap 'rm -f -- "$destination/state.tar.partial"' EXIT HUP INT TERM + tar -C "$data_root" "${excludes[@]}" -cf - . >&3 || exit 1 + exec 3>&- + sync -f "$destination/state.tar.partial" || exit 1 + ln -- "$destination/state.tar.partial" "$destination/state.tar" || exit 1 + rm -- "$destination/state.tar.partial" || exit 1 + trap - EXIT HUP INT TERM + ) +} - # Summary - TOTAL=$(du -sh "$BACKUP_DIR" | cut -f1) - FILES=$(ls "$BACKUP_DIR" | wc -l) - echo "" - log "Backup complete: ${FILES} files, ${TOTAL} total → ${BACKUP_DIR}/" +cmd_backup() { + local timestamp backup_dir source name backend status has_accounts + timestamp=$(date +%Y%m%d-%H%M%S) + backup_dir="${1:-./backups/corescope-$timestamp}" + [ -n "$backup_dir" ] || backup_dir="./backups/corescope-$timestamp" + backend=$(selected_backend prod) || return 1 + [ "$(storage_field prod state)" = ready ] || { err "Complete validated setup before taking a native backup."; return 1; } + umask 077 + if [ -d "$backup_dir" ] && [ -n "$(find "$backup_dir" -mindepth 1 -maxdepth 1 -print -quit)" ]; then err "Choose a new empty backup directory; prior recovery files were preserved."; return 1; fi + mkdir -p "$backup_dir" + chmod 700 "$backup_dir" + case "$backend" in + sqlite) + info "Writing native SQLite snapshots to $backup_dir/" + sqlite_dump_file prod telemetry "$backup_dir/telemetry.db" || return 1 + if sqlite_source_exists prod accounts; then + sqlite_dump_file prod accounts "$backup_dir/accounts.db" || return 1 + else + status=$? + [ "$status" = 3 ] || { err "Account storage is inaccessible or incomplete; backup refused."; return 1; } + : > "$backup_dir/accounts.absent" + fi + ;; + postgres) + info "Writing native PostgreSQL archives to $backup_dir/" + has_accounts=$(storage_field prod has_accounts) || return 1 + case "$has_accounts" in true|false) ;; *) err "Cannot read the recorded account target."; return 1 ;; esac + pg_dump_file prod corescope_telemetry "$backup_dir/telemetry.dump" || return 1 + if [ "$has_accounts" = true ]; then + pg_dump_file prod corescope_accounts "$backup_dir/accounts.dump" || return 1 + else + : > "$backup_dir/accounts.absent" + fi + ;; + esac + backup_state "$backup_dir" "$backend" || return 1 + printf '%s\n' "$backend" > "$backup_dir/backend.txt" + for name in config.json theme.json; do + source="$PROD_DATA/$name" + [ -f "$source" ] || source="$name" + if [ -f "$source" ]; then cp -- "$source" "$backup_dir/$name"; chmod 600 "$backup_dir/$name"; fi + done + if [ -f .env ]; then cp -- .env "$backup_dir/compose.env"; chmod 600 "$backup_dir/compose.env"; fi + if [ -f caddy-config/Caddyfile ]; then cp -- caddy-config/Caddyfile "$backup_dir/Caddyfile"; chmod 600 "$backup_dir/Caddyfile"; fi + log "Native $backend backups and available configuration saved. Keep the bundle private and encrypted off-host." } # ─── Restore ────────────────────────────────────────────────────────────── -cmd_restore() { - if [ -z "$1" ]; then - err "Usage: ./manage.sh restore " - if [ -d "./backups" ]; then - echo "" - echo " Available backups:" - ls -dt ./backups/meshcore-* ./backups/corescope-* 2>/dev/null | head -10 | while read d; do - if [ -d "$d" ]; then - echo " $d/ ($(ls "$d" | wc -l) files)" - elif [ -f "$d" ]; then - echo " $d ($(du -h "$d" | cut -f1))" - fi - done - fi - exit 1 - fi - - # Accept either a directory (full backup) or a single .db file - if [ -d "$1" ]; then - DB_FILE="$1/meshcore.db" - CONFIG_FILE="$1/config.json" - CADDY_FILE="$1/Caddyfile" - THEME_FILE="$1/theme.json" - elif [ -f "$1" ]; then - DB_FILE="$1" - CONFIG_FILE="" - CADDY_FILE="" - THEME_FILE="" - else - err "Not found: $1" - exit 1 - fi - - if [ ! -f "$DB_FILE" ]; then - err "No meshcore.db found in $1" - exit 1 - fi - - echo "" - info "Will restore from: $1" - [ -f "$DB_FILE" ] && echo " • Database" - [ -n "$CONFIG_FILE" ] && [ -f "$CONFIG_FILE" ] && echo " • config.json" - [ -n "$CADDY_FILE" ] && [ -f "$CADDY_FILE" ] && echo " • Caddyfile" - [ -n "$THEME_FILE" ] && [ -f "$THEME_FILE" ] && echo " • theme.json" - echo "" - - if ! confirm "Continue? (current state will be backed up first)"; then - echo " Aborted." - exit 0 - fi - - # Backup current state first - info "Backing up current state..." - cmd_backup "./backups/corescope-pre-restore-$(date +%Y%m%d-%H%M%S)" - - dc_prod stop prod 2>/dev/null || true - - # Restore database - mkdir -p "$PROD_DATA" - DEST_DB="$PROD_DATA/meshcore.db" - cp "$DB_FILE" "$DEST_DB" - log "Database restored" - - # Restore config if present - if [ -n "$CONFIG_FILE" ] && [ -f "$CONFIG_FILE" ]; then - cp "$CONFIG_FILE" "$PROD_DATA/config.json" - log "config.json restored to ${PROD_DATA}/" +stage_backup_state() { + local bundle="$1" destination="$2" stage + bundle=$(cd "$bundle" && pwd -P) || return 1 + [ -f "$bundle/state.tar" ] || { err "This restore requires the matching private state archive; use the native recovery guide for older snapshots."; return 1; } + mkdir -p "$(dirname "$destination")" + stage=$(mktemp -d "$destination.restore-XXXXXXXX") || return 1 + chmod 700 "$stage" + # Managed archives contain relative regular files/directories only. Keep an + # unsupported archive and its staged recovery material for manual inspection. + tar -tf "$bundle/state.tar" > "$stage/.restore-names" || return 1 + LC_ALL=C tar -tvf "$bundle/state.tar" > "$stage/.restore-types" || return 1 + if ! awk '/^\// || /(^|\/)\.\.(\/|$)/ { bad=1 } END { exit bad }' "$stage/.restore-names" || + ! awk 'substr($0,1,1)!="-" && substr($0,1,1)!="d" { bad=1 } END { exit bad }' "$stage/.restore-types"; then + err "State archive contains unsupported paths or links. Preserve it for manual recovery." + return 1 fi + rm -- "$stage/.restore-names" "$stage/.restore-types" + tar --no-same-owner --no-same-permissions -xf "$bundle/state.tar" -C "$stage" || return 1 + printf '%s\n' "$stage" +} - # Restore Caddyfile if present - if [ -n "$CADDY_FILE" ] && [ -f "$CADDY_FILE" ]; then - mkdir -p caddy-config - cp "$CADDY_FILE" caddy-config/Caddyfile - log "Caddyfile restored" +restore_sqlite_bundle() { + local bundle="$1" destination stage state backend kind target relative result device + bundle=$(cd "$bundle" && pwd -P) || return 1 + [ -f "$bundle/state.tar" ] || { err "This restore requires the matching private state archive; use the native recovery guide for older snapshots."; return 1; } + if container_running corescope-prod; then err "Stop CoreScope before restoring."; return 1; fi + destination=$(realpath -m -- "$PROD_DATA") || return 1 + if [ -L "$PROD_DATA" ] || { [ -d "$destination" ] && [ -n "$(find "$destination" -mindepth 1 -maxdepth 1 -print -quit)" ]; }; then + err "SQLite restore requires a new empty state directory. Preserve the current directory and select a fresh PROD_DATA_DIR." + return 1 fi - - # Restore theme if present - if [ -n "$THEME_FILE" ] && [ -f "$THEME_FILE" ]; then - DEST_THEME="$PROD_DATA/theme.json" - cp "$THEME_FILE" "$DEST_THEME" - log "theme.json restored" + if ! confirm "Restore both native SQLite snapshots and state into this empty directory?"; then return 0; fi + stage=$(stage_backup_state "$bundle" "$destination") || return 1 + state=$(storage_at prod "$stage" field state) || return 1 + backend=$(storage_at prod "$stage" field backend) || return 1 + [ "$state" = ready ] && [ "$backend" = sqlite ] || { err "State archive is not a complete SQLite installation snapshot."; return 1; } + if [ "$(storage_at prod "$stage" field has_accounts)" = true ] && [ -e "$bundle/accounts.absent" ]; then + err "The selection records initialized accounts; an absence marker cannot replace their native snapshot." + return 1 fi + for kind in telemetry accounts; do + if [ "$kind" = accounts ] && [ "$(storage_at prod "$stage" field has_accounts)" = false ]; then + [ -f "$bundle/accounts.absent" ] && [ ! -e "$bundle/accounts.db" ] || { err "Account backup and selection disagree."; return 1; } + continue + fi + target=$(storage_at prod "$stage" field "$kind.sqlite_path") || return 1 + case "$target" in /app/data/*) ;; *) err "Managed restore requires SQLite targets inside the persistent data mount."; return 1 ;; esac + relative=${target#/app/data/} + target=$(realpath -m -- "$stage/$relative") || return 1 + case "$target" in "$stage"/*) ;; *) err "SQLite target escaped the staged state directory."; return 1 ;; esac + [ ! -e "$target" ] && [ ! -e "$target-wal" ] && [ ! -e "$target-shm" ] || { err "State archive already contains a selected database; native snapshot restore was refused."; return 1; } + if [ "$kind" = accounts ] && [ -f "$bundle/accounts.absent" ]; then + [ ! -e "$bundle/accounts.db" ] || { err "Account backup metadata is inconsistent."; return 1; } + continue + fi + [ -f "$bundle/$kind.db" ] && [ "$(head -c 15 "$bundle/$kind.db")" = 'SQLite format 3' ] || { err "A native SQLite snapshot is missing or invalid."; return 1; } + result=$(dc_prod_base run --rm --no-deps -v "$bundle:/backup:ro" --entrypoint sqlite3 prod -readonly "/backup/$kind.db" 'PRAGMA quick_check') || return 1 + [ "$result" = ok ] || { err "SQLite snapshot integrity check failed."; return 1; } + mkdir -p "$(dirname "$target")" + cp -- "$bundle/$kind.db" "$target" || return 1 + chmod 600 "$target" + done + # Validate both schemas on the staged mount before publishing any restored + # selection or starting a process. This also handles supported old schemas. + storage_at prod "$stage" setup -backend=sqlite || return 1 + device=$(stat -c %d "$stage") || return 1 + [ ! -e "$destination" ] || { [ -d "$destination" ] && [ "$(stat -c %d "$destination")" = "$device" ] && [ -z "$(find "$destination" -mindepth 1 -maxdepth 1 -print -quit)" ]; } || { + err "Restore destination changed or is a separate mount; staged recovery remains at $stage." + return 1 + } + mv -T -- "$stage" "$destination" || { err "Restore publication failed; preserve staged recovery at $stage."; return 1; } + log "SQLite state restored. Services remain stopped: review restored accounts, sessions and tokens before start." +} - dc_prod up -d prod - log "Restored and restarted." +cmd_restore() { + local bundle="$1" file source destination stamp has_accounts stage database + local databases=(corescope_telemetry) + if [ ! -d "$bundle" ]; then err "Usage: ./manage.sh restore ; see docs/storage.md for older standalone snapshots."; return 1; fi + local backend + backend=$(cat "$bundle/backend.txt" 2>/dev/null || printf postgres) + case "$backend" in + sqlite) restore_sqlite_bundle "$bundle"; return $? ;; + postgres) ;; + *) err "Unknown backup backend."; return 1 ;; + esac + if container_running corescope-prod; then err "Stop CoreScope before restoring; leave target PostgreSQL running."; return 1; fi + require_managed_postgres prod || return 1 + has_accounts=$(storage_field prod has_accounts) || return 1 + case "$has_accounts" in + true) + [ ! -e "$bundle/accounts.absent" ] || { err "The selection records initialized accounts; their native archive is required."; return 1; } + databases+=(corescope_accounts) + ;; + false) + [ -f "$bundle/accounts.absent" ] && [ ! -e "$bundle/accounts.dump" ] || { err "Account backup and selection disagree."; return 1; } + destination=$(realpath -m -- "$PROD_DATA") || return 1 + stage=$(stage_backup_state "$bundle" "$destination") || return 1 + storage_at prod "$stage" managed-postgres || return 1 + [ "$(storage_at prod "$stage" field has_accounts)" = false ] || { err "Account absence marker disagrees with the retained backup selection."; return 1; } + ;; + *) err "Cannot read the recorded account target."; return 1 ;; + esac + for database in "${databases[@]}"; do + file="${database#corescope_}.dump" + if [ ! -f "$bundle/$file" ] || [ "$(head -c 5 "$bundle/$file")" != PGDMP ]; then err "Backup must contain a native $file; SQLite files are not accepted."; return 1; fi + done + for database in "${databases[@]}"; do + if ! pg_empty prod "$database"; then err "Restore requires empty PostgreSQL databases. Preserve the current cluster and configure a fresh destination."; return 1; fi + done + if ! confirm "Restore the selected native archives into these empty databases?"; then return 0; fi + for database in "${databases[@]}"; do + pg_restore_file prod "$database" "$bundle/${database#corescope_}.dump" || return 1 + done + mkdir -p "$PROD_DATA" caddy-config + stamp="$(date +%Y%m%d-%H%M%S)-$$" + for file in config.json theme.json Caddyfile; do + source="$bundle/$file" + [ -f "$source" ] || continue + destination="$PROD_DATA/$file" + [ "$file" != Caddyfile ] || destination=caddy-config/Caddyfile + if [ -f "$destination" ]; then cp -- "$destination" "$destination.pre-restore-$stamp"; fi + cp -- "$source" "$destination" + done + log "Native archives restored. Services remain stopped: review restored accounts, sessions and tokens before ./manage.sh start." } # ─── MQTT Test ──────────────────────────────────────────────────────────── @@ -1551,6 +1876,68 @@ cmd_reset() { # ─── Help ───────────────────────────────────────────────────────────────── +cmd_storage() { + local action="${1:-status}" argument="${2:-}" state source destination target requires_postgres=false + local STORAGE_OPERATION="$action" extra=() + case "$action" in + status) storage_action prod status; return $? ;; + switch) + case "$argument" in sqlite|postgres) ;; *) err "Usage: ./manage.sh storage switch sqlite|postgres"; return 1 ;; esac + state=$(storage_field prod state) || return 1 + [ "$state" = ready ] || { err "Finish setup or explicitly resume/abort the pending job before another switch."; return 1; } + source=$(storage_field prod backend) || return 1 + [ "$source" != "$argument" ] || { info "Already using recorded $source storage."; return 0; } + [ "$source" != postgres ] || require_managed_postgres prod || return 1 + prepare_database_credentials || return 1 + if ! confirm "Stop production writers and run the verified offline switch to $argument?"; then return 0; fi + dc_prod_base stop prod || return 1 + # Starting only PostgreSQL leaves the conversion destination uninitialized. + dc_postgres prod up -d --wait postgres || return 1 + if [ "$argument" = sqlite ]; then + destination="$(storage_field prod state_dir)/sqlite-$(date -u +%Y%m%d-%H%M%S)-$$" || return 1 + extra=(-sqlite-path "$destination/meshcore.db" -users-sqlite-path "$destination/users.db") + fi + if ! dc_postgres prod run --rm --no-deps --entrypoint /app/storage.sh bootstrap switch "-backend=$argument" "${extra[@]}"; then + err "Switch failed. Services remain stopped; preserve source/recovery files and inspect storage status for its resume/abort job ID." + return 1 + fi + write_private_env CORESCOPE_DB_BACKEND "$argument" || return 1 + [ "$argument" != sqlite ] || dc_postgres prod stop postgres || return 1 + log "Verified backend switch completed. Services remain stopped; inspect storage status and run start when ready." + ;; + resume|abort) + [[ "$argument" =~ ^[0-9a-f]{32}$ ]] || { err "Use storage $action from storage status."; return 1; } + state=$(storage_field prod state) || return 1 + [ "$state" = pending ] || { err "No pending storage job matches this recovery action."; return 1; } + [ "$(storage_field prod job_id)" = "$argument" ] || { err "Job ID does not match the pending switch."; return 1; } + if [ "$action" = abort ]; then + if ! confirm "Keep writers stopped and abort storage job $argument?"; then return 0; fi + dc_prod_base stop prod || return 1 + storage_action prod abort "-job-id=$argument" || return 1 + log "Storage job aborted before publication. Source, staged targets and recovery files remain; inspect status before setup/start." + return 0 + fi + source=$(storage_field prod source_backend) || { err "Recovery source identity is unavailable; inspect storage status."; return 1; } + target=$(storage_field prod target_backend) || { err "Recovery target identity is unavailable; inspect storage status."; return 1; } + case "$source:$target" in :sqlite|:postgres|sqlite:sqlite|sqlite:postgres|postgres:sqlite|postgres:postgres) ;; *) err "The job has no bound target yet. Preserve recovery files, abort this job, then start a new setup/switch."; return 1 ;; esac + if [ "$source" = postgres ] || [ "$target" = postgres ]; then requires_postgres=true; require_database_credentials || return 1; fi + if ! confirm "Keep writers stopped and $action storage job $argument?"; then return 0; fi + dc_prod_base stop prod || return 1 + if $requires_postgres; then + dc_postgres prod up -d --wait postgres || return 1 + dc_postgres prod run --rm --no-deps --entrypoint /app/storage.sh bootstrap "$action" "-job-id=$argument" || return 1 + else + storage_action prod "$action" "-job-id=$argument" || return 1 + fi + source=$(storage_field prod backend) || return 1 + write_private_env CORESCOPE_DB_BACKEND "$source" || return 1 + if $requires_postgres && [ "$source" = sqlite ]; then dc_postgres prod stop postgres || return 1; fi + log "Storage recovery completed. Original and staged recovery files remain; validate before start." + ;; + *) err "Usage: ./manage.sh storage status|switch sqlite|postgres|resume JOB_ID|abort JOB_ID"; return 1 ;; + esac +} + cmd_help() { echo "" echo "CoreScope — Management Script" @@ -1563,7 +1950,7 @@ cmd_help() { echo "" printf '%b\n' " ${BOLD}Run${NC}" echo " start Start production container" - echo " start --with-staging Start production + staging-go (copies prod DB + config)" + echo " start --with-staging Start production + staging-go (clones into empty staging telemetry + config)" echo " stop [prod|staging|all] Stop specific or all containers (default: all)" echo " restart [prod|staging|all] Restart specific or all containers" echo " status Show health, stats, and service status" @@ -1572,8 +1959,11 @@ cmd_help() { printf '%b\n' " ${BOLD}Maintain${NC}" echo " update [version] Update to version (no arg=latest tag, 'latest'=master tip, or e.g. v3.1.0)" echo " promote Promote staging → production (backup + restart)" - echo " backup [dir] Full backup: database + config + theme" - echo " restore Restore from backup dir or .db file" + echo " backup [dir] Native selected-backend snapshots + private state/config" + echo " restore Restore native backups into fresh/empty targets" + echo " storage status Show recorded backend and pending recovery job" + echo " storage switch sqlite|postgres Verified offline conversion; keeps recovery copies" + echo " storage resume|abort JOB_ID Explicit interrupted-switch recovery" echo " mqtt-test Check if MQTT data is flowing" echo "" echo "Prod uses docker-compose.yml; staging uses ${STAGING_COMPOSE_FILE}." @@ -1584,6 +1974,7 @@ cmd_help() { case "${1:-help}" in setup) cmd_setup ;; + storage) shift; cmd_storage "$@" ;; start) cmd_start "$2" ;; stop) cmd_stop "$2" ;; restart) cmd_restart "$2" ;; diff --git a/public/admin-audit.js b/public/admin-audit.js index c141b1a5b..1618a9129 100644 --- a/public/admin-audit.js +++ b/public/admin-audit.js @@ -22,7 +22,7 @@ ['user.email.*', 'Address changes'], ['user.mail.refresh', 'Mail status refreshed'], ['user.export', 'Data exports'], - ['user.backup', 'users.db downloads'], + ['user.backup', 'Account backup downloads'], ['proposal.*', 'Channel proposals'], ['notify.*', 'Notification settings'] ]; diff --git a/public/geofilter-docs.html b/public/geofilter-docs.html index f445cebc3..3f492363c 100644 --- a/public/geofilter-docs.html +++ b/public/geofilter-docs.html @@ -120,22 +120,14 @@

Legacy bounding box

Prefer the polygon format — it supports irregular shapes and the bufferKm margin.

Cleaning up historical nodes

-

The ingestor prevents new out-of-bounds nodes from being ingested, but does not retroactively remove nodes stored before the filter was configured. Use the prune script for that:

-
# Dry run — shows what would be deleted without making any changes
-python3 scripts/prune-nodes-outside-geo-filter.py --dry-run
-
-# Default paths: /app/data/meshcore.db and /app/config.json
-python3 scripts/prune-nodes-outside-geo-filter.py
-
-# Custom paths
-python3 scripts/prune-nodes-outside-geo-filter.py /path/to/meshcore.db \
-  --config /path/to/config.json
-
-# In Docker — run inside the container
-docker exec -it meshcore-analyzer \
-  python3 /app/scripts/prune-nodes-outside-geo-filter.py --dry-run
-

The script reads geo_filter.polygon and geo_filter.bufferKm from config, lists nodes that fall outside, then asks for yes confirmation before deleting. Nodes without coordinates are always kept.

-

This is a one-time migration tool — run it once after first configuring geo_filter to clean up pre-filter data.

+

The ingestor prevents new out-of-bounds nodes from being ingested. To review older nodes, open the Customizer's Admin Geo Filter settings and save the desired polygon and buffer first.

+
    +
  1. In Prune historical nodes, choose Preview prune.
    2. +
  2. Review the listed nodes outside the active filter. Nodes without coordinates are kept.
    3. +
  3. Choose Delete nodes only after reviewing the preview.
    4. +
+

The server submits the request to the shared state-directory queue; the ingestor performs the database changes. Keep both processes on the same configured state directory and allow the ingestor to finish before checking the result.

+

The old Python prune script is offline legacy SQLite tooling. It must not be used for PostgreSQL maintenance.

diff --git a/public/perf.js b/public/perf.js index 1ec00dd5b..cef098ae7 100644 --- a/public/perf.js +++ b/public/perf.js @@ -117,11 +117,11 @@ function renderVersionCard(health) { try { // #1258: /api/health was awaited AFTER Promise.all, adding a full RTT // (~50-200ms) on every 5s refresh. Issue it in parallel with the rest. - const [server, client, ioStats, sqliteStats, writeSources, health] = await Promise.all([ + const [server, client, ioStats, databaseStats, writeSources, health] = await Promise.all([ fetch('/api/perf').then(r => r.json()), Promise.resolve(window.apiPerf ? window.apiPerf() : null), fetch('/api/perf/io').then(r => r.json()).catch(() => null), - fetch('/api/perf/sqlite').then(r => r.json()).catch(() => null), + fetch('/api/perf/database').then(r => r.json()).catch(() => null), fetch('/api/perf/write-sources').then(r => r.json()).catch(() => null), fetch('/api/health').then(r => r.json()).catch(() => null) ]); @@ -213,7 +213,7 @@ function renderVersionCard(health) { const keys = Object.keys(src).sort((a, b) => (src[b] || 0) - (src[a] || 0)); html += '

Write Sources

'; if (keys.length === 0) { - html += '

No ingestor stats yet (waiting for /tmp/corescope-ingestor-stats.json)

'; + html += '

No ingestor stats yet (waiting for the ingestor statistics snapshot)

'; } else { // Anomaly detection (#1120 acceptance): flag any component whose // per-second write rate exceeds 10× its 5-minute rolling baseline. @@ -246,17 +246,26 @@ function renderVersionCard(health) { } } - // SQLite perf (separate from existing SQLite block — focused on WAL + cache hit) (#1120) - if (sqliteStats) { - const walMB = sqliteStats.walSizeMB || 0; - const walFlag = walMB > 100 ? ' ' : ''; - const hitRate = (sqliteStats.cacheHitRate || 0) * 100; + // Database-wide PostgreSQL counters and this server's connection pool. + if (databaseStats && databaseStats.engine === 'postgresql') { + const hitRate = databaseStats.cacheHitRate == null ? null : Number(databaseStats.cacheHitRate) * 100; const hitFlag = hitRate > 0 && hitRate < 90 ? ' ' : ''; - html += `

SQLite (WAL + Cache Hit)

-
${walMB.toFixed(1)}MB${walFlag}
WAL Size
-
${(sqliteStats.pageCount || 0).toLocaleString()}
Page Count
-
${sqliteStats.pageSize || 0}
Page Size
-
${hitRate.toFixed(1)}%${hitFlag}
Cache Hit Rate
+ const hitText = hitRate != null && Number.isFinite(hitRate) ? hitRate.toFixed(1) + '%' : '—'; + html += `

PostgreSQL connections

+
${Number(databaseStats.inUseConnections || 0)} / ${Number(databaseStats.openConnections || 0)}
Connections in use
+
${Number(databaseStats.connectionWaitCount || 0).toLocaleString()}
Connection waits
+
${Number(databaseStats.connectionWaitMs || 0).toFixed(1)}ms
Total connection wait
+
${hitText}${hitFlag}
Database Block Cache Hit Rate
+
`; + } + + if (databaseStats && databaseStats.engine === 'sqlite') { + const walMB = databaseStats.walSizeMB; + html += `

SQLite storage

+
${walMB == null ? '—' : Number(walMB).toFixed(1) + 'MB'}
WAL Size
+
${databaseStats.pageCount == null ? '—' : Number(databaseStats.pageCount).toLocaleString()}
Page Count
+
${databaseStats.pageSize == null ? '—' : Number(databaseStats.pageSize)}
Page Size
+
${databaseStats.plannerStats == null ? 'Unknown' : databaseStats.plannerStats ? 'Available' : 'Pending'}
Planner Statistics
`; } @@ -299,23 +308,29 @@ function renderVersionCard(health) {
`; } - // SQLite stats - if (server.sqlite && !server.sqlite.error) { - const sq = server.sqlite; - const walColor = sq.walSizeMB > 50 ? 'var(--status-red)' : sq.walSizeMB > 10 ? 'var(--status-yellow)' : 'var(--status-green)'; - const freelistColor = sq.freelistMB > 10 ? 'var(--status-yellow)' : 'var(--status-green)'; - html += `

SQLite

+ // Native engine sizes retain their distinct meanings; PostgreSQL excludes cluster WAL. + const selectedDatabase = server.database || server.sqlite || server.postgres; + const databaseLabel = selectedDatabase && selectedDatabase.engine === 'sqlite' ? 'SQLite' : 'PostgreSQL'; + if (selectedDatabase && selectedDatabase.rows && selectedDatabase.dbSizeMB != null) { + const sq = selectedDatabase; + html += `

${databaseLabel} data

${sq.dbSizeMB}MB
DB Size
-
${sq.walSizeMB}MB
WAL Size
-
${sq.freelistMB}MB
Freelist
${(sq.rows.transmissions || 0).toLocaleString()}
Transmissions
${(sq.rows.observations || 0).toLocaleString()}
Observations
${sq.rows.nodes || 0}
Nodes
${sq.rows.observers || 0}
Observers
`; - if (sq.walPages) { - html += `
${sq.walPages.busy}
WAL Busy Pages
`; + if (sq.engine === 'sqlite' && sq.freelistMB != null) { + html += `
${Number(sq.freelistMB).toFixed(1)}MB
Reusable Free Space
`; } html += `
`; + if (sq.sampledAt) { + const sampled = new Date(sq.sampledAt); + const sampleTime = Number.isFinite(sampled.getTime()) ? sampled.toLocaleString() : 'unknown time'; + const interval = Number(sq.sampleIntervalSeconds); + html += `
${sq.stale ? 'Last verified ' + databaseLabel + ' sample' : databaseLabel + ' sample'}: ${sampleTime}${Number.isFinite(interval) && interval > 0 ? ' · refreshes every ' + interval + 's' : ''}${sq.stale ? ' · refresh unavailable' : ''}
`; + } + } else if (selectedDatabase) { + html += `

${databaseLabel} data

${databaseLabel} storage unavailable
`; } // Server endpoints table — sort by total time (count * avg) DESC. diff --git a/qa/README.md b/qa/README.md index 938b8ac1f..3a9310f78 100644 --- a/qa/README.md +++ b/qa/README.md @@ -10,7 +10,9 @@ qa/ ├── plans/ │ └── .md ← per-release test plans (one file per RC) └── scripts/ - └── api-contract-diff.sh ← CoreScope-tuned API contract diff + ├── api-contract-diff.sh ← CoreScope-tuned API contract diff + ├── blacklist-test.sh ← hide/retain/recover deployment check + └── test-blacklist-sql.sh ← disposable native PostgreSQL security gate ``` ## How to run @@ -43,3 +45,40 @@ Authoring rules from the qa-suite skill: - Distinguish HTTP errors from jq parse failures - Don't silence stderr — script bugs must surface - Exit code = number of failures + +## PostgreSQL blacklist retention probe + +The active blacklist check reads PostgreSQL using a restricted telemetry reader. +Set `CORESCOPE_READER_DATABASE_URL` privately on the QA runner, or configure native +libpq variables/service files (`PGDATABASE`, `PGUSER`, `PGPASSFILE`, `PGSERVICE`, +and connection/TLS settings). A URL belongs in `CORESCOPE_READER_DATABASE_URL`; +`PGDATABASE` itself does not expand URLs. Do not put credentials in command-line +arguments. `TARGET_DB_PATH` is obsolete and rejected. + +With no explicit local connection, the script probes the target container and +then the target host over its existing SSH transport. Configure a private reader +connection there. PostgreSQL 18 `psql` is required; URL parsing also needs Python +3 (`CORESCOPE_QA_PYTHON` can select the executable). Native libpq settings do not +need Python on the target. The shipped container has `psql`; use its private +libpq settings or run the URL-based reader probe locally/on a Python-equipped host. + +Every probe checks actual read-only grants and the completed telemetry readiness +marker, then binds the pubkey with psql's native extended-query protocol. SQL and +hex-encoded values travel on stdin. Connection credentials remain in the private +environment. Failures report SQLSTATE classes without printing connection strings. + +The local security gate contacts no HTTP endpoint or SSH host: + +```sh +# Export CORESCOPE_TEST_POSTGRES_URL privately for a disposable PostgreSQL 18 admin. +bash qa/scripts/test-blacklist-sql.sh +``` + +The gate requires Python 3, `psql`, Go and a C compiler. It creates temporary +schemas and separate owner/reader LOGIN roles, bootstraps with the shipping +migration tool, and verifies both legitimate and hostile pubkeys. It also prepares +and imports a disposable copy of the committed historical SQLite fixture before +comparing native PostgreSQL counts. SQLite is used only by these explicit offline +tools. Missing database/client prerequisites fail the gate; no SQL checks skip. +`CORESCOPE_QA_MIGRATOR` and `CORESCOPE_QA_PREPARER` may select previously built +matching helper binaries. Temporary objects and private files are removed on exit. diff --git a/qa/plans/v3.6.0-rc.md b/qa/plans/v3.6.0-rc.md index 77bcb8a72..35eefcf6d 100644 --- a/qa/plans/v3.6.0-rc.md +++ b/qa/plans/v3.6.0-rc.md @@ -95,7 +95,7 @@ Run `scripts/api-contract-diff.sh BASELINE_URL TARGET_URL` once. Report the scri | 10.1 | Add node pubkey to nodeBlacklist config; restart | Hidden from listings/map/neighbor graph | #742 | auto: blacklist-test.sh | | 10.2 | Packets still in DB | Yes (filter not delete) | #742 | auto: blacklist-test.sh | -`blacklist-test.sh` covers both 10.1 and 10.2 in one run. Required env: `TEST_NODE_PUBKEY` (hex, of a real visible node on TARGET), `TARGET_SSH_HOST`, `TARGET_CONFIG_PATH`, `TARGET_CONTAINER`. Optional: `TARGET_DB_PATH` or `ADMIN_API_TOKEN` for §10.2 probe; `TARGET_SSH_KEY` (default `/root/.ssh/id_ed25519`). Mandatory teardown removes the pubkey and verifies the node returns to listings. +`blacklist-test.sh` covers both 10.1 and 10.2 in one run. Required env: `TEST_NODE_PUBKEY` (hex, of a real visible node on TARGET), `TARGET_SSH_HOST`, `TARGET_CONFIG_PATH`, `TARGET_CONTAINER`. Configure §10.2 with a private `CORESCOPE_READER_DATABASE_URL` or native libpq reader settings locally/on the target; `ADMIN_API_TOKEN` remains optional. `TARGET_DB_PATH` is obsolete. `TARGET_SSH_KEY` defaults to `/root/.ssh/id_ed25519`. See `qa/README.md` for PostgreSQL client/role requirements. Mandatory teardown removes the pubkey and verifies the node returns to listings. ### 11. Deploy/ops | # | Step | Pass criteria | Source | Mode | diff --git a/qa/scripts/blacklist-test.sh b/qa/scripts/blacklist-test.sh index 205ff8f7e..14467cc00 100755 --- a/qa/scripts/blacklist-test.sh +++ b/qa/scripts/blacklist-test.sh @@ -15,8 +15,12 @@ # TARGET_CONFIG_PATH — absolute path to config.json on the target # TARGET_CONTAINER — docker container name on the target # Optional env: -# TARGET_DB_PATH — sqlite db path on the target (for §10.2 sqlite probe) -# ADMIN_API_TOKEN — if /api/admin/transmissions exists, use it instead of ssh+sqlite +# CORESCOPE_DB_BACKEND — target's selected backend: sqlite (default) or postgres +# TARGET_DB_PATH — selected SQLite path on the target for the native reader +# CORESCOPE_READER_DATABASE_URL — private local PostgreSQL reader URL for §10.2 +# PGSERVICE / PGDATABASE — alternatively configure private native libpq settings +# CORESCOPE_QA_PYTHON — Python 3 executable for URL parsing (default python3) +# ADMIN_API_TOKEN — if /api/admin/transmissions exists, use it instead of the DB probe # (read from env, not argv — never appears in ps) # CURL_TIMEOUT — per-request curl timeout, seconds (default 60) # RESTART_WAIT_S — max wait for /api/stats after restart (default 120) @@ -27,9 +31,9 @@ # hide-failed → blacklisted pubkey still surfaced via API (§10.1 fail) # retain-failed → no transmissions.from_pubkey rows (ADVERTs) for the # blacklisted pubkey in the DB (§10.2 fail), or the -# §10.2 probe could not run at all — no sqlite3 on the target -# able to bind a parameter. The message names what is needed; -# there is no fallback to interpolated SQL. +# §10.2 probe cannot verify its selected native reader. +# Native parameter binding is mandatory; neither an +# interpolated query nor another backend is a fallback. # teardown-failed→ post-test removal did not restore listing # # Exit code = number of failures (0 = pass). @@ -158,125 +162,206 @@ node_visible() { # ----------------------------------------------------------------------------- # §10.2 DB probe — bind the pubkey, do not interpolate it (issue #1977) # ----------------------------------------------------------------------------- -# Batch flags, all in service of "the count is parseable and errors are visible": -# -bail stop at the first SQL error instead of running on -# -init /dev/null ignore the operator's ~/.sqliterc — a stray .mode there -# would make the count unparseable -# -noheader -list stdout is exactly the number, nothing else -SQLITE_ARGS=(-batch -bail -init /dev/null -noheader -list) -# Round-trip probe token. The value is arbitrary; it only has to come back intact. -SQLITE_PROBE_TOKEN="corescope-probe-ok" -SQLITE_RUNNER="" # "container" | "host", set by resolve_sqlite_runner -RETAIN_COUNT="" # set by read_retain_count +# Native psql uses the extended query protocol (\bind), not text substitution. +# Queries and hex-only parameter values travel on stdin; connection credentials +# stay in private libpq environment/service settings, never process arguments. +POSTGRES_RUNNER="" # local | container | host +SQL_BACKEND="${CORESCOPE_DB_BACKEND:-sqlite}" +POSTGRES_PROBE_TOKEN="corescope-probe-ok" +RETAIN_COUNT="" +POSTGRES_CONNECT_PY="$(cat "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/postgres-connect.py")" -# Hex-encode a value for embedding in SQL as a blob literal. -# -# Why hex rather than quoting: the output alphabet is [0-9a-f], so no byte the -# caller passes can terminate a string literal or add a dot-command argument. -# That holds for arbitrary input, which is the point — the SQL layer stops -# depending on main()'s hex gate in order to be safe. -# -# `od -v` is load-bearing: without it od collapses runs of identical lines to -# '*' and long repetitive values encode wrongly. +# A psql meta-command argument whose only variable bytes are [0-9a-f]. od -v +# prevents repeated long values collapsing to '*'. Empty input binds as ''. sql_hex_literal() { - printf "x'%s'" "$(printf '%s' "$1" | od -An -v -tx1 | tr -d ' \n')" + [[ "$SQL_BACKEND" != sqlite ]] || printf x + printf "'%s'" "$(printf '%s' "$1" | od -An -v -tx1 | tr -d ' \n')" } -# SQL for the §10.2 count, fed to sqlite3 on stdin. The SELECT text is a -# constant; the pubkey arrives as a bound parameter. -# -# Note the nested cast rather than `.parameter set :pubkey ''`: -# dot-command arguments are split on whitespace, so a value containing a space -# (e.g. "' OR 1=1 --") makes sqlite3 print the .parameter help to STDOUT, exit -# 0, and leave :pubkey unbound. COUNT(*) then returns 0 — which reads exactly -# like a passing security fix. -bail does not catch it either. -# -# The column is transmissions.from_pubkey (cmd/ingestor/db.go CREATE TABLE and -# the from_pubkey_v1 migration; asserted by internal/dbschema). The ingestor -# fills it only for ADVERTs, with hex.EncodeToString output — lowercase. The -# script's hex gate and the server's nodeBlacklist both accept any case, so the -# bound value is lowercased inside SQL; the parameter itself is still bound. -# There is no from_node column: querying it errors on every real database. -transmission_count_sql() { +postgres_read_guard() { + cat <<'SQL' +BEGIN READ ONLY; +DO $qa$ +BEGIN + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname=current_user AND + (rolsuper OR rolcreaterole OR rolcreatedb OR rolreplication OR rolbypassrls)) + OR has_database_privilege(current_database(),'CREATE') + OR EXISTS (SELECT 1 FROM pg_namespace WHERE nspname !~ '^pg_' AND nspname <> 'information_schema' + AND (has_schema_privilege(oid,'CREATE') OR pg_has_role(current_user,nspowner,'MEMBER'))) + OR EXISTS (SELECT 1 FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace + WHERE n.nspname !~ '^pg_' AND n.nspname <> 'information_schema' + AND CASE WHEN c.relkind IN ('r','p','v','m','f') + THEN has_table_privilege(c.oid,'INSERT,UPDATE,DELETE,TRUNCATE,TRIGGER') + OR pg_has_role(current_user,c.relowner,'MEMBER') ELSE false END) + OR EXISTS (SELECT 1 FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace + WHERE n.nspname !~ '^pg_' AND n.nspname <> 'information_schema' + AND CASE WHEN c.relkind='S' THEN has_sequence_privilege(c.oid,'USAGE,UPDATE') ELSE false END) + THEN RAISE EXCEPTION 'QA requires a restricted PostgreSQL reader role' USING ERRCODE='42501'; END IF; + -- Keep the runtime QA reader at dbschema.Version; the native regression uses + -- the shipping migrator's schema and rejects old, future and unfinished markers. + IF (SELECT count(*) FROM corescope_schema WHERE kind='telemetry' AND version=2 AND ready) <> 1 + THEN RAISE EXCEPTION 'telemetry schema is not ready' USING ERRCODE='55000'; END IF; +END $qa$; +SQL +} +postgres_transmission_count_sql() { + postgres_read_guard + printf '%s\n' "SELECT COUNT(*) FROM transmissions WHERE from_pubkey = lower(convert_from(decode(\$1,'hex'),'UTF8'))" + printf '\\bind %s\n\\g\nCOMMIT;\n' "$(sql_hex_literal "$1")" +} +postgres_probe_sql() { + postgres_read_guard + printf '%s\n' "SELECT convert_from(decode(\$1,'hex'),'UTF8')" + printf '\\bind %s\n\\g\nCOMMIT;\n' "$(sql_hex_literal "$POSTGRES_PROBE_TOKEN")" +} + +# POSIX shell quoting also works inside the image's /bin/sh. These arguments +# contain only public launcher code/container names, never a URL or password. +postgres_shell_quote() { + local value="$1" + value=${value//\'/\'\\\'\'} + printf "'%s'" "$value" +} +postgres_command() { + local python + python=$(postgres_shell_quote "$POSTGRES_CONNECT_PY") + printf '%s' "set +x; if [ -n \"\${CORESCOPE_READER_DATABASE_URL:-}\" ]; then exec \"\${CORESCOPE_QA_PYTHON:-python3}\" -c $python; elif [ -n \"\${PGSERVICE:-}\" ] || [ -n \"\${PGDATABASE:-}\" ]; then case \"\${PGDATABASE:-}\" in *://*) echo 'put connection URLs in CORESCOPE_READER_DATABASE_URL' >&2; exit 2;; esac; export PGCONNECT_TIMEOUT=10 PGCLIENTENCODING=UTF8; exec psql -X -qAt -w -v ON_ERROR_STOP=1 -v VERBOSITY=sqlstate; else echo 'configure a private PostgreSQL reader connection' >&2; exit 2; fi" +} +run_postgres() { + local cmd + cmd=$(postgres_command) + case "$POSTGRES_RUNNER" in + local) sh -c "$cmd" ;; + container) ssh_t "docker exec -i $(printf %q "$TARGET_CONTAINER") sh -c $(postgres_shell_quote "$cmd")" ;; + host) ssh_t "sh -c $(postgres_shell_quote "$cmd")" ;; + *) echo 'run_postgres: no reader runner resolved' >&2; return 127 ;; + esac | tr -d '\r' +} +resolve_postgres_runner() { + local runner out + POSTGRES_RUNNER="" + # An explicit local connection is authoritative: never silently fall through + # to a different database after a bad URL, elevated role, or failed import. + if [[ -n "${CORESCOPE_READER_DATABASE_URL:-}${PGSERVICE:-}${PGDATABASE:-}" ]]; then + local candidates=(local) + else + local candidates=(container host) + fi + for runner in "${candidates[@]}"; do + POSTGRES_RUNNER="$runner" + if out=$(postgres_probe_sql | run_postgres 2>>"$TMP/postgres-probe.err") && [[ "$out" == "$POSTGRES_PROBE_TOKEN" ]]; then return 0; fi + done + POSTGRES_RUNNER="" + return 1 +} +postgres_error_summary() { + # libpq connection errors can contain caller-supplied strings. Publish only + # SQLSTATE classes, retaining the complete diagnostic in the private temp file. + local summary + summary=$(sed -nE 's/.*(ERROR|FATAL):[[:space:]]+([0-9A-Z]{5}).*/PostgreSQL error \2/p' "$1" | tail -3) + if [[ -n "$summary" ]]; then printf '%s\n' "$summary" >&2; else + echo 'Check PostgreSQL 18 client availability and private reader connection settings.' >&2 + fi +} + +# Native SQLite CLI bindings from the original SQLite QA path (#1977). The +# blob-to-text cast keeps spaces/newlines out of dot-command argument parsing. +# -readonly forbids persistent writes; .parameter uses only a temporary table. +SQLITE_ARGS=(-readonly -batch -bail -init /dev/null -noheader -list) +SQLITE_PROBE_TOKEN="corescope-probe-ok" +SQLITE_RUNNER="" +sqlite_transmission_count_sql() { printf '.parameter init\n' printf '.parameter set :pubkey "cast(%s as text)"\n' "$(sql_hex_literal "$1")" printf 'SELECT COUNT(*) FROM transmissions WHERE from_pubkey = lower(:pubkey);\n' } - -# Capability probe: bind a known value and read it back. A version number only -# implies that .parameter works; binding something and getting it back proves it -# on the binary actually in front of us, which is the operator's, not ours. +transmission_count_sql() { + case "$SQL_BACKEND" in + sqlite) sqlite_transmission_count_sql "$1" ;; + postgres) postgres_transmission_count_sql "$1" ;; + *) echo 'Unsupported QA database backend' >&2; return 2 ;; + esac +} sqlite_probe_sql() { printf '.parameter init\n' printf '.parameter set :probe "cast(%s as text)"\n' "$(sql_hex_literal "$SQLITE_PROBE_TOKEN")" printf 'SELECT :probe;\n' } - -# Find a sqlite3 that can bind a parameter — in the container first, then on the -# host. Sets SQLITE_RUNNER; returns 1 if neither qualifies. There is deliberately -# no interpolating fallback: that would leave the vulnerable path in place under -# a nicer name. -# -# Probe stderr is collected rather than discarded, but only printed if BOTH -# probes fail. The container miss is the known-normal case — the app image has -# no sqlite3 (pure-Go driver, no CGO; Dockerfile:15) — so surfacing it on every -# run would be noise. resolve_sqlite_runner() { local probe out probe=$(sqlite_probe_sql) SQLITE_RUNNER="" - out=$(ssh_t "docker exec -i $(printf %q "$TARGET_CONTAINER") sqlite3 ${SQLITE_ARGS[*]} :memory:" \ - <<<"$probe" 2>>"$TMP/sqlite-probe.err") - if [[ "$out" == "$SQLITE_PROBE_TOKEN" ]]; then SQLITE_RUNNER="container"; return 0; fi - out=$(ssh_t "sqlite3 ${SQLITE_ARGS[*]} :memory:" <<<"$probe" 2>>"$TMP/sqlite-probe.err") - if [[ "$out" == "$SQLITE_PROBE_TOKEN" ]]; then SQLITE_RUNNER="host"; return 0; fi + if out=$(ssh_t "docker exec -i $(printf %q "$TARGET_CONTAINER") sqlite3 ${SQLITE_ARGS[*]} :memory:" \ + <<<"$probe" 2>>"$TMP/sqlite-probe.err" | tr -d '\r') && [[ "$out" == "$SQLITE_PROBE_TOKEN" ]]; then + SQLITE_RUNNER=container; return 0 + fi + if out=$(ssh_t "sqlite3 ${SQLITE_ARGS[*]} :memory:" <<<"$probe" 2>>"$TMP/sqlite-probe.err" | tr -d '\r') && [[ "$out" == "$SQLITE_PROBE_TOKEN" ]]; then + SQLITE_RUNNER=host; return 0 + fi return 1 } - -# Run SQL from stdin against TARGET_DB_PATH via the resolved runner. Stderr is -# left alone so the caller can capture it, and the exit status is sqlite3's. -# The SQL crosses on stdin, so only the container name and db path still need -# printf %q for the remote shell. docker exec needs -i to attach stdin. run_sqlite() { case "$SQLITE_RUNNER" in container) ssh_t "docker exec -i $(printf %q "$TARGET_CONTAINER") sqlite3 ${SQLITE_ARGS[*]} $(printf %q "$TARGET_DB_PATH")" ;; - host) ssh_t "sqlite3 ${SQLITE_ARGS[*]} $(printf %q "$TARGET_DB_PATH")" ;; - *) echo "run_sqlite: no runner resolved" >&2; return 127 ;; - esac + host) ssh_t "sqlite3 ${SQLITE_ARGS[*]} $(printf %q "$TARGET_DB_PATH")" ;; + *) echo 'run_sqlite: no runner resolved' >&2; return 127 ;; + esac | tr -d '\r' +} +read_sqlite_retain_count() { + if [[ -z "${TARGET_DB_PATH:-}" ]]; then + echo ' ❌ retain-failed: TARGET_DB_PATH is required for the selected SQLite reader' + return 1 + fi + if ! resolve_sqlite_runner; then + echo ' ❌ retain-failed: no SQLite client able to bind a parameter on the target' + cat "$TMP/sqlite-probe.err" >&2 + return 1 + fi + echo " SQLite reader runner: $SQLITE_RUNNER" + if ! RETAIN_COUNT=$(transmission_count_sql "$TEST_PUBKEY" | run_sqlite 2>"$TMP/sqlite.err"); then + echo " ❌ retain-failed: SQLite query failed via $SQLITE_RUNNER" + cat "$TMP/sqlite.err" >&2 + RETAIN_COUNT="" + return 1 + fi + if ! [[ "$RETAIN_COUNT" =~ ^[0-9]+$ ]]; then + echo ' ❌ retain-failed: SQLite query did not return one numeric count' + RETAIN_COUNT="" + return 1 + fi } - -# Read the retained-transmission count into RETAIN_COUNT. Prints a classified -# "retain-failed" line and returns 1 on failure, so §10.2 has exactly one place -# that increments $fails. read_retain_count() { RETAIN_COUNT="" local code if [[ -n "$ADMIN_API_TOKEN" ]]; then - # Read auth header from stdin so the token never enters argv (ps-safe). code=$(printf 'header = "Authorization: Bearer %s"\n' "$ADMIN_API_TOKEN" | \ curl -s -m "$CURL_TIMEOUT" -K - -o "$TMP/admin.json" -w "%{http_code}" \ - "$TARGET_URL/api/admin/transmissions?from_node=$TEST_PUBKEY&count=1" 2>/dev/null || echo "000") - if [[ "$code" == "200" ]]; then - RETAIN_COUNT=$(jq -r '.count // ((.transmissions // []) | length)' "$TMP/admin.json" 2>/dev/null || echo "") + "$TARGET_URL/api/admin/transmissions?from_node=$TEST_PUBKEY&count=1" 2>/dev/null || echo '000') + if [[ "$code" == 200 ]]; then + RETAIN_COUNT=$(jq -r '.count // ((.transmissions // []) | length)' "$TMP/admin.json" 2>/dev/null || echo '') fi - if [[ -n "$RETAIN_COUNT" ]]; then return 0; fi + if [[ "$RETAIN_COUNT" =~ ^[0-9]+$ ]]; then return 0; fi + RETAIN_COUNT="" fi - - if [[ -z "$TARGET_DB_PATH" ]]; then - echo " ❌ retain-failed: TARGET_DB_PATH unset and no ADMIN_API_TOKEN — cannot probe" + case "$SQL_BACKEND" in + sqlite) read_sqlite_retain_count; return $? ;; + postgres) ;; + *) echo ' ❌ retain-failed: unsupported QA database backend'; return 1 ;; + esac + if ! resolve_postgres_runner; then + echo ' ❌ retain-failed: no ready, restricted PostgreSQL reader with native parameter binding' + postgres_error_summary "$TMP/postgres-probe.err" return 1 fi - if ! resolve_sqlite_runner; then - echo " ❌ retain-failed: no sqlite3 able to bind a parameter on the target" - echo " tried: docker exec -i $TARGET_CONTAINER sqlite3, then sqlite3 on $TARGET_SSH_HOST" - echo " need: the sqlite3 CLI reachable over ssh, supporting '.parameter set'" - cat "$TMP/sqlite-probe.err" >&2 + echo " PostgreSQL reader runner: $POSTGRES_RUNNER" + if ! RETAIN_COUNT=$(transmission_count_sql "$TEST_PUBKEY" | run_postgres 2>"$TMP/postgres.err"); then + echo " ❌ retain-failed: PostgreSQL query failed via $POSTGRES_RUNNER" + postgres_error_summary "$TMP/postgres.err" + RETAIN_COUNT="" return 1 fi - echo " sqlite3 runner: $SQLITE_RUNNER" - if ! RETAIN_COUNT=$(run_sqlite <<<"$(transmission_count_sql "$TEST_PUBKEY")" 2>"$TMP/sqlite.err"); then - echo " ❌ retain-failed: sqlite3 query failed via $SQLITE_RUNNER" - cat "$TMP/sqlite.err" >&2 + if ! [[ "$RETAIN_COUNT" =~ ^[0-9]+$ ]]; then + echo ' ❌ retain-failed: PostgreSQL query did not return one numeric count' RETAIN_COUNT="" return 1 fi @@ -301,6 +386,7 @@ main() { TARGET_CONTAINER="${TARGET_CONTAINER:-}" TARGET_DB_PATH="${TARGET_DB_PATH:-}" ADMIN_API_TOKEN="${ADMIN_API_TOKEN:-}" + case "$SQL_BACKEND" in sqlite|postgres) ;; *) echo 'error: CORESCOPE_DB_BACKEND must be sqlite or postgres' >&2; exit 2 ;; esac if [[ -z "$TEST_PUBKEY" || -z "$TARGET_SSH_HOST" || -z "$TARGET_CONFIG_PATH" || -z "$TARGET_CONTAINER" ]]; then echo "error: TEST_NODE_PUBKEY, TARGET_SSH_HOST, TARGET_CONFIG_PATH, TARGET_CONTAINER are required" >&2 @@ -326,8 +412,12 @@ main() { echo "error: TARGET_CONFIG_PATH must be a sane absolute path" >&2 exit 2 fi - if [[ -n "$TARGET_DB_PATH" ]] && ! [[ "$TARGET_DB_PATH" =~ ^/[A-Za-z0-9_./-]+$ ]]; then - echo "error: TARGET_DB_PATH must be a sane absolute path" >&2 + if [[ "$SQL_BACKEND" == postgres && -n "$TARGET_DB_PATH" ]]; then + echo 'error: TARGET_DB_PATH is a SQLite path; configure the selected PostgreSQL reader connection' >&2 + exit 2 + fi + if [[ -n "$TARGET_DB_PATH" && ! "$TARGET_DB_PATH" =~ ^/[A-Za-z0-9_./-]+$ ]]; then + echo 'error: TARGET_DB_PATH must be a sane absolute SQLite path' >&2 exit 2 fi @@ -378,8 +468,7 @@ main() { echo "=== §10.2 verify packets retained in DB ===" if ! read_retain_count; then # read_retain_count already printed the classified reason. Counting here and - # nowhere else: the old code incremented $fails for the "TARGET_DB_PATH - # unset" case and then again for the empty count it left behind. + # nowhere else, so one failed reader probe remains one classified failure. fails=$((fails+1)) elif [[ "$RETAIN_COUNT" =~ ^[0-9]+$ ]] && (( RETAIN_COUNT > 0 )); then echo " ✅ DB retains $RETAIN_COUNT packets from $TEST_PUBKEY" diff --git a/qa/scripts/postgres-connect.py b/qa/scripts/postgres-connect.py new file mode 100644 index 000000000..74713dcef --- /dev/null +++ b/qa/scripts/postgres-connect.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +"""Run psql with connection credentials exclusively in private libpq variables. + +PGDATABASE does not expand a URL. This QA launcher accepts the application's +reader URL, including search_path, or preconfigured native libpq settings. +Queries arrive on stdin. Their caller is responsible for checking reader grants. +""" +import os +import re +import subprocess +import sys +import urllib.parse + + +def connection_env(source): + env = dict(source) + url = env.get("CORESCOPE_READER_DATABASE_URL", "") + if url: + try: + if re.search(r"%(?![0-9a-fA-F]{2})|[\x00-\x20\x7f]", url): + raise ValueError() + parsed = urllib.parse.urlsplit(url) + if ";" in parsed.query: + raise ValueError() + host = urllib.parse.unquote(parsed.hostname or "") + if parsed.scheme not in ("postgres", "postgresql") or not host or "," in host or not parsed.username or not parsed.path.strip("/") or parsed.fragment: + raise ValueError() + port = parsed.port + if port is None: + port = env.get("PGPORT") or "5432" + if not re.fullmatch(r"[0-9]+", str(port)) or not 1 <= int(port) <= 65535: + raise ValueError() + env.update(PGHOST=host, PGPORT=str(int(port)), + PGUSER=urllib.parse.unquote(parsed.username), + PGDATABASE=urllib.parse.unquote(parsed.path[1:])) + if parsed.password is not None: + env["PGPASSWORD"] = urllib.parse.unquote(parsed.password) + env.pop("PGSERVICE", None) + env.pop("PGHOSTADDR", None) + env.pop("PGSERVICEFILE", None) + env.pop("PGOPTIONS", None) + settings = {"sslmode": "PGSSLMODE", "sslrootcert": "PGSSLROOTCERT", "sslcert": "PGSSLCERT", + "sslkey": "PGSSLKEY", "sslpassword": "PGSSLPASSWORD", "connect_timeout": "PGCONNECT_TIMEOUT", + "target_session_attrs": "PGTARGETSESSIONATTRS", "application_name": "PGAPPNAME"} + seen = set() + for key, value in urllib.parse.parse_qsl(parsed.query, keep_blank_values=True, strict_parsing=True): + if key in seen: + raise ValueError() + seen.add(key) + if key == "search_path": + env["PGOPTIONS"] = "-c search_path=" + re.sub(r"([\\\s])", r"\\\1", value) + elif key in settings: + if key == "sslmode" and value not in ("disable", "allow", "prefer", "require", "verify-ca", "verify-full"): + raise ValueError() + env[settings[key]] = value + else: + raise ValueError() + except (ValueError, UnicodeError): + raise ValueError("invalid PostgreSQL reader URL; check its private configuration") from None + elif not env.get("PGSERVICE") and not env.get("PGDATABASE"): + raise ValueError("configure CORESCOPE_READER_DATABASE_URL or private libpq database/service settings") + elif "://" in env.get("PGDATABASE", ""): + raise ValueError("PGDATABASE requires a database name; put the URL in CORESCOPE_READER_DATABASE_URL") + try: + env["PGCONNECT_TIMEOUT"] = str(min(10, max(1, int(env.get("PGCONNECT_TIMEOUT", "10"))))) + except ValueError: + raise ValueError("invalid PostgreSQL connection timeout") from None + env["PGCLIENTENCODING"] = "UTF8" + env.setdefault("PGAPPNAME", "corescope-qa") + return env + + +if __name__ == "__main__": + try: + env = connection_env(os.environ) + result = subprocess.run(["psql", "-X", "-qAt", "-w", "-v", "ON_ERROR_STOP=1", "-v", "VERBOSITY=sqlstate"], + env=env, stderr=subprocess.PIPE, text=True, encoding="utf-8", errors="replace") + if result.returncode: + # libpq parse/connection errors can quote an invalid setting value. + # Keep server error classes; never relay those arbitrary strings. + codes = re.findall(r"^(?:ERROR|FATAL):\s+([0-9A-Z]{5})\s*$", result.stderr, re.MULTILINE) + if codes: + for code in codes[-3:]: + print("ERROR: " + code, file=sys.stderr) + else: + print("PostgreSQL client failed; check private connection settings and PostgreSQL 18 client availability", file=sys.stderr) + sys.exit(result.returncode) + except ValueError as error: + print(str(error), file=sys.stderr) + sys.exit(2) + except OSError: + print("psql is unavailable; install the PostgreSQL 18 client", file=sys.stderr) + sys.exit(127) diff --git a/qa/scripts/test-blacklist-postgres.sh b/qa/scripts/test-blacklist-postgres.sh new file mode 100644 index 000000000..0fd3fd386 --- /dev/null +++ b/qa/scripts/test-blacklist-postgres.sh @@ -0,0 +1,269 @@ +#!/usr/bin/env bash +# Native PostgreSQL regressions for the blacklist retention probe. No external +# endpoints are contacted: SSH is stubbed and all schemas/roles are disposable. +set -uo pipefail +set +x +export CORESCOPE_DB_BACKEND=postgres +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +# shellcheck source=blacklist-test.sh +. "$SCRIPT_DIR/blacklist-test.sh" +PYTHON="${CORESCOPE_QA_PYTHON:-python3}" +CONNECT="$SCRIPT_DIR/postgres-connect.py" +export QA_CONNECT="$CONNECT" +PASS=0; FAIL=0 +assert_eq() { + if [[ "$2" == "$3" ]]; then PASS=$((PASS+1)); else + FAIL=$((FAIL+1)); printf 'FAIL: %s — expected %q got %q\n' "$1" "$2" "$3" >&2 + fi +} +assert_true() { local label="$1"; shift; if "$@"; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); echo "FAIL: $label" >&2; fi; } +contains() { [[ "$1" == *"$2"* ]]; } +lacks() { [[ "$1" != *"$2"* ]]; } +hex_only() { [[ "$1" =~ ^\'[0-9a-f]*\'$ ]]; } +assert_eq 'hex of deadbeef' "'6465616462656566'" "$(sql_hex_literal deadbeef)" +assert_eq 'hex of empty value' "''" "$(sql_hex_literal '')" +for payload in "' OR 1=1 --" '"; DROP TABLE transmissions; --' 'a\b' '$(id) `id`' "$(printf 'a\nb')" 'héllo'; do + assert_true 'bound parameter alphabet is only hexadecimal' hex_only "$(sql_hex_literal "$payload")" +done +LONG=$(printf 'x%.0s' $(seq 1 4096)); LONG_HEX=$(sql_hex_literal "$LONG") +assert_true 'long bound parameter alphabet is hexadecimal' hex_only "$LONG_HEX" +assert_eq 'repeated lines are not collapsed by od' 8192 "$(( ${#LONG_HEX} - 2 ))" + +command -v "$PYTHON" >/dev/null || { echo 'FAIL: Python 3 is required' >&2; exit 1; } +"$PYTHON" "$SCRIPT_DIR/test-postgres-connect.py" || exit 1 +command -v psql >/dev/null || { echo 'FAIL: PostgreSQL 18 psql is required' >&2; exit 1; } +[[ -n "${CORESCOPE_TEST_POSTGRES_URL:-}" ]] || { echo 'FAIL: set CORESCOPE_TEST_POSTGRES_URL to a disposable PostgreSQL administrator URL' >&2; exit 1; } +FIXTURE_ROOT="$(cd "${TMPDIR:-/tmp}" && pwd -P)" +FIXTURE_DIR=$(mktemp -d "$FIXTURE_ROOT/corescope-qa-XXXXXX") +FIXTURE_DIR="$(cd "$FIXTURE_DIR" && pwd -P)" +SCHEMA="corescope_qa_${BASHPID}_${RANDOM}" +OWNER="${SCHEMA}_owner"; READER="${SCHEMA}_reader" +FIXTURE_PASSWORD=$(od -An -N24 -v -tx1 /dev/urandom | tr -d ' \n') +CREATED=0 +admin_sql() { CORESCOPE_READER_DATABASE_URL="$CORESCOPE_TEST_POSTGRES_URL" "$PYTHON" "$CONNECT" | tr -d '\r'; } +cleanup() { + local rc=$? + if [[ "$CREATED" == 1 && "$SCHEMA" =~ ^corescope_qa_[0-9]+_[0-9]+$ ]]; then + if ! admin_sql >/dev/null <&2; rc=1; fi + fi + case "$FIXTURE_DIR" in "$FIXTURE_ROOT"/corescope-qa-*) rm -rf -- "$FIXTURE_DIR" ;; *) echo 'FAIL: refusing unsafe temporary cleanup' >&2; rc=1 ;; esac + exit "$rc" +} +trap cleanup EXIT +# Build private role URLs without putting the original URL/password in argv or +# printing either. The connection launcher validates the URL before psql runs. +set_role_url() { + local base="$CORESCOPE_TEST_POSTGRES_URL" authority tail query part rest="" + authority=${base#*://}; tail=${authority#*/}; authority=${authority%%/*} + QA_ROLE_URL="${base%%://*}://$1:$FIXTURE_PASSWORD@${authority##*@}/${tail%%\?*}" + if [[ "$tail" == *\?* ]]; then + query=${tail#*\?} + while [[ -n "$query" ]]; do + part=${query%%&*}; if [[ "$query" == *'&'* ]]; then query=${query#*&}; else query=""; fi + [[ "${part%%=*}" == search_path ]] || rest+="${part}&" + done + fi + QA_ROLE_URL+="?${rest}search_path=$2" +} +if ! admin_sql >/dev/null <&2; exit 1; fi +CREATED=1 +set_role_url "$OWNER" "$SCHEMA"; OWNER_URL="$QA_ROLE_URL" +set_role_url "$READER" "$SCHEMA"; READER_URL="$QA_ROLE_URL" +set_role_url "$READER" "${SCHEMA}_bad"; BAD_URL="$QA_ROLE_URL" +set_role_url "$READER" "${SCHEMA}_empty"; EMPTY_URL="$QA_ROLE_URL" +set_role_url "$OWNER" "${SCHEMA}_import"; IMPORT_OWNER_URL="$QA_ROLE_URL" +set_role_url "$READER" "${SCHEMA}_import"; IMPORT_READER_URL="$QA_ROLE_URL" +MIGRATOR="${CORESCOPE_QA_MIGRATOR:-$FIXTURE_DIR/migrate}" +PREPARER="${CORESCOPE_QA_PREPARER:-$FIXTURE_DIR/prepare-fixture}" +if [[ -z "${CORESCOPE_QA_MIGRATOR:-}" ]]; then + (cd "$REPO_ROOT/cmd/migrate" && CGO_ENABLED=1 go build -o "$MIGRATOR" .) || exit 1 +fi +if [[ -z "${CORESCOPE_QA_PREPARER:-}" ]]; then + (cd "$REPO_ROOT/cmd/migrate" && CGO_ENABLED=1 go build -o "$PREPARER" ../../scripts/prepare-postgres-fixture.go) || exit 1 +fi +if ! CORESCOPE_DATABASE_URL="$OWNER_URL" CORESCOPE_USERS_DATABASE_URL="" "$MIGRATOR" >"$FIXTURE_DIR/bootstrap.out" 2>"$FIXTURE_DIR/bootstrap.err"; then + echo 'FAIL: shipping migration tool could not bootstrap the owner schema' >&2; exit 1 +fi +PK_A="0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" +PK_B="fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210" +PK_ABSENT="00000000000000000000000000000000000000000000000000000000000000ff" +PK_A_UPPER=$(printf '%s' "$PK_A" | tr 'a-f' 'A-F') +LONG_PK=$(printf 'ab%.0s' $(seq 1 5000)) +if ! admin_sql >/dev/null <&2; exit 1; fi +run_local() { CORESCOPE_READER_DATABASE_URL="$1" "$PYTHON" "$CONNECT" | tr -d '\r'; } +count() { transmission_count_sql "$1" | run_local "$2"; } + +# Positive controls must succeed, otherwise zero-valued injection results prove +# nothing. These exercise the actual canonical schema and restricted LOGIN role. +assert_eq 'fixture holds six rows' 6 "$(run_local "$READER_URL" <<<'SELECT count(*) FROM transmissions;')" +out=$(count "$PK_A" "$READER_URL"); rc=$? +assert_eq 'legitimate pubkey has three rows' 3 "$out" +assert_eq 'legitimate pubkey returns success' 0 "$rc" +assert_eq 'native parameter capability probe round-trips' "$POSTGRES_PROBE_TOKEN" "$(postgres_probe_sql | run_local "$READER_URL")" +# Derive the expected version from the shipping migrator's fresh schema. This +# catches a stale QA guard whenever the telemetry schema advances. +current_version=$(run_local "$READER_URL" <<<"SELECT version FROM corescope_schema WHERE kind='telemetry' AND ready;") +assert_true 'shipping schema has a current version' bash -c '[[ "$1" =~ ^[1-9][0-9]*$ ]]' _ "$current_version" +for spec in "unfinished|$current_version|false" "old|1|true" "future|$((current_version+1))|true" "invalid|0|true"; do + IFS='|' read -r label version ready <<<"$spec" + admin_sql >/dev/null <<<"UPDATE $SCHEMA.corescope_schema SET version=$version,ready=$ready WHERE kind='telemetry';" || exit 1 + out=$(count "$PK_A" "$READER_URL" 2>"$FIXTURE_DIR/ready.err"); rc=$? + assert_true "$label schema returns nonzero" test "$rc" -ne 0 + assert_true "$label schema reports not-ready SQLSTATE" contains "$(cat "$FIXTURE_DIR/ready.err")" '55000' + assert_eq "$label schema produces no count" '' "$out" + out=$(postgres_probe_sql | run_local "$READER_URL" 2>"$FIXTURE_DIR/probe-ready.err"); rc=$? + assert_true "$label schema capability probe returns nonzero" test "$rc" -ne 0 + assert_eq "$label schema capability probe produces no token" '' "$out" +done +admin_sql >/dev/null <<<"DELETE FROM $SCHEMA.corescope_schema WHERE kind='telemetry';" || exit 1 +out=$(count "$PK_A" "$READER_URL" 2>"$FIXTURE_DIR/missing-ready.err"); rc=$? +assert_true 'missing readiness marker returns nonzero' test "$rc" -ne 0 +assert_true 'missing readiness marker reports not-ready SQLSTATE' contains "$(cat "$FIXTURE_DIR/missing-ready.err")" '55000' +assert_eq 'missing readiness marker produces no count' '' "$out" +admin_sql >/dev/null <<<"INSERT INTO $SCHEMA.corescope_schema(kind,version,ready) VALUES('telemetry',$current_version,true);" || exit 1 +assert_eq 'current ready schema is accepted after rejected states' 3 "$(count "$PK_A" "$READER_URL")" +COUNT_SQL=$(transmission_count_sql "$PK_A") +assert_true 'query uses the real attribution column' contains "$COUNT_SQL" 'WHERE from_pubkey = lower(convert_from(decode($1' +assert_true 'query has no invented from_node column' lacks "$COUNT_SQL" from_node +assert_true 'query contains no raw pubkey' lacks "$COUNT_SQL" "$PK_A" +assert_eq 'other legitimate pubkey has one row' 1 "$(count "$PK_B" "$READER_URL")" +assert_eq 'upper-case key matches' 3 "$(count "$PK_A_UPPER" "$READER_URL")" +assert_eq 'absent key has no rows' 0 "$(count "$PK_ABSENT" "$READER_URL")" +assert_eq 'prefix is not an exact key' 0 "$(count "${PK_A:0:16}" "$READER_URL")" +for payload in "' OR 1=1 --" "') OR 1=1 --" '" OR 1=1 --' "x' OR from_pubkey IS NOT NULL --" "$PK_A' OR '1'='1" \ + '"); .shell id; --' "$(printf 'a\n.shell id\nSELECT 99;')" "$(printf 'a\n\\! id\nSELECT 99;')" '$(id) `id`' 'a\b'; do + out=$(count "$payload" "$READER_URL" 2>"$FIXTURE_DIR/payload.err"); rc=$? + assert_eq 'hostile pubkey is a literal, not SQL/meta-command/shell code' 0 "$out" + assert_eq 'hostile literal query succeeds' 0 "$rc" +done +assert_eq 'unsafe interpolated control would leak all rows' 6 "$(run_local "$READER_URL" <<<"SELECT count(*) FROM transmissions WHERE from_pubkey = '' OR 1=1 --';")" +admin_sql >/dev/null <"$FIXTURE_DIR/schema.err"); rc=$? + assert_true 'bad schema returns nonzero' test "$rc" -ne 0 + assert_true 'bad schema exposes its PostgreSQL error class' contains "$(cat "$FIXTURE_DIR/schema.err")" "${spec##*|}" + assert_eq 'bad schema produces no count' '' "$out" +done +# Prove actual grants, independently of the read-only transaction setting. +out=$(run_local "$READER_URL" <<<'BEGIN READ WRITE; DELETE FROM transmissions;' 2>"$FIXTURE_DIR/write.err"); rc=$? +assert_true 'restricted reader cannot delete even in read-write transaction' test "$rc" -ne 0 +assert_true 'delete fails for privilege, not transaction mode' contains "$(cat "$FIXTURE_DIR/write.err")" '42501' +out=$(count "$PK_A" "$OWNER_URL" 2>"$FIXTURE_DIR/owner.err"); rc=$? +assert_true 'owner credential refused by runtime probe' test "$rc" -ne 0 +assert_eq 'owner refusal has no count' '' "$out" +admin_sql >/dev/null <<<"UPDATE $SCHEMA.corescope_schema SET ready=false;" +out=$(count "$PK_A" "$READER_URL" 2>"$FIXTURE_DIR/ready.err"); rc=$? +assert_true 'unfinished import refused' test "$rc" -ne 0 +assert_eq 'unfinished import produces no count' '' "$out" +admin_sql >/dev/null <<<"UPDATE $SCHEMA.corescope_schema SET ready=true;" + +# Preserve the committed fixture comparison through its explicit offline repair +# and shipping importer. The original SQLite file is never modified or queried +# as a runtime backend. +REAL_FIXTURE="$REPO_ROOT/test-fixtures/e2e-fixture.db" +if ! "$PREPARER" -source "$REAL_FIXTURE" -destination "$FIXTURE_DIR/prepared.sqlite" >"$FIXTURE_DIR/prepare.out" 2>"$FIXTURE_DIR/prepare.err"; then + echo 'FAIL: offline historical fixture preparation failed' >&2; exit 1 +fi +if ! CORESCOPE_DATABASE_URL="$IMPORT_OWNER_URL" CORESCOPE_USERS_DATABASE_URL="" "$MIGRATOR" -offline -from-sqlite "$FIXTURE_DIR/prepared.sqlite" -state-dir "$FIXTURE_DIR/import-state" >"$FIXTURE_DIR/import.out" 2>"$FIXTURE_DIR/import.err"; then + echo 'FAIL: shipping importer failed for the historical fixture' >&2; exit 1 +fi +admin_sql >/dev/null <<<"GRANT SELECT ON ALL TABLES IN SCHEMA ${SCHEMA}_import TO $READER;" +real_row=$(run_local "$IMPORT_READER_URL" <<<'SELECT from_pubkey,count(*) FROM transmissions WHERE from_pubkey IS NOT NULL GROUP BY from_pubkey ORDER BY count(*) DESC,from_pubkey LIMIT 1;') +real_pk=${real_row%|*}; real_n=${real_row##*|} +assert_true 'imported historical fixture has attributed pubkey' bash -c '[[ "$1" =~ ^[0-9a-f]{64}$ ]]' _ "$real_pk" +assert_true 'imported fixture has positive retained count' bash -c '[[ "$1" =~ ^[1-9][0-9]*$ ]]' _ "$real_n" +assert_eq 'bound query matches direct count on imported fixture' "$real_n" "$(count "$real_pk" "$IMPORT_READER_URL")" + +# Transport uses the real query/client but stubs all SSH/container activity. +STUB_ARGV="$FIXTURE_DIR/ssh.argv"; STUB_STDIN="$FIXTURE_DIR/ssh.stdin" +STUB_CONTAINER=0 +ssh_t() { + local cmd="$1" input + printf '%s\n' "$cmd" >>"$STUB_ARGV" + input=$(cat); printf '%s\n' "$input" >>"$STUB_STDIN" + case "$cmd" in + 'docker exec -i '*) + if [[ "$STUB_CONTAINER" != 1 ]]; then echo 'client unavailable in stub container' >&2; return 127; fi + cmd=${cmd#docker exec -i }; cmd=${cmd#* } ;; + esac + printf '%s\n' "$input" | CORESCOPE_READER_DATABASE_URL="$READER_URL" bash -c "$cmd" +} +reset_stub() { : >"$STUB_ARGV"; : >"$STUB_STDIN"; } +TMP="$FIXTURE_DIR"; ADMIN_API_TOKEN=''; TEST_PUBKEY="$PK_A" +TARGET_CONTAINER='corescope-stub'; TARGET_SSH_HOST='stub-host' +unset CORESCOPE_READER_DATABASE_URL PGSERVICE PGDATABASE +for runner in host container; do + reset_stub; [[ "$runner" == container ]] && STUB_CONTAINER=1 + read_retain_count >/dev/null 2>"$FIXTURE_DIR/transport.err"; rc=$? + assert_eq 'remote reader count succeeds' 0 "$rc" + assert_eq 'resolved runner is expected target' "$runner" "${POSTGRES_RUNNER:-}" + assert_eq 'transport preserves retained count' 3 "$RETAIN_COUNT" + argv=$(cat "$STUB_ARGV"); input=$(cat "$STUB_STDIN") + assert_true 'pubkey is absent from remote argv' lacks "$argv" "$PK_A" + assert_true 'credentials are absent from remote argv' lacks "$argv" "$FIXTURE_PASSWORD" + assert_true 'SQL is absent from remote argv' lacks "$argv" 'SELECT' + assert_true 'native parameter query arrives on stdin' contains "$input" 'decode($1' + assert_true 'raw pubkey is absent from stdin' lacks "$input" "$PK_A" +done +reset_stub +CORESCOPE_READER_DATABASE_URL="$READER_URL" read_retain_count >/dev/null 2>"$FIXTURE_DIR/local.err"; rc=$? +assert_eq 'explicit local reader URL succeeds' 0 "$rc" +assert_eq 'explicit URL selects local client' local "${POSTGRES_RUNNER:-}" +assert_eq 'explicit URL does not contact SSH' '' "$(cat "$STUB_ARGV")" +# Missing/unusable binding capability and query errors must fail closed. +ssh_t() { cat >/dev/null; printf '\\bind unsupported\n0\n'; } +read_retain_count >/dev/null 2>&1; rc=$? +assert_eq 'client that does not bind is rejected' 1 "$rc" +assert_eq 'failed capability leaves no count' '' "$RETAIN_COUNT" +POSTGRES_RUNNER='' +if run_postgres /dev/null 2>&1; then FAIL=$((FAIL+1)); echo 'FAIL: unresolved runner accepted' >&2; else PASS=$((PASS+1)); fi +for malformed in 'postgresql://test:secret-sentinel@/db' 'postgresql://test:secret-sentinel@localhost/db?unsupported=yes' 'postgresql://test:secret-sentinel%xx@localhost/db' "${READER_URL}&target_session_attrs=secret-sentinel"; do + out=$(CORESCOPE_READER_DATABASE_URL="$malformed" "$PYTHON" "$CONNECT" &1); rc=$? + assert_true 'malformed reader URL fails' test "$rc" -ne 0 + assert_true 'malformed URL never reveals credentials' lacks "$out" 'secret-sentinel' +done +# Native libpq configuration is also accepted without putting values in argv. +out=$(CORESCOPE_READER_DATABASE_URL="$READER_URL" "$PYTHON" -c 'import os,runpy,subprocess,sys; m=runpy.run_path(os.environ["QA_CONNECT"]); e=m["connection_env"](os.environ); e.pop("CORESCOPE_READER_DATABASE_URL",None); sys.exit(subprocess.run(["psql","-X","-qAt","-w","-v","ON_ERROR_STOP=1"],env=e).returncode)' <<<'SELECT 1977;' 2>"$FIXTURE_DIR/libpq.err" | tr -d '\r') +assert_eq 'private libpq fields reach the real database' 1977 "$out" +space_url=${READER_URL/search_path=$SCHEMA/search_path=$SCHEMA%2C%20pg_catalog} +assert_eq 'search_path option whitespace is escaped on the real client' "$SCHEMA, pg_catalog" "$(run_local "$space_url" <<<"SELECT current_setting('search_path');")" +echo "test-blacklist-sql.sh: $PASS passed, $FAIL failed" +[[ "$FAIL" == 0 ]] diff --git a/qa/scripts/test-blacklist-sql.sh b/qa/scripts/test-blacklist-sql.sh index 4dfdc8b4b..7e59014c1 100755 --- a/qa/scripts/test-blacklist-sql.sh +++ b/qa/scripts/test-blacklist-sql.sh @@ -1,287 +1,12 @@ #!/usr/bin/env bash -# test-blacklist-sql.sh — unit tests for the §10.2 SQL construction in -# qa/scripts/blacklist-test.sh (issue #1977). Sources the script and exercises -# its pure helpers, plus a real local sqlite3 against throwaway fixture DBs. -# -# Run: bash qa/scripts/test-blacklist-sql.sh -# Exits non-zero if any case fails. -# -# The point of the sqlite3 group is that BOTH directions are asserted. A test -# that only checks "the injection payload returns 0" passes just as happily when -# the query is silently broken and returns 0 for everything, so the legitimate -# pubkey must be shown to still return the row it should. -# -# The fixture schema is NOT hand-written: it is the transmissions CREATE TABLE -# extracted from cmd/ingestor/db.go, and the query is also run against the -# committed staging-captured test-fixtures/e2e-fixture.db. An earlier version -# invented a `from_node` column that no CoreScope database has, and so passed -# while the real probe could never succeed. - -set -uo pipefail - +# Run the selected native engine's full SQL binding/security regressions. +# The default SQLite job deliberately requires no PostgreSQL service. +set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" -INGESTOR_DB_GO="$REPO_ROOT/cmd/ingestor/db.go" -REAL_FIXTURE="$REPO_ROOT/test-fixtures/e2e-fixture.db" -# shellcheck source=blacklist-test.sh -. "$SCRIPT_DIR/blacklist-test.sh" - -PASS=0 -FAIL=0 - -assert_eq() { - local label="$1" expected="$2" actual="$3" - if [ "$expected" = "$actual" ]; then - PASS=$((PASS + 1)) - else - FAIL=$((FAIL + 1)) - echo "FAIL: $label — expected '$expected' got '$actual'" >&2 - fi -} - -assert_match() { - local label="$1" pattern="$2" actual="$3" - if [[ "$actual" =~ $pattern ]]; then - PASS=$((PASS + 1)) - else - FAIL=$((FAIL + 1)) - echo "FAIL: $label — '$actual' does not match /$pattern/" >&2 - fi -} - -assert_true() { - local label="$1"; shift - if "$@"; then PASS=$((PASS + 1)); else - FAIL=$((FAIL + 1)); echo "FAIL: $label" >&2 - fi -} - -contains() { [[ "$1" == *"$2"* ]]; } -lacks() { [[ "$1" != *"$2"* ]]; } - -# ----- sql_hex_literal ------------------------------------------------------ -# The security property: whatever goes in, the SQL text it produces is drawn -# from [0-9a-f] only. No caller-supplied byte can close a string literal or add -# a dot-command argument. Needs no sqlite3, so this group always runs. -assert_eq "hex of deadbeef" "x'6465616462656566'" "$(sql_hex_literal deadbeef)" -assert_eq "hex of empty" "x''" "$(sql_hex_literal "")" - -HEX_ONLY="^x'[0-9a-f]*'\$" -assert_match "alphabet: sql quote payload" "$HEX_ONLY" "$(sql_hex_literal "' OR 1=1 --")" -assert_match "alphabet: drop table" "$HEX_ONLY" "$(sql_hex_literal '"; DROP TABLE transmissions; --')" -assert_match "alphabet: backslash" "$HEX_ONLY" "$(sql_hex_literal 'a\b')" -assert_match "alphabet: dollar and backtick" "$HEX_ONLY" "$(sql_hex_literal '$(id) `id`')" -assert_match "alphabet: embedded newline" "$HEX_ONLY" "$(sql_hex_literal "$(printf 'a\nb')")" -assert_match "alphabet: multibyte" "$HEX_ONLY" "$(sql_hex_literal 'héllo')" - -# `od` without -v collapses runs of identical lines to '*'. A long repetitive -# value is the case that catches losing the flag. -LONG=$(printf 'x%.0s' $(seq 1 4096)) -LONG_HEX=$(sql_hex_literal "$LONG") -assert_match "alphabet: 4096 repeated bytes" "$HEX_ONLY" "$LONG_HEX" -# 4096 bytes → 8192 hex digits, plus the 3 chars of x''. A collapsed run would -# be far shorter and would also fail the alphabet check on '*'. -assert_eq "no od line-collapse in 4096-byte value" "8192" "$(( ${#LONG_HEX} - 3 ))" - -# ----- query text ------------------------------------------------------------- -# Synthetic 32-byte pubkeys in the ingestor's form (hex.EncodeToString → lowercase). -PK_A="0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" -PK_B="fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210" -PK_ABSENT="00000000000000000000000000000000000000000000000000000000000000ff" -PK_A_UPPER=$(printf '%s' "$PK_A" | tr 'a-f' 'A-F') - -COUNT_SQL=$(transmission_count_sql "$PK_A") -assert_true "query targets transmissions.from_pubkey" contains "$COUNT_SQL" "WHERE from_pubkey = lower(:pubkey)" -assert_true "query does not reference from_node" lacks "$COUNT_SQL" "from_node" -assert_true "query text does not carry the raw pubkey" lacks "$COUNT_SQL" "$PK_A" - -# ----- schema taken from the ingestor ------------------------------------------ -# Pull the transmissions DDL out of cmd/ingestor/db.go instead of restating it. -transmissions_ddl() { - awk '/CREATE TABLE IF NOT EXISTS transmissions \(/{p=1} p{print} p&&/^[[:space:]]*\);/{exit}' "$INGESTOR_DB_GO" -} -DDL=$(transmissions_ddl) -assert_match "ingestor DDL extracted" "CREATE TABLE IF NOT EXISTS transmissions \\(" "$DDL" -assert_match "ingestor DDL has from_pubkey" "from_pubkey[[:space:]]+TEXT" "$DDL" -assert_true "ingestor DDL has no from_node" lacks "$DDL" "from_node" - -# ----- against a real sqlite3 ---------------------------------------------- -if ! command -v sqlite3 >/dev/null 2>&1; then - echo "SKIP: sqlite3 not on PATH — skipping the ${#SQLITE_ARGS[@]}-flag query group" >&2 - echo " (the alphabet, query-text and schema assertions above still ran)" >&2 -else - FIXTURE_DIR=$(mktemp -d) - trap 'rm -rf "$FIXTURE_DIR"' EXIT - DB="$FIXTURE_DIR/fixture.db" - LEGACY_DB="$FIXTURE_DIR/legacy-from-node.db" - EMPTY_DB="$FIXTURE_DIR/no-table.db" - - # Schema-realistic fixture: the ingestor's own DDL. Three ADVERTs from A, one - # from B, and two non-ADVERT rows whose from_pubkey is NULL (the ingestor only - # attributes ADVERTs), so a structural injection has 6 rows to leak. - printf '%s\n' "$DDL" | sqlite3 "$DB" - sqlite3 "$DB" <&1); rc=$? - assert_eq "payload binds literally: $(printf %q "$payload")" "0" "$out" - assert_eq "payload exit 0: $(printf %q "$payload")" "0" "$rc" - done - - # Interpolating the same payload the old way returns the whole table. This is - # the behaviour the change removes; asserting it keeps the test honest about - # what "0" above is worth. - legacy="SELECT COUNT(*) FROM transmissions WHERE from_pubkey = '' OR 1=1 --';" - assert_eq "interpolated form leaks the table" "6" "$(run_local "$DB" <<<"$legacy")" - - # Multibyte, whitespace, empty and long values bind as themselves. - LONG_PK=$(printf 'ab%.0s' $(seq 1 5000)) - sqlite3 "$DB" "INSERT INTO transmissions(raw_hex,hash,first_seen,from_pubkey) VALUES - ('00','m1','t','héllo wörld'),('00','m2','t',''),('00','m3','t',' '),('00','m4','t','$LONG_PK');" - assert_eq "multibyte value with a space binds" "1" "$(count 'héllo wörld' "$DB")" - assert_eq "empty value binds as ''" "1" "$(count '' "$DB")" - assert_eq "whitespace value binds as itself" "1" "$(count ' ' "$DB")" - assert_eq "10000-byte value binds" "1" "$(count "$LONG_PK" "$DB")" - - # REAL DATA: the committed staging-captured fixture. Its most-attributed - # pubkey, counted by the script's query, must equal a direct count. - if [ -f "$REAL_FIXTURE" ]; then - cp "$REAL_FIXTURE" "$FIXTURE_DIR/real.db" - real_pk=$(run_local "$FIXTURE_DIR/real.db" <<<"SELECT from_pubkey FROM transmissions WHERE from_pubkey IS NOT NULL GROUP BY from_pubkey ORDER BY COUNT(*) DESC, from_pubkey LIMIT 1;") - real_n=$(run_local "$FIXTURE_DIR/real.db" <<<"SELECT COUNT(*) FROM transmissions WHERE from_pubkey = '$real_pk';") - assert_match "real fixture has an attributed pubkey" '^[0-9a-f]{64}$' "$real_pk" - assert_match "real fixture count > 0" '^[1-9][0-9]*$' "$real_n" - assert_eq "script query on real fixture" "$real_n" "$(count "$real_pk" "$FIXTURE_DIR/real.db")" - else - FAIL=$((FAIL + 1)); echo "FAIL: $REAL_FIXTURE missing" >&2 - fi - - # ERROR SURFACING: a broken query must be distinguishable from an empty - # result — non-zero exit and something on stderr, not a silent "" or "0". - for bad in "$LEGACY_DB:no such column: from_pubkey" "$EMPTY_DB:no such table: transmissions"; do - bad_db=${bad%%:*}; want=${bad#*:} - err_file="$FIXTURE_DIR/err" - out=$(count "$PK_A" "$bad_db" 2>"$err_file"); rc=$? - assert_true "$want → non-zero exit (got $rc)" test "$rc" -ne 0 - assert_true "$want → named on stderr" contains "$(cat "$err_file")" "$want" - assert_eq "$want → no count on stdout" "" "$out" - done - - # ----- runner selection and transport, with ssh stubbed --------------------- - # ssh_t is replaced by a stub that logs its argv, captures stdin, and runs the - # remote command string through a real `bash -c`, so the script's own quoting - # is what gets parsed. "docker exec -i NAME" is peeled off when the stub - # container is said to have sqlite3; otherwise it fails like the app image. - STUB_ARGV="$FIXTURE_DIR/ssh.argv"; STUB_STDIN="$FIXTURE_DIR/ssh.stdin" - STUB_CONTAINER_SQLITE=0 - STUB_PATH="$PATH" - ssh_t() { - local cmd="$1" in - printf '%s\n' "$*" >>"$STUB_ARGV" - in=$(cat); printf '%s\n' "$in" >>"$STUB_STDIN" - case "$cmd" in - "docker exec -i "*) - if [ "$STUB_CONTAINER_SQLITE" != 1 ]; then - echo 'exec: "sqlite3": executable file not found in $PATH' >&2; return 126 - fi - cmd=${cmd#docker exec -i }; cmd=${cmd#* } ;; - esac - printf '%s\n' "$in" | PATH="$STUB_PATH" bash -c "$cmd" - } - reset_stub() { : >"$STUB_ARGV"; : >"$STUB_STDIN"; } - TMP="$FIXTURE_DIR"; ADMIN_API_TOKEN=""; TARGET_CONTAINER="corescope-stub" - TARGET_SSH_HOST="stub-host"; TARGET_DB_PATH="$DB"; TEST_PUBKEY="$PK_A" - - # Container has no sqlite3 → host runner, correct count. - reset_stub - read_retain_count >/dev/null 2>&1; rc=$? - assert_eq "fallback: rc" "0" "$rc" - assert_eq "fallback: runner" "host" "$SQLITE_RUNNER" - assert_eq "fallback: count" "3" "$RETAIN_COUNT" - argv=$(cat "$STUB_ARGV"); stdin=$(cat "$STUB_STDIN") - assert_true "transport: container tried first" contains "$(head -1 "$STUB_ARGV")" "docker exec -i corescope-stub sqlite3" - assert_true "transport: pubkey not in remote argv" lacks "$argv" "$PK_A" - assert_true "transport: SQL not in remote argv" lacks "$argv" "SELECT" - assert_true "transport: column not in remote argv" lacks "$argv" "from_pubkey" - assert_true "transport: SQL arrives on stdin" contains "$stdin" "SELECT COUNT(*) FROM transmissions WHERE from_pubkey = lower(:pubkey);" - assert_true "transport: pubkey arrives hex-encoded on stdin" contains "$stdin" "$(sql_hex_literal "$PK_A")" - assert_true "transport: raw pubkey never on stdin" lacks "$stdin" "$PK_A" - - # Container has sqlite3 → container runner. - reset_stub; STUB_CONTAINER_SQLITE=1 - read_retain_count >/dev/null 2>&1; rc=$? - assert_eq "container: rc" "0" "$rc" - assert_eq "container: runner" "container" "$SQLITE_RUNNER" - assert_eq "container: count" "3" "$RETAIN_COUNT" - STUB_CONTAINER_SQLITE=0 - - # A sqlite3 that cannot bind (.parameter prints help to stdout, exit 0, - # parameter left NULL) must be rejected by the probe, not trusted. - FAKE_BIN="$FIXTURE_DIR/fakebin"; mkdir -p "$FAKE_BIN" - cat >"$FAKE_BIN/sqlite3" <<'FAKE' -#!/usr/bin/env bash -cat >/dev/null -echo ".parameter CMD ... Manage SQL parameter bindings" -echo "0" -FAKE - chmod +x "$FAKE_BIN/sqlite3" - reset_stub; STUB_PATH="$FAKE_BIN:$PATH" - read_retain_count >/dev/null 2>&1; rc=$? - assert_eq "unbindable sqlite3: rejected" "1" "$rc" - assert_eq "unbindable sqlite3: no runner" "" "$SQLITE_RUNNER" - assert_eq "unbindable sqlite3: no count" "" "$RETAIN_COUNT" - STUB_PATH="$PATH" - - # A query error on the target is a failure with no count, never "0". - reset_stub; TARGET_DB_PATH="$LEGACY_DB" - read_retain_count >/dev/null 2>&1; rc=$? - assert_eq "remote schema error: rc" "1" "$rc" - assert_eq "remote schema error: count" "" "$RETAIN_COUNT" - assert_true "remote schema error: stderr kept" contains "$(cat "$TMP/sqlite.err")" "no such column: from_pubkey" - TARGET_DB_PATH="$DB" - - # run_sqlite with no resolved runner must refuse rather than guess. - SQLITE_RUNNER="" - if run_sqlite /dev/null 2>&1; then - FAIL=$((FAIL + 1)); echo "FAIL: run_sqlite with no runner — expected non-zero exit" >&2 - else - PASS=$((PASS + 1)) - fi -fi - -echo "test-blacklist-sql.sh: $PASS passed, $FAIL failed" -[ "$FAIL" -eq 0 ] +case "${CORESCOPE_TEST_BACKEND:-sqlite}" in + sqlite|postgres) + export CORESCOPE_DB_BACKEND="${CORESCOPE_TEST_BACKEND:-sqlite}" + exec bash "$SCRIPT_DIR/test-blacklist-$CORESCOPE_DB_BACKEND.sh" + ;; + *) echo 'FAIL: unsupported CORESCOPE_TEST_BACKEND; choose sqlite or postgres' >&2; exit 2 ;; +esac diff --git a/qa/scripts/test-blacklist-sqlite.sh b/qa/scripts/test-blacklist-sqlite.sh new file mode 100644 index 000000000..e38139c5d --- /dev/null +++ b/qa/scripts/test-blacklist-sqlite.sh @@ -0,0 +1,303 @@ +#!/usr/bin/env bash +# Native SQLite binding/transport regressions for blacklist retention. No SSH +# endpoint is contacted. Canonical fixtures come from the shipping bootstrap. +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +REAL_FIXTURE="$REPO_ROOT/test-fixtures/e2e-fixture.db" +REAL_FIXTURE_SHA=$(sha256sum "$REAL_FIXTURE") || exit 1 +export CORESCOPE_DB_BACKEND=sqlite +# shellcheck source=blacklist-test.sh +. "$SCRIPT_DIR/blacklist-test.sh" + +PASS=0 +FAIL=0 + +assert_eq() { + local label="$1" expected="$2" actual="$3" + if [ "$expected" = "$actual" ]; then + PASS=$((PASS + 1)) + else + FAIL=$((FAIL + 1)) + echo "FAIL: $label — expected '$expected' got '$actual'" >&2 + fi +} + +assert_match() { + local label="$1" pattern="$2" actual="$3" + if [[ "$actual" =~ $pattern ]]; then + PASS=$((PASS + 1)) + else + FAIL=$((FAIL + 1)) + echo "FAIL: $label — '$actual' does not match /$pattern/" >&2 + fi +} + +assert_true() { + local label="$1"; shift + if "$@"; then PASS=$((PASS + 1)); else + FAIL=$((FAIL + 1)); echo "FAIL: $label" >&2 + fi +} + +contains() { [[ "$1" == *"$2"* ]]; } +lacks() { [[ "$1" != *"$2"* ]]; } + +# ----- sql_hex_literal ------------------------------------------------------ +# The security property: whatever goes in, the SQL text it produces is drawn +# from [0-9a-f] only. No caller-supplied byte can close a string literal or add +# a dot-command argument. Needs no sqlite3, so this group always runs. +assert_eq "hex of deadbeef" "x'6465616462656566'" "$(sql_hex_literal deadbeef)" +assert_eq "hex of empty" "x''" "$(sql_hex_literal "")" + +HEX_ONLY="^x'[0-9a-f]*'\$" +assert_match "alphabet: sql quote payload" "$HEX_ONLY" "$(sql_hex_literal "' OR 1=1 --")" +assert_match "alphabet: drop table" "$HEX_ONLY" "$(sql_hex_literal '"; DROP TABLE transmissions; --')" +assert_match "alphabet: backslash" "$HEX_ONLY" "$(sql_hex_literal 'a\b')" +assert_match "alphabet: dollar and backtick" "$HEX_ONLY" "$(sql_hex_literal '$(id) `id`')" +assert_match "alphabet: embedded newline" "$HEX_ONLY" "$(sql_hex_literal "$(printf 'a\nb')")" +assert_match "alphabet: multibyte" "$HEX_ONLY" "$(sql_hex_literal 'héllo')" + +# `od` without -v collapses runs of identical lines to '*'. A long repetitive +# value is the case that catches losing the flag. +LONG=$(printf 'x%.0s' $(seq 1 4096)) +LONG_HEX=$(sql_hex_literal "$LONG") +assert_match "alphabet: 4096 repeated bytes" "$HEX_ONLY" "$LONG_HEX" +# 4096 bytes → 8192 hex digits, plus the 3 chars of x''. A collapsed run would +# be far shorter and would also fail the alphabet check on '*'. +assert_eq "no od line-collapse in 4096-byte value" "8192" "$(( ${#LONG_HEX} - 3 ))" + +# ----- query text ------------------------------------------------------------- +# Synthetic 32-byte pubkeys in the ingestor's form (hex.EncodeToString → lowercase). +PK_A="0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" +PK_B="fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210" +PK_ABSENT="00000000000000000000000000000000000000000000000000000000000000ff" +PK_A_UPPER=$(printf '%s' "$PK_A" | tr 'a-f' 'A-F') + +COUNT_SQL=$(transmission_count_sql "$PK_A") +assert_true "query targets transmissions.from_pubkey" contains "$COUNT_SQL" "WHERE from_pubkey = lower(:pubkey)" +assert_true "query does not reference from_node" lacks "$COUNT_SQL" "from_node" +assert_true "query text does not carry the raw pubkey" lacks "$COUNT_SQL" "$PK_A" + +command -v sqlite3 >/dev/null 2>&1 || { echo 'FAIL: sqlite3 CLI is required for the selected SQLite backend' >&2; exit 1; } +FIXTURE_ROOT="$(cd "${TMPDIR:-/tmp}" && pwd -P)" +FIXTURE_DIR=$(mktemp -d "$FIXTURE_ROOT/corescope-qa-sqlite-XXXXXX") +cleanup() { + local rc=$? + case "$FIXTURE_DIR" in "$FIXTURE_ROOT"/corescope-qa-sqlite-*) rm -rf -- "$FIXTURE_DIR" ;; *) echo 'FAIL: refusing unsafe temporary cleanup' >&2; rc=1 ;; esac + exit "$rc" +} +trap cleanup EXIT +DB="$FIXTURE_DIR/fixture.db" +LEGACY_DB="$FIXTURE_DIR/legacy-from-node.db" +EMPTY_DB="$FIXTURE_DIR/no-table.db" +mkdir -p "$FIXTURE_DIR/state" +SELECTION_FILE="$FIXTURE_DIR/state/storage-selection.json" +case "$(uname -s)" in MINGW*|MSYS*|CYGWIN*) SELECTION_FILE=$(cygpath -w "$SELECTION_FILE") ;; esac +MIGRATOR="${CORESCOPE_QA_MIGRATOR:-$FIXTURE_DIR/migrate}" +if [[ -z "${CORESCOPE_QA_MIGRATOR:-}" ]]; then + (cd "$REPO_ROOT/cmd/migrate" && CGO_ENABLED=1 go build -o "$MIGRATOR" .) || exit 1 +fi +if ! CORESCOPE_DATABASE_URL='' CORESCOPE_USERS_DATABASE_URL='' CORESCOPE_USERS_OWNER_DATABASE_URL='' "$MIGRATOR" \ + -storage-action=init -backend=sqlite -offline -config-dir "$FIXTURE_DIR" -selection-file "$SELECTION_FILE" \ + -sqlite-path "$DB" -users-sqlite-path "$FIXTURE_DIR/users.db" >"$FIXTURE_DIR/bootstrap.out" 2>"$FIXTURE_DIR/bootstrap.err"; then + echo 'FAIL: shipping migration tool could not bootstrap native SQLite' >&2 + cat "$FIXTURE_DIR/bootstrap.err" >&2; exit 1 +fi +DDL=$(sqlite3 -readonly "$DB" "SELECT sql FROM sqlite_master WHERE type='table' AND name='transmissions';" | tr -d '\r') +assert_match 'shipping schema is native transmissions table' 'CREATE TABLE.*transmissions' "$DDL" +assert_match 'shipping schema has from_pubkey TEXT' 'from_pubkey[[:space:]]+TEXT' "$DDL" +assert_true 'shipping schema has no invented from_node' lacks "$DDL" from_node + +# Schema-realistic fixture: shipping SQLite bootstrap. Three ADVERTs from A, one +# from B, and two non-ADVERT rows whose from_pubkey is NULL (the ingestor only +# attributes ADVERTs), so a structural injection has 6 rows to leak. +sqlite3 "$DB" <&1); rc=$? + assert_eq "payload binds literally: $(printf %q "$payload")" "0" "$out" + assert_eq "payload exit 0: $(printf %q "$payload")" "0" "$rc" +done + +# Interpolating the same payload the old way returns the whole table. This is +# the behaviour the change removes; asserting it keeps the test honest about +# what "0" above is worth. +legacy="SELECT COUNT(*) FROM transmissions WHERE from_pubkey = '' OR 1=1 --';" +assert_eq "interpolated form leaks the table" "6" "$(run_local "$DB" <<<"$legacy")" + +# Multibyte, whitespace, empty and long values bind as themselves. +LONG_PK=$(printf 'ab%.0s' $(seq 1 5000)) +sqlite3 "$DB" "INSERT INTO transmissions(raw_hex,hash,first_seen,from_pubkey) VALUES + ('00','m1','t','héllo wörld'),('00','m2','t',''),('00','m3','t',' '),('00','m4','t','$LONG_PK');" +assert_eq "multibyte value with a space binds" "1" "$(count 'héllo wörld' "$DB")" +assert_eq "empty value binds as ''" "1" "$(count '' "$DB")" +assert_eq "whitespace value binds as itself" "1" "$(count ' ' "$DB")" +assert_eq "10000-byte value binds" "1" "$(count "$LONG_PK" "$DB")" + +out=$(run_local "$DB" <<<'DELETE FROM transmissions;' 2>"$FIXTURE_DIR/write.err"); rc=$? +assert_true 'native reader refuses writes' test "$rc" -ne 0 +assert_true 'write refusal is a read-only database error' contains "$(cat "$FIXTURE_DIR/write.err")" 'readonly' +assert_eq 'failed write preserves all ten rows' 10 "$(run_local "$DB" <<<'SELECT count(*) FROM transmissions;')" + +# REAL DATA: the committed staging-captured fixture. Its most-attributed +# pubkey, counted by the script's query, must equal a direct count. +if [ -f "$REAL_FIXTURE" ]; then + cp "$REAL_FIXTURE" "$FIXTURE_DIR/real.db" + real_row=$(run_local "$FIXTURE_DIR/real.db" <<<"SELECT from_pubkey,count(*) FROM transmissions WHERE from_pubkey IS NOT NULL GROUP BY from_pubkey ORDER BY count(*) DESC,from_pubkey LIMIT 1;") + real_pk=${real_row%|*}; real_n=${real_row##*|} + assert_match "real fixture has an attributed pubkey" '^[0-9a-f]{64}$' "$real_pk" + assert_match "real fixture count > 0" '^[1-9][0-9]*$' "$real_n" + assert_eq "script query on real fixture" "$real_n" "$(count "$real_pk" "$FIXTURE_DIR/real.db")" +else + FAIL=$((FAIL + 1)); echo "FAIL: $REAL_FIXTURE missing" >&2 +fi + +# ERROR SURFACING: a broken query must be distinguishable from an empty +# result — non-zero exit and something on stderr, not a silent "" or "0". +for bad in "$LEGACY_DB:no such column: from_pubkey" "$EMPTY_DB:no such table: transmissions"; do + bad_db=${bad%%:*}; want=${bad#*:} + err_file="$FIXTURE_DIR/err" + out=$(count "$PK_A" "$bad_db" 2>"$err_file"); rc=$? + assert_true "$want → non-zero exit (got $rc)" test "$rc" -ne 0 + assert_true "$want → named on stderr" contains "$(cat "$err_file")" "$want" + assert_eq "$want → no count on stdout" "" "$out" +done + +# ----- runner selection and transport, with ssh stubbed --------------------- +# ssh_t is replaced by a stub that logs its argv, captures stdin, and runs the +# remote command string through a real `bash -c`, so the script's own quoting +# is what gets parsed. "docker exec -i NAME" is peeled off when the stub +# container is said to have sqlite3; otherwise it fails like the app image. +STUB_ARGV="$FIXTURE_DIR/ssh.argv"; STUB_STDIN="$FIXTURE_DIR/ssh.stdin" +STUB_CONTAINER_SQLITE=0 +STUB_PATH="$PATH" +ssh_t() { + local cmd="$1" in + printf '%s\n' "$*" >>"$STUB_ARGV" + in=$(cat); printf '%s\n' "$in" >>"$STUB_STDIN" + case "$cmd" in + "docker exec -i "*) + if [ "$STUB_CONTAINER_SQLITE" != 1 ]; then + echo 'exec: "sqlite3": executable file not found in $PATH' >&2; return 126 + fi + cmd=${cmd#docker exec -i }; cmd=${cmd#* } ;; + esac + printf '%s\n' "$in" | PATH="$STUB_PATH" bash -c "$cmd" +} +reset_stub() { : >"$STUB_ARGV"; : >"$STUB_STDIN"; } +TMP="$FIXTURE_DIR"; ADMIN_API_TOKEN=""; TARGET_CONTAINER="corescope-stub" +TARGET_SSH_HOST="stub-host"; TARGET_DB_PATH="$DB"; TEST_PUBKEY="$PK_A" + +# Container has no sqlite3 → host runner, correct count. +reset_stub +read_retain_count >/dev/null 2>&1; rc=$? +assert_eq "fallback: rc" "0" "$rc" +assert_eq "fallback: runner" "host" "$SQLITE_RUNNER" +assert_eq "fallback: count" "3" "$RETAIN_COUNT" +argv=$(cat "$STUB_ARGV"); stdin=$(cat "$STUB_STDIN") +assert_true "transport: container tried first" contains "$(head -1 "$STUB_ARGV")" "docker exec -i corescope-stub sqlite3" +assert_true "transport: pubkey not in remote argv" lacks "$argv" "$PK_A" +assert_true "transport: SQL not in remote argv" lacks "$argv" "SELECT" +assert_true "transport: column not in remote argv" lacks "$argv" "from_pubkey" +assert_true "transport: SQL arrives on stdin" contains "$stdin" "SELECT COUNT(*) FROM transmissions WHERE from_pubkey = lower(:pubkey);" +assert_true "transport: pubkey arrives hex-encoded on stdin" contains "$stdin" "$(sql_hex_literal "$PK_A")" +assert_true "transport: raw pubkey never on stdin" lacks "$stdin" "$PK_A" + +# Container has sqlite3 → container runner. +reset_stub; STUB_CONTAINER_SQLITE=1 +read_retain_count >/dev/null 2>&1; rc=$? +assert_eq "container: rc" "0" "$rc" +assert_eq "container: runner" "container" "$SQLITE_RUNNER" +assert_eq "container: count" "3" "$RETAIN_COUNT" +STUB_CONTAINER_SQLITE=0 + +# A sqlite3 that cannot bind (.parameter prints help to stdout, exit 0, +# parameter left NULL) must be rejected by the probe, not trusted. +FAKE_BIN="$FIXTURE_DIR/fakebin"; mkdir -p "$FAKE_BIN" +cat >"$FAKE_BIN/sqlite3" <<'FAKE' +#!/usr/bin/env bash +cat >/dev/null +echo ".parameter CMD ... Manage SQL parameter bindings" +echo "0" +FAKE +chmod +x "$FAKE_BIN/sqlite3" +reset_stub; STUB_PATH="$FAKE_BIN:$PATH" +read_retain_count >/dev/null 2>&1; rc=$? +assert_eq "unbindable sqlite3: rejected" "1" "$rc" +assert_eq "unbindable sqlite3: no runner" "" "$SQLITE_RUNNER" +assert_eq "unbindable sqlite3: no count" "" "$RETAIN_COUNT" +STUB_PATH="$PATH" + +# A matching token from a failed client is not a successful capability probe. +cat >"$FAKE_BIN/sqlite3" <<'FAKE' +#!/usr/bin/env bash +cat >/dev/null +echo 'corescope-probe-ok' +exit 1 +FAKE +reset_stub; STUB_PATH="$FAKE_BIN:$PATH" +read_retain_count >/dev/null 2>&1; rc=$? +assert_eq 'failed client with matching probe token is rejected' 1 "$rc" +assert_eq 'failed client leaves no runner' '' "$SQLITE_RUNNER" +assert_eq 'failed client leaves no count' '' "$RETAIN_COUNT" +STUB_PATH="$PATH" + +# A query error on the target is a failure with no count, never "0". +reset_stub; TARGET_DB_PATH="$LEGACY_DB" +read_retain_count >/dev/null 2>&1; rc=$? +assert_eq "remote schema error: rc" "1" "$rc" +assert_eq "remote schema error: count" "" "$RETAIN_COUNT" +assert_true "remote schema error: stderr kept" contains "$(cat "$TMP/sqlite.err")" "no such column: from_pubkey" +TARGET_DB_PATH="$DB" + +# run_sqlite with no resolved runner must refuse rather than guess. +SQLITE_RUNNER="" +if run_sqlite /dev/null 2>&1; then + FAIL=$((FAIL + 1)); echo "FAIL: run_sqlite with no runner — expected non-zero exit" >&2 +else + PASS=$((PASS + 1)) +fi + +out=$(CORESCOPE_TEST_BACKEND=unsupported bash "$SCRIPT_DIR/test-blacklist-sql.sh" 2>&1); rc=$? +assert_eq 'unknown test backend is refused' 2 "$rc" +assert_true 'unknown backend fails before a fixture or connection' contains "$out" 'unsupported CORESCOPE_TEST_BACKEND' +assert_eq 'committed historical fixture is unchanged' "$REAL_FIXTURE_SHA" "$(sha256sum "$REAL_FIXTURE")" + +echo "test-blacklist-sqlite.sh: $PASS passed, $FAIL failed" +[ "$FAIL" -eq 0 ] diff --git a/qa/scripts/test-postgres-connect.py b/qa/scripts/test-postgres-connect.py new file mode 100644 index 000000000..5c2919af0 --- /dev/null +++ b/qa/scripts/test-postgres-connect.py @@ -0,0 +1,136 @@ +import contextlib +import importlib.util +import io +import os +from pathlib import Path +import runpy +import subprocess +import sys +import unittest +from unittest import mock + +sys.dont_write_bytecode = True +PATH = Path(__file__).with_name("postgres-connect.py") +SPEC = importlib.util.spec_from_file_location("postgres_connect", PATH) +CONNECT = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(CONNECT) + + +class ConnectionTests(unittest.TestCase): + def test_url_decodes_credentials_into_environment_and_retains_tls(self): + env = CONNECT.connection_env({"CORESCOPE_READER_DATABASE_URL": + "postgresql://qa_reader:p%40ss%2Fword@[::1]:5444/measurements?sslmode=verify-full&sslrootcert=%2Fprivate%2Fca.pem", + "PGSERVICE": "unrelated", "PGOPTIONS": "-c role=owner"}) + self.assertEqual((env["PGHOST"], env["PGPORT"], env["PGUSER"], env["PGDATABASE"]), ("::1", "5444", "qa_reader", "measurements")) + self.assertEqual(env["PGPASSWORD"], "p@ss/word") + self.assertEqual((env["PGSSLMODE"], env["PGSSLROOTCERT"]), ("verify-full", "/private/ca.pem")) + self.assertNotIn("PGSERVICE", env) + self.assertNotIn("PGOPTIONS", env) + + def test_url_without_tls_override_preserves_private_tls_setting(self): + env = CONNECT.connection_env({"CORESCOPE_READER_DATABASE_URL": "postgres://reader@localhost/data", "PGSSLMODE": "verify-full"}) + self.assertEqual(env["PGSSLMODE"], "verify-full") + + def test_raw_query_semicolons_cannot_hide_tls_parameters(self): + for query in ("application_name=qa;sslmode=verify-full", + "application_name=qa&sslrootcert=ca;sslmode=verify-full"): + with self.subTest(query=query), self.assertRaises(ValueError): + CONNECT.connection_env({"CORESCOPE_READER_DATABASE_URL": "postgres://reader@localhost/data?" + query}) + + def test_encoded_semicolon_is_a_value_and_preserves_explicit_tls(self): + env = CONNECT.connection_env({"CORESCOPE_READER_DATABASE_URL": + "postgres://reader@localhost/data?application_name=qa%3Bsslmode%3Ddisable&sslmode=verify-full"}) + self.assertEqual(env["PGAPPNAME"], "qa;sslmode=disable") + self.assertEqual(env["PGSSLMODE"], "verify-full") + + def test_explicit_url_clears_inherited_host_address(self): + source = {"CORESCOPE_READER_DATABASE_URL": "postgres://reader@localhost:5444/data", + "PGHOST": "unrelated-profile", "PGHOSTADDR": "stale-address"} + env = CONNECT.connection_env(source) + self.assertEqual((env["PGHOST"], env["PGPORT"]), ("localhost", "5444")) + self.assertNotIn("PGHOSTADDR", env) + self.assertEqual(source["PGHOSTADDR"], "stale-address") + self.assertEqual(CONNECT.connection_env({"PGDATABASE": "data", "PGHOSTADDR": "private-address"})["PGHOSTADDR"], "private-address") + + def test_omitted_url_port_uses_environment_or_default(self): + for inherited, expected in ((None, "5432"), ("", "5432"), ("25432", "25432"), ("1", "1"), ("65535", "65535")): + source = {"CORESCOPE_READER_DATABASE_URL": "postgres://reader@localhost/data"} + if inherited is not None: + source["PGPORT"] = inherited + with self.subTest(inherited=inherited): + self.assertEqual(CONNECT.connection_env(source)["PGPORT"], expected) + + def test_explicit_url_port_overrides_even_invalid_inherited_port(self): + for port in (1, 5444, 65535): + env = CONNECT.connection_env({"CORESCOPE_READER_DATABASE_URL": f"postgres://reader@localhost:{port}/data", "PGPORT": "invalid-inherited"}) + self.assertEqual(env["PGPORT"], str(port)) + + def test_invalid_effective_url_port_is_rejected_safely(self): + for port in ("0", "65536", "-1", "invalid-secret", " 5432", "5432.0"): + for explicit in (False, True): + url = f"postgres://reader:secret-sentinel@localhost{':' + port if explicit else ''}/data" + with self.subTest(explicit=explicit, port=port): + with self.assertRaises(ValueError) as raised: + CONNECT.connection_env({"CORESCOPE_READER_DATABASE_URL": url, "PGPORT": "5432" if explicit else port}) + self.assertNotIn("secret", str(raised.exception)) + self.assertNotIn(port, str(raised.exception)) + + def test_search_path_escapes_option_delimiters(self): + env = CONNECT.connection_env({"CORESCOPE_READER_DATABASE_URL": + "postgres://reader@localhost/data?search_path=qa%2C%20%22space%20%5C%20-crole%3Downer%22"}) + self.assertEqual(env["PGOPTIONS"], '-c search_path=qa,\\ "space\\ \\\\\\ -crole=owner"') + + def test_native_private_settings_need_no_url(self): + self.assertEqual(CONNECT.connection_env({"PGSERVICE": "qa-reader", "PGPASSFILE": "/private/pass"})["PGPASSFILE"], "/private/pass") + self.assertEqual(CONNECT.connection_env({"PGDATABASE": "data", "PGUSER": "reader"})["PGDATABASE"], "data") + with self.assertRaises(ValueError): + CONNECT.connection_env({"PGDATABASE": "postgres://reader:secret@localhost/data"}) + + def test_invalid_urls_reject_without_exposing_values(self): + for url in ("postgres://u:secret@/db", "postgres://u:secret@a,b/db", "postgres://u:secret@localhost/", + "postgres://u:secret@localhost/db?unsupported=yes", "postgres://u:secret@localhost/db?sslmode=require&sslmode=disable", + "postgres://u:secret@localhost/db?sslmode=", "postgres://u:secret@localhost/db?sslmode=invalid", + "postgres://u:secret%xx@localhost/db", "postgres://u:secret@localhost/db\n", "postgres://u:secret@localhost:bad/db"): + with self.subTest(url_kind=url.split("?")[0].split(":")[-1]): + with self.assertRaises(ValueError) as raised: + CONNECT.connection_env({"CORESCOPE_READER_DATABASE_URL": url}) + self.assertNotIn("secret", str(raised.exception)) + self.assertNotIn(url, str(raised.exception)) + + def test_psql_argv_never_contains_credentials(self): + url = "postgres://reader:secret-sentinel@localhost/data?sslmode=verify-full" + with mock.patch.dict(os.environ, {"CORESCOPE_READER_DATABASE_URL": url}, clear=True), \ + mock.patch.object(subprocess, "run", return_value=mock.Mock(returncode=0)) as run: + with self.assertRaises(SystemExit) as exited: + runpy.run_path(str(PATH), run_name="__main__") + self.assertEqual(exited.exception.code, 0) + argv = run.call_args.args[0] + self.assertNotIn(url, argv) + self.assertNotIn("secret-sentinel", " ".join(argv)) + self.assertEqual(run.call_args.kwargs["env"]["PGPASSWORD"], "secret-sentinel") + + def test_malformed_url_error_is_safe_for_job_output(self): + stream = io.StringIO() + with mock.patch.dict(os.environ, {"CORESCOPE_READER_DATABASE_URL": "postgres://u:secret-sentinel@/db"}, clear=True), contextlib.redirect_stderr(stream): + with self.assertRaises(SystemExit): + runpy.run_path(str(PATH), run_name="__main__") + self.assertNotIn("secret-sentinel", stream.getvalue()) + self.assertIn("invalid PostgreSQL reader URL", stream.getvalue()) + + def test_libpq_errors_cannot_echo_connection_values(self): + stream = io.StringIO() + def fail(*args, **kwargs): + diagnostic = 'psql: invalid connection setting "secret-sentinel"\n' + if kwargs.get("stderr") != subprocess.PIPE: + print(diagnostic, file=sys.stderr) + return mock.Mock(returncode=2, stderr=diagnostic) + with mock.patch.dict(os.environ, {"PGDATABASE": "data"}, clear=True), \ + mock.patch.object(subprocess, "run", side_effect=fail), contextlib.redirect_stderr(stream): + with self.assertRaises(SystemExit): + runpy.run_path(str(PATH), run_name="__main__") + self.assertNotIn("secret-sentinel", stream.getvalue()) + self.assertIn("private connection settings", stream.getvalue()) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/bash-path.js b/scripts/bash-path.js new file mode 100644 index 000000000..cb1551afb --- /dev/null +++ b/scripts/bash-path.js @@ -0,0 +1,15 @@ +// Resolve the shell used by tests that exercise the real management/CI scripts. +const fs = require('node:fs'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); + +module.exports = function bashPath() { + if (process.env.BASH_PATH) return process.env.BASH_PATH; + if (process.platform === 'win32') { + // --exec-path is stable whether PATH finds Git in cmd/, bin/ or mingw64/bin/. + const git = spawnSync('git', ['--exec-path'], { encoding: 'utf8' }); + const candidate = path.resolve((git.stdout || '').trim(), '../../../bin/bash.exe'); + if (git.status === 0 && fs.existsSync(candidate)) return candidate; + } + return 'bash'; +}; diff --git a/scripts/check-dockerfile-internal-pkgs.sh b/scripts/check-dockerfile-internal-pkgs.sh index c23918932..0e38aed8f 100644 --- a/scripts/check-dockerfile-internal-pkgs.sh +++ b/scripts/check-dockerfile-internal-pkgs.sh @@ -2,7 +2,7 @@ # scripts/check-dockerfile-internal-pkgs.sh # # Asserts every internal/ referenced via a "replace" directive in -# cmd/server/go.mod or cmd/ingestor/go.mod has a matching +# each shipped command's go.mod has a matching # "COPY internal//" line in Dockerfile for each builder section that # needs it. # @@ -18,7 +18,7 @@ set -euo pipefail ROOT="$(cd "$(dirname "$0")/.." && pwd)" DOCKERFILE="$ROOT/Dockerfile" -MODS=("$ROOT/cmd/server/go.mod" "$ROOT/cmd/ingestor/go.mod") +MODS=("$ROOT/cmd/server/go.mod" "$ROOT/cmd/ingestor/go.mod" "$ROOT/cmd/decrypt/go.mod" "$ROOT/cmd/migrate/go.mod") ERRORS=0 # PKG_COUNT[pkg] = number of go.mod files that reference internal/. diff --git a/scripts/install-postgres-ci.sh b/scripts/install-postgres-ci.sh new file mode 100755 index 000000000..3af4b0ff1 --- /dev/null +++ b/scripts/install-postgres-ci.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Exact PostgreSQL client/server packages for disposable Ubuntu CI runners. +set -euo pipefail +mode=${1:-client} +[[ ${GITHUB_ACTIONS:-} == true ]] || { echo 'This installer is for disposable GitHub Actions runners.' >&2; exit 1; } +[[ $mode == client || $mode == server ]] || { echo 'Usage: install-postgres-ci.sh client|server' >&2; exit 1; } +sudo install -d /usr/share/postgresql-common/pgdg +curl --fail --show-error --silent https://www.postgresql.org/media/keys/ACCC4CF8.asc | + sudo tee /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc >/dev/null +. /etc/os-release +printf 'deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt-archive.postgresql.org/pub/repos/apt %s-pgdg-archive main\n' "$VERSION_CODENAME" | + sudo tee /etc/apt/sources.list.d/corescope-pgdg.list >/dev/null +sudo apt-get update -qq +client_version=$(apt-cache madison postgresql-client-18 | awk '$3 ~ /^18[.]6-/ && !found {print $3; found=1} END {if(!found) exit 1}') +packages=("postgresql-client-18=$client_version" sqlite3) +if [[ $mode == server ]]; then + # The benchmark owns its own cluster. Package installation must not launch + # an extra unmeasured PostgreSQL service on the runner. + sudo install -d /etc/postgresql-common + printf 'create_main_cluster = false\n' | sudo tee /etc/postgresql-common/createcluster.conf >/dev/null + server_version=$(apt-cache madison postgresql-18 | awk '$3 ~ /^18[.]6-/ && !found {print $3; found=1} END {if(!found) exit 1}') + packages+=("postgresql-18=$server_version") +fi +sudo apt-get install -y --no-install-recommends "${packages[@]}" +echo /usr/lib/postgresql/18/bin >> "$GITHUB_PATH" +/usr/lib/postgresql/18/bin/pg_dump --version | grep -E '18[.]6([ (]|$)' +if [[ $mode == server ]]; then /usr/lib/postgresql/18/bin/postgres --version | grep -E '18[.]6([ (]|$)'; fi diff --git a/scripts/migrate-dedup.js b/scripts/migrate-dedup.js index 8e2fd3a4f..cdbdd831c 100644 --- a/scripts/migrate-dedup.js +++ b/scripts/migrate-dedup.js @@ -1,20 +1,26 @@ #!/usr/bin/env node /** - * Milestone 1: Packet Dedup Schema Migration + * Offline legacy SQLite tooling. Not part of PostgreSQL upgrades. * * Creates `transmissions` and `observations` tables from the existing `packets` table. * Idempotent — drops and recreates new tables on each run. * Does NOT touch the original `packets` table. * - * Usage: node scripts/migrate-dedup.js + * Usage: node scripts/migrate-dedup.js --legacy-sqlite-offline */ +if (process.argv[2] !== '--legacy-sqlite-offline') { + console.error('Offline legacy SQLite tooling only. Use docs/postgresql-upgrade.md for current upgrades. To operate on a stopped, disposable recovery copy, pass --legacy-sqlite-offline explicitly.'); + process.exit(1); +} +process.argv.splice(2, 1); + const Database = require('better-sqlite3'); const path = require('path'); const dbPath = process.argv[2]; if (!dbPath) { - console.error('Usage: node scripts/migrate-dedup.js '); + console.error('Usage: node scripts/migrate-dedup.js --legacy-sqlite-offline '); process.exit(1); } diff --git a/scripts/migrate-fixture-hashes.go b/scripts/migrate-fixture-hashes.go index a0acded93..7c356ecf6 100644 --- a/scripts/migrate-fixture-hashes.go +++ b/scripts/migrate-fixture-hashes.go @@ -1,136 +1,109 @@ -// migrate-fixture-hashes recomputes content hashes in a fixture DB using the -// current ComputeContentHash formula. Run once; idempotent. +// migrate-fixture-hashes updates only an explicit disposable legacy SQLite fixture. +// Run from cmd/migrate before offline PostgreSQL import. It shares the runtime +// packet identity algorithm and refuses collisions instead of dropping records. package main import ( - "crypto/sha256" "database/sql" - "encoding/hex" "fmt" "log" + "net/url" "os" + "path/filepath" + "runtime" _ "github.com/mattn/go-sqlite3" + "github.com/meshcore-analyzer/packetpath" ) -func computeContentHash(rawHex string) string { - buf, err := hex.DecodeString(rawHex) - if err != nil || len(buf) < 2 { - if len(rawHex) >= 16 { - return rawHex[:16] - } - return rawHex - } - - headerByte := buf[0] - offset := 1 - routeType := int(headerByte & 0x03) - if routeType == 2 || routeType == 3 { // transport - offset += 4 +func main() { + if len(os.Args) != 2 { + log.Fatal("usage: migrate-fixture-hashes ") } - if offset >= len(buf) { - if len(rawHex) >= 16 { - return rawHex[:16] - } - return rawHex + if err := migrateFixtureHashes(os.Args[1]); err != nil { + log.Fatal(err) } - pathByte := buf[offset] - offset++ - hashSize := int((pathByte>>6)&0x3) + 1 - hashCount := int(pathByte & 0x3F) - pathBytes := hashSize * hashCount +} - payloadStart := offset + pathBytes - if payloadStart > len(buf) { - if len(rawHex) >= 16 { - return rawHex[:16] - } - return rawHex +func migrateFixtureHashes(path string) error { + info, err := os.Stat(path) + if err != nil { + return err } - - payload := buf[payloadStart:] - payloadType := (headerByte >> 2) & 0x0F - toHash := []byte{payloadType} - - // TRACE = payload type 7 - if int(payloadType) == 7 { - toHash = append(toHash, pathByte, 0x00) + if !info.Mode().IsRegular() { + return fmt.Errorf("fixture must be an existing regular file") } - toHash = append(toHash, payload...) - - h := sha256.Sum256(toHash) - return hex.EncodeToString(h[:])[:16] -} - -func main() { - if len(os.Args) < 2 { - fmt.Fprintf(os.Stderr, "usage: %s \n", os.Args[0]) - os.Exit(1) + absolute, err := filepath.Abs(path) + if err != nil { + return err } - dbPath := os.Args[1] - - db, err := sql.Open("sqlite3", dbPath) + absolute = filepath.ToSlash(absolute) + if runtime.GOOS == "windows" { + absolute = "/" + absolute + } + u := url.URL{Scheme: "file", Path: absolute} + q := url.Values{"mode": {"rw"}, "_foreign_keys": {"on"}, "_synchronous": {"FULL"}} + u.RawQuery = q.Encode() + db, err := sql.Open("sqlite3", u.String()) if err != nil { - log.Fatal(err) + return err } defer db.Close() - - rows, err := db.Query("SELECT id, raw_hex, hash FROM transmissions") + db.SetMaxOpenConns(1) + if _, err := db.Exec(`PRAGMA trusted_schema=OFF`); err != nil { + return err + } + rows, err := db.Query(`SELECT id,raw_hex,hash FROM transmissions ORDER BY id LIMIT 10001`) if err != nil { - log.Fatal(err) + return err } - type update struct { - id int - newHash string + id int64 + hash string } var updates []update - + seen := make(map[string]int64) + count := 0 for rows.Next() { - var id int - var rawHex, oldHash string - if err := rows.Scan(&id, &rawHex, &oldHash); err != nil { - log.Printf("scan: %v", err) - continue + var id int64 + var raw, old string + if err := rows.Scan(&id, &raw, &old); err != nil { + rows.Close() + return err + } + count++ + if count > 10000 { + rows.Close() + return fmt.Errorf("fixture exceeds 10000 transmissions; this is not a production repair tool") } - newHash := computeContentHash(rawHex) - if newHash != oldHash { - updates = append(updates, update{id, newHash}) + hash := packetpath.ContentHash(raw) + if previous, exists := seen[hash]; exists { + rows.Close() + return fmt.Errorf("content-hash collision between fixture IDs %d and %d requires explicit review; no rows changed", previous, id) + } + seen[hash] = id + if hash != old { + updates = append(updates, update{id, hash}) } } - rows.Close() - - if len(updates) == 0 { - fmt.Println("All hashes already match current formula.") - return + if err := rows.Err(); err != nil { + rows.Close() + return err } - + rows.Close() tx, err := db.Begin() if err != nil { - log.Fatal(err) - } - stmt, err := tx.Prepare("UPDATE transmissions SET hash = ? WHERE id = ?") - if err != nil { - log.Fatal(err) + return err } - merged := 0 + defer tx.Rollback() for _, u := range updates { - if _, err := stmt.Exec(u.newHash, u.id); err != nil { - // UNIQUE constraint = duplicate (same content, different old hash). - // Move observations to the surviving tx, then delete the dup. - log.Printf("update id %d: %v — merging duplicate", u.id, err) - // Find surviving tx id - var survID int - if err2 := tx.QueryRow("SELECT id FROM transmissions WHERE hash = ?", u.newHash).Scan(&survID); err2 == nil { - tx.Exec("UPDATE observations SET transmission_id = ? WHERE transmission_id = ?", survID, u.id) - tx.Exec("DELETE FROM transmissions WHERE id = ?", u.id) - merged++ - } + if _, err := tx.Exec(`UPDATE transmissions SET hash=? WHERE id=?`, u.hash, u.id); err != nil { + return fmt.Errorf("fixture hash update failed; no changes committed: %w", err) } } - stmt.Close() if err := tx.Commit(); err != nil { - log.Fatal(err) + return err } - fmt.Printf("Migrated %d hashes, merged %d duplicates.\n", len(updates)-merged, merged) + fmt.Printf("Updated %d fixture hashes; preserved %d transmissions and every observation.\n", len(updates), count) + return nil } diff --git a/scripts/non-unit-tests.json b/scripts/non-unit-tests.json index 4388d5967..78ae4acc5 100644 --- a/scripts/non-unit-tests.json +++ b/scripts/non-unit-tests.json @@ -113,6 +113,7 @@ "tests/e2e/test-packets-node-filter-e2e.js", "tests/e2e/test-packets-scope-column.js", "tests/e2e/test-path-inspector-coverage-e2e.js", + "tests/e2e/test-postgres-backup-e2e.js", "tests/e2e/test-rx-coverage-mobile-nav-e2e.js", "tests/e2e/test-rx-coverage-gaps-e2e.js", "tests/e2e/test-rx-coverage-noise-e2e.js", diff --git a/scripts/postgres-benchmark/.gitignore b/scripts/postgres-benchmark/.gitignore new file mode 100644 index 000000000..c18dd8d83 --- /dev/null +++ b/scripts/postgres-benchmark/.gitignore @@ -0,0 +1 @@ +__pycache__/ diff --git a/scripts/postgres-benchmark/README.md b/scripts/postgres-benchmark/README.md new file mode 100644 index 000000000..c155592e0 --- /dev/null +++ b/scripts/postgres-benchmark/README.md @@ -0,0 +1,267 @@ +# PostgreSQL comparison harness + +This harness measures the actual CoreScope application methods and server. It +does not replace them with standalone SQL insert loops. Production files are +archived at exact commits and remain unchanged; Go build overlays add only test +drivers. Fixture construction happens outside measured ingestion. + +## Status + +The generator, protocol vectors, fixture/handler controls and Linux S/B runs +have been exercised. The [performance report](../../docs/postgresql-performance.md) +identifies measured revisions, results and limitations. A completed run qualifies +only its exact candidate and workload; source changes require remeasurement. + +## Prerequisites + +- One disposable Linux host: at least four CPUs and 8 GiB RAM for `primary`. +- Go **1.27.2**, a C compiler, Git, Python 3.12+, and native PostgreSQL **18.6**. +- PostgreSQL tools default to `/usr/lib/postgresql/18/bin`; override with + `--postgres-bin`. The controller uses `postgres`, `initdb`, `psql`, and + `pg_isready`, and never contacts an existing PostgreSQL service. +- Run as a non-root user. PostgreSQL starts on a newly selected loopback port + with a private data directory and synthetic, local roles. +- Primary runs require a delegated cgroup v2 parent with CPU, memory and I/O + controllers enabled. Required resource counters are checked before builds. + The runner creates its own child and writes its limits + before any application/database child executes. All descendants inherit the + same **3 CPU / 6 GiB** application budget. The coordinator and event generator + stay outside that child. +- The coordinator must be inside a sibling of the measured child under the + delegated parent, or otherwise have permission to migrate its children at + their common cgroup ancestor. Chowning an unrelated destination alone is not + sufficient. `memory.reclaim` is required to remove inactive prior-backend + charges between serial runs; the script never clears the host page cache. +- Capacity guards are S:4 GiB, B:12 GiB, L:128 GiB free before generation. A + second guard uses eight times the *measured* canonical file size before each + restore/import. These are conservative guards, not dataset-size claims. + +## GitHub Actions opt-in + +The existing CI workflow can run the comparison for a pull request. Add this +checked item to the PR body **before pushing the reviewed candidate**: + +```markdown +- [x] Run PostgreSQL comparison +``` + +The checkbox is evaluated on the normal PR events: opened, reopened and head +updates. Editing the body alone does not launch or cancel CI. Without the checked +item, the benchmark is skipped. A documentation-only PR also skips it through +the change-scope gate. + +The comparison checks out the PR's exact head SHA from its head repository, +including forks. It does not benchmark GitHub's synthetic merge commit. The +executed workflow and benchmark harness must match that head; mismatches fail +before running the experiment. The job has read-only repository access, persists +no checkout credentials, inherits no deployment secrets and cannot publish or +deploy. Other PR checks continue normally. + +Wait for the comparison to finish, then **uncheck the item before pushing a +documentation-only results/report update**. Change scope covers the full PR diff, +so a PR that still contains implementation changes would otherwise request +another primary run. Existing PR concurrency can also cancel an unfinished run +when a new commit arrives. + +Manual dispatch remains available: use `deploy.yml` with +`postgres_benchmark=true` and the full `candidate_sha`, from the matching +candidate branch/commit. That dispatch runs the comparison alone and skips normal +publishing/deployment jobs. Both entry points run the small pilot first and then +five supplemental pairs; upload only the sanitized `public/` outputs. + +## Commands + +First run the portable controls and compile/protocol checks: + +```sh +python3 -m unittest discover -s scripts/postgres-benchmark -p 'test_*.py' +``` + +Pilot on the same Linux host intended for the comparison: + +```sh +python3 scripts/postgres-benchmark/run.py --repo . \ + --candidate-sha --corpus S --pairs 1 \ + --profile smoke --ingest-rate 50 --http-rate 20 \ + --output /tmp/corescope-bench-pilot +``` + +Smoke results are explicitly ineligible for a primary comparison. The current +supplemental comparison uses the five-pair `primary` execution mode: + +```sh +python3 scripts/postgres-benchmark/run.py --repo . \ + --candidate-sha --corpus B --pairs 5 \ + --ingest-rate 50 --http-rate 20 \ + --profile primary --cgroup "$CORESCOPE_BENCH_CGROUP" \ + --output /tmp/corescope-bench-supplemental-50eps +``` + +The baseline is fixed to `3e21b179ecccae005aa5f6f45bc095353c94b343`, including +upstream's store-lock and reach-query fixes (#2147, #2155). Earlier reports used `9dbc287`; their results +are historical and must not be pooled with this comparison. The candidate's +verified PostgreSQL import is explicitly adopted into its persistent selection +before either timed startup; selection setup is outside the startup measurement. +Floating +revisions and existing output directories are refused. Primary defaults to +60 seconds of warmup and 180 measured seconds per backend, with 50 offered +ingest events/second and 20 HTTP requests/second: 12,000 scheduled ingest events +per backend. Five alternating A/B pairs +therefore spend 40 minutes in load windows, plus preparation, imports, +validation, startup and drain time. The longer design profile is explicit: +`--warmup 120 --seconds 600`. Record and compare identical locked settings; +changing rates after a pilot produces a separately identified experiment. + +The initial 100-event/s B qualification at candidate `7b2c19a` +([Actions run 37897747657](https://github.com/Kpa-clawbot/CoreScope/actions/runs/37897747657)) +failed its SQLite leg with 82 dropped schedules. All drops fell +at 205.92–206.76 seconds, after retention was scheduled at 195 seconds; that pass took +11.7446 seconds and ingestion queue wait reached 11.054 seconds. The bounded +1,024-event queue was full. Durable effects matched the 23,918 completed events, +but the missing events keep that run failed. Its artifacts remain separate. +The supplemental 50-event/s rate gives the same queue 20.48 seconds of offered +work, versus 10.24 seconds at 100/s. This gives the observed maintenance burst +headroom; every comparison must still prove zero drops, including during warmup. +The manifest records the actual offered rate. Do not pool the two experiments. + +Both engines use a 2,048 MiB packet-store allowance and a 3,072 MiB server Go +memory limit. The ingestor remains at 512 MiB and the whole application/database +cgroup remains capped at 3 CPUs / 6 GiB. These are common settings for full +startup, hot startup and replay; the manifest records them. + +The earlier 1,024 MiB store / 1,536 MiB server profile failed B capacity +qualification: the unchanged baseline loaded all 112,000 retained transmissions +and 1,792,000 observations, then its own memory accounting triggered a 25% +eviction. The corrected corpus accounts for about 1,222 MiB before replay. +A conservative check using the baseline's actual estimators bounds all 24,000 +events from the original 100-event/s profile at another 190 MiB, even counting +every event as a new transmission plus observation with padded field widths. +That bound also covers the current 12,000-event profile. The common store +allowance leaves more than 630 MiB above that bound. This is capacity evidence, +not a performance result; measurements from the earlier profile must not be +pooled with this one. Corpus sizes, durability and count gates stay fixed. + +## Data and correctness + +The immutable baseline creates the complete canonical SQLite schema and seeds +all 18 telemetry tables. S has 30,000 transmissions / 90,000 observations; B has +128,000 / 2,048,000; L has 1,000,000 / 16,000,000. S cycles through fanouts 1–5; +B/L use 50%×4, 20%×22, 20%×23 and 10%×50. Inputs include missing observers, +nullable and fractional signal values, integral scores, resolved/unresolved paths, and +one-, two-, and three-byte hop hashes. S/B have seven retained days and one +expired day. The current day includes a real hot-startup window. + +Every generated observation also has a unique normalized observer/path key +within its transmission, matching the unchanged server's in-memory identity. +NULL observer positions and fanout counts stay fixed. If short path prefixes +collide for two NULL observations, the generator deterministically tries the next +valid node/path variant, up to 64 attempts, preserving the wire payload/hash and +route/hop/hash-size shape. Exhaustion fails generation. This avoids accidental +duplicate rows that SQLite's nullable unique index permits but the loader merges. +Compiled generator controls, including the complete repeating B/L path key space, +run during both builds before corpus generation. The full retained-count gate +continues to compare every eligible row. + +Scores in the timed corpus and replay are integral values supported by both +revisions. The unchanged SQLite server silently drops fractional-score rows; +including those would compare different loaded datasets after the PostgreSQL +loader correction. Separate native PostgreSQL regressions exercise fractional, +large integral and null scores through every loader and the existing HTTP +response. The benchmark never patches the baseline application to fix that bug. + +Adverts have deterministic Ed25519 identities and unique synthetic advert +labels; channel frames use the repository's existing AES/HMAC test helper. +Both are checked through the real decoder. Protocol references are MeshCore +`a366955cb2f67b8e6842d4f00d2b6a554dddd88a`: `src/Packet.cpp`, +`src/Mesh.cpp::createAdvert`, `src/helpers/AdvertDataHelpers.h`, and +`docs/packet_format.md`. No real packet bodies, identities, or locations are +fetched. The generated coordinates, names and messages are fictional. + +Each pair receives one recorded receipt-time offset. All persisted receipt and +bookkeeping timestamps shift together, preserving fractional precision. Signed +wire payloads and emission timestamps stay unchanged, so their identities +remain stable. Both backends use exactly that same pair copy and pre-generated +event file. The event file contains 50% new transmissions, 30% distinct later +observations, 15% exact duplicates and 5% late observations. Replayed existing +transmissions are selected from retained history so retention does not +introduce nondeterministic delete/recreate races. + +The shipping importer loads PostgreSQL. Before timing, every table's ordered, +typed logical digest and row count must match the pair's SQLite copy; the source +must remain unchanged. After replay and retention, counts, write errors, +relationships, and full logical parity are checked again. Only documented +operational receipt/creation times are excluded from the final digest; packet +timestamps, raw frames, IDs, scores and nulls remain part of it. A separate +200-envelope `handleMessage` control follows parity and verifies real durable +effects, rather than trusting the void callback return. + +## Measurements + +- Real server launch to first HTTP, ready header/health, and full background + startup completion, with `hotStartupHours` 0 and 1. Loaded transmission and + observation counts must agree across each pair under identical memory limits. + For S and B, both counts must also equal the complete retained corpus computed + independently from SQLite; two equally incomplete loads cannot pass. L records + bounded memory coverage for optional capacity qualification. +- Startup uses new application processes after complete data validation, so the + OS cache is warm. Mixed requests have a separate application warmup. The + manifest records these actual per-phase policies; there is no cache-state + switch or unsupported OS-cold label. +- Real `InsertTransmission` latency and durable effects; a bounded 1024-event + queue reports every dropped schedule and drain time. +- The fixed weighted API mix and production caches: nodes 20%, channels/messages + 20%, observers 10%, metrics 10%, reach/RX 10%, packets/details 15%, analytics + 10%, stats 5%. Nonempty data and requested page sizes are checked. Individual + HTTP cache hits are **unobserved**, not inferred from a fake cache-bust flag. +- Separate real channel-message SQL misses (using the existing cache-clear seam) + and observer-metric SQL queries. These are not confused with warm HTTP hits. +- Real WebSocket receipts correlated to committed new-transmission hashes. + Negative receipt-minus-return lags are retained and counted: the baseline can + expose a transmission before all its auto-committed writes finish. Multiple + hash receipts may be legitimate observation updates, and are not mislabeled + as duplicate packet bugs. +- The real 250-transmission retention path during replay, followed by an empty + retention pass. A packet-detail request for expired history becomes an + explicitly separate expected-404 class after retention starts. +- Whole application/database cgroup CPU, memory and I/O; per-process RSS/PSS, + database/WAL storage, and remaining filesystem headroom. RSS is not used as a + substitute for shared-memory-aware PSS/cgroup memory. + The optional kernel `memory.peak` counter is reset and read through one held + descriptor per backend leg, including preparation and migration. It is not a + measured-window-only peak; use windowed `memory.current` samples for that + comparison. Kernels without reset support leave this counter unmeasured. +- Actual compiled retention and advert-preservation query plans in a separate + diagnostic phase. PostgreSQL uses `EXPLAIN (ANALYZE, BUFFERS, SETTINGS, FORMAT + JSON)`; SQLite uses `EXPLAIN QUERY PLAN`. Plan collection is outside headlines. + +The runner never weakens WAL durability, adds a cache service, or changes the +poll interval. SQLite WAL/FULL/FK and PostgreSQL fsync/synchronous-commit/full-page +writes are checked on real connections. Actual toolchain versions, binary +hashes, schemas, run order, and production-tree hashes before and after the run +are retained. The unit of variation is a paired run, not millions of requests +treated as independent experiments. Per-class p99 requires at least 1,000 +successful samples; unavailable values remain null, never zero. + +## Artifacts and limits + +Upload **only `/public/`**. It is an explicit sanitized whitelist and is +also produced on controller failure. It contains manifests, typed validation +digests, per-run JSONL/CSV samples, plans, summaries and the report. Do not upload +`.runtime`, control files, binaries, databases, raw event payload streams or +unfiltered build logs. Completed temporary per-run databases are removed only +after verification; failure preserves them. The original canonical corpus stays +available locally for investigation. + +`summary.csv` contains absolute per-run values. `paired-summary.json` contains +paired ratios, sample/pair counts and observed spread. Raw data remains the +authority. Missing prerequisites, wrong source hashes, unsafe destinations, +unverified cgroup limits, empty results, errors, loss or parity mismatches fail +the run. A timeout, error or partial artifact must never be relabeled zero work. + +The controller does not claim verified OS-cold results, a 65-minute maintenance +soak, size-target 5.5/13-GiB qualification, or an optional million-mobile-RX +extension. L is capacity qualification only when actually run. Preparation, +receipt-time freshening and protocol generation stay outside timed ingestion. +Migration, startup, diagnostics, and steady-state profiles remain separate in +the resource samples. Results measure the conversion as shipped, including its +transaction/batching/index changes; they cannot isolate the database engine alone. diff --git a/scripts/postgres-benchmark/overlays/clock_linux_test.go b/scripts/postgres-benchmark/overlays/clock_linux_test.go new file mode 100644 index 000000000..0a75fd57c --- /dev/null +++ b/scripts/postgres-benchmark/overlays/clock_linux_test.go @@ -0,0 +1,18 @@ +//go:build linux + +package main + +import ( + "syscall" + "unsafe" +) + +// CLOCK_MONOTONIC is shared across processes on this one Linux host. +func benchMono() int64 { + var ts syscall.Timespec + _, _, e := syscall.RawSyscall(syscall.SYS_CLOCK_GETTIME, 1, uintptr(unsafe.Pointer(&ts)), 0) + if e != 0 { + panic(e) + } + return ts.Sec*1e9 + ts.Nsec +} diff --git a/scripts/postgres-benchmark/overlays/clock_other_test.go b/scripts/postgres-benchmark/overlays/clock_other_test.go new file mode 100644 index 000000000..8ebc3054f --- /dev/null +++ b/scripts/postgres-benchmark/overlays/clock_other_test.go @@ -0,0 +1,6 @@ +//go:build !linux + +package main + +// Portable protocol-only tests compile here; measured runs are Linux-only. +func benchMono() int64 { panic("measurement requires Linux CLOCK_MONOTONIC") } diff --git a/scripts/postgres-benchmark/overlays/common_test.go b/scripts/postgres-benchmark/overlays/common_test.go new file mode 100644 index 000000000..de2d9a8e9 --- /dev/null +++ b/scripts/postgres-benchmark/overlays/common_test.go @@ -0,0 +1,122 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +type benchShape struct { + Transmissions int `json:"transmissions"` + Observations int `json:"observations"` + Nodes int `json:"nodes"` + Observers int `json:"observers"` + Days int `json:"days"` +} +type benchConfig struct { + EventsFile string `json:"events_file"` + CorpusInfo string `json:"corpus_info"` + Mode string `json:"mode"` + Corpus string `json:"corpus"` + Shape benchShape `json:"shape"` + Seed int64 `json:"seed"` + WireEpoch int64 `json:"wire_epoch"` + Epoch int64 `json:"epoch"` + SQLite string `json:"sqlite"` + Output string `json:"output"` + StateDir string `json:"state_dir"` + BaseURL string `json:"base_url"` + StartFile string `json:"start_file"` + Warmup int `json:"warmup"` + Seconds int `json:"seconds"` + IngestRate int `json:"ingest_rate"` + HTTPRate int `json:"http_rate"` +} + +func benchRead(t *testing.T) benchConfig { + t.Helper() + path := os.Getenv("CORESCOPE_BENCH_CONFIG") + if path == "" { + t.Skip("opt-in benchmark overlay") + } + b, e := os.ReadFile(path) + if e != nil { + t.Fatal(e) + } + var c benchConfig + if e = json.Unmarshal(b, &c); e != nil { + t.Fatal(e) + } + if !strings.Contains(filepath.Clean(c.Output), "corescope-bench-") { + t.Fatal("output is not an allowlisted disposable benchmark directory") + } + if c.Output == "" || c.Shape.Transmissions <= 0 || c.Shape.Nodes <= 0 || c.Shape.Observers < 1 { + t.Fatal("incomplete benchmark configuration") + } + return c +} +func benchJSON(t *testing.T, path string, value any) { + t.Helper() + b, e := json.MarshalIndent(value, "", " ") + if e != nil { + t.Fatal(e) + } + if e = os.WriteFile(path, append(b, '\n'), 0600); e != nil { + t.Fatal(e) + } +} +func benchOutput(t *testing.T, c benchConfig, name string) *os.File { + t.Helper() + f, e := os.OpenFile(filepath.Join(c.Output, name), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600) + if e != nil { + t.Fatal(e) + } + t.Cleanup(func() { f.Close() }) + return f +} +func benchStart(c benchConfig) (int64, error) { + for i := 0; i < 2400; i++ { + if b, e := os.ReadFile(c.StartFile); e == nil { + var start struct { + Mono int64 `json:"mono_ns"` + } + if e = json.Unmarshal(b, &start); e != nil { + return 0, e + } + return start.Mono, nil + } + time.Sleep(50 * time.Millisecond) + } + return 0, fmt.Errorf("benchmark start barrier timed out") +} +func benchSleep(until int64) { + if delay := until - benchMono(); delay > 0 { + time.Sleep(time.Duration(delay)) + } +} +func benchFanout(c benchConfig, i int) int { + if c.Corpus == "S" { + return i%5 + 1 + } + switch { + case i%10 < 5: + return 4 + case i%10 < 7: + return 22 + case i%10 < 9: + return 23 + default: + return 50 + } +} +func benchStamp(c benchConfig, i int) int64 { + day := (i / 10) % c.Shape.Days + if day == 0 { + return c.Epoch - 2700 + int64(i%1800) + } + return c.Epoch - int64(day)*86400 - 43200 + int64(i%3600) +} diff --git a/scripts/postgres-benchmark/overlays/ingestor_test.go b/scripts/postgres-benchmark/overlays/ingestor_test.go new file mode 100644 index 000000000..9e92eb6b1 --- /dev/null +++ b/scripts/postgres-benchmark/overlays/ingestor_test.go @@ -0,0 +1,621 @@ +package main + +import ( + "bytes" + "crypto/ed25519" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "encoding/json" + "fmt" + "math" + "os" + "path/filepath" + "runtime" + "strings" + "sync" + "testing" + "time" +) + +// Wire format is pinned to MeshCore a366955: Packet.cpp and Mesh.cpp::createAdvert; +// group encryption reuses this revision's firmware-derived test helper. +type benchFixture struct { + c benchConfig + public []string + private []ed25519.PrivateKey + channels map[string]string +} + +func benchFixtureFor(c benchConfig) *benchFixture { + f := &benchFixture{c: c, channels: map[string]string{}} + for i := 0; i < c.Shape.Nodes; i++ { + seed := sha256.Sum256([]byte(fmt.Sprintf("corescope-bench/%d/node/%d", c.Seed, i))) + key := ed25519.NewKeyFromSeed(seed[:]) + f.private = append(f.private, key) + f.public = append(f.public, hex.EncodeToString(key.Public().(ed25519.PublicKey))) + } + for i := 0; i < 20; i++ { + name := fmt.Sprintf("#bench-%02d", i) + f.channels[name] = deriveHashtagChannelKey(name) + } + return f +} +func (f *benchFixture) packet(index, variant int, stamp int64) (*PacketData, error) { + node := index % len(f.public) + payloadType := 5 + var payload []byte + if index%10 == 0 { + payloadType = 4 + public := f.private[node].Public().(ed25519.PublicKey) + app := []byte{0x92} + location := make([]byte, 8) + binary.LittleEndian.PutUint32(location, uint32(20_000_000+node*100)) + binary.LittleEndian.PutUint32(location[4:], uint32(30_000_000+node*100)) + app = append(app, location...) + app = append(app, []byte(fmt.Sprintf("Bench%04d-%07d", node, index))...) + signed := append([]byte{}, public...) + clock := make([]byte, 4) + binary.LittleEndian.PutUint32(clock, uint32(stamp)) + signed = append(signed, clock...) + signed = append(signed, app...) + payload = append(payload, public...) + payload = append(payload, clock...) + payload = append(payload, ed25519.Sign(f.private[node], signed)...) + payload = append(payload, app...) + } else { + channelIndex := 0 + if index%10 >= 5 { + channelIndex = 1 + index%19 + } + channel := fmt.Sprintf("#bench-%02d", channelIndex) + key := f.channels[channel] + inner := make([]byte, 5) + binary.LittleEndian.PutUint32(inner, uint32(stamp)) + inner = append(inner, []byte(fmt.Sprintf("Bench%04d: synthetic sequence %d seed %d %s", node, index, f.c.Seed, strings.Repeat("x", index%40)))...) + cipher, mac := buildChannelEncrypted(key, inner) + keyBytes, _ := hex.DecodeString(key) + sum := sha256.Sum256(keyBytes) + macBytes, _ := hex.DecodeString(mac) + cipherBytes, _ := hex.DecodeString(cipher) + payload = append([]byte{sum[0]}, macBytes...) + payload = append(payload, cipherBytes...) + } + hashSize := 1 + index%3 + hopCount := 1 + index%3 + route := 1 + if index%5 == 0 && variant == 0 { + hopCount = 0 + } + if payloadType == 4 && variant == 1 { + route = 2 + hopCount = 0 + } + var path []byte + for hop := 0; hop < hopCount; hop++ { + key, _ := hex.DecodeString(f.public[(index+variant*7+hop+1)%len(f.public)]) + path = append(path, key[:hashSize]...) + } + if variant >= 1000 { + hashSize = 3 + hopCount = 2 + route = 1 + path = []byte{0xfa, byte(variant >> 8), byte(variant), 0xfb, byte(index >> 8), byte(index)} + } + wire := append([]byte{byte(payloadType<<2 | route), byte((hashSize-1)<<6 | hopCount)}, path...) + wire = append(wire, payload...) + raw := hex.EncodeToString(wire) + decoded, err := DecodePacket(raw, f.channels, true) + if err != nil { + return nil, err + } + score, snr, rssi := float64(index%5), float64(index%20)-10.5, -100.5+float64(index%25) + msg := &MQTTPacketMessage{Raw: raw, SNR: &snr, RSSI: &rssi, Score: &score, Timestamp: time.Unix(stamp, 0).UTC().Format(time.RFC3339)} + p := BuildPacketData(msg, decoded, f.public[0], "AAA", nil) + p.Timestamp = msg.Timestamp + if p.Hash == "" || p.PayloadType != payloadType { + return nil, fmt.Errorf("generated frame failed native decoder") + } + return p, nil +} + +type benchGeneratedObservation struct { + Packet *PacketData + ObserverIndex any + PathVariant int +} + +func (f *benchFixture) observation(index, ordinal int, stamp int64, used map[string]bool) (benchGeneratedObservation, error) { + observerIndex := ordinal%f.c.Shape.Observers + 1 + result := benchGeneratedObservation{ObserverIndex: observerIndex} + observerID := f.public[observerIndex-1] + if ordinal%17 == 16 { + result.ObserverIndex, observerID = nil, "" + } + // Keep every observation and NULL identity. A path-prefix collision for + // the same identity needs a different valid wire path, not different JSON + // spelling. The bound stays below packet()'s special replay variants. + for attempt := 0; attempt < 64; attempt++ { + result.PathVariant = ordinal + attempt + packet, err := f.packet(index, result.PathVariant, stamp) + if err != nil { + return result, err + } + key := observerID + "|" + packet.PathJSON + if !used[key] { + used[key] = true + result.Packet = packet + return result, nil + } + } + return result, fmt.Errorf("no unique observer/path identity after 64 valid path variants") +} + +func TestCoreScopeBenchmarkObservationKeyspace(t *testing.T) { + for _, shape := range []struct { + name string + nodes, observers int + }{{"B", 2000, 128}, {"L", 5000, 512}} { + t.Run(shape.name, func(t *testing.T) { + c := benchConfig{Corpus: shape.name, Seed: 20261008, Epoch: 1791451200, Shape: benchShape{Nodes: shape.nodes, Observers: shape.observers, Days: 8}} + f := benchFixtureFor(c) + // The path recipe depends on index modulo Nodes and modulo 3; + // route/fanout periods 5/10 divide both node counts. Thus these + // 6,000/15,000 positions cover the complete B/L path key space. + corrections, missing := 0, 0 + for index := 0; index < 3*shape.nodes; index++ { + used := make(map[string]bool) + count := benchFanout(c, index) + if count > shape.observers { + t.Fatal("known observer identities would repeat within a transmission") + } + // Known observer IDs are distinct. Only NULL identities can + // share a key, so exercise every such pair with real decoding. + for ordinal := 16; ordinal < count; ordinal += 17 { + original, err := f.packet(index, ordinal, c.Epoch) + if err != nil { + t.Fatal(err) + } + got, err := f.observation(index, ordinal, c.Epoch, used) + if err != nil { + t.Fatal(err) + } + if got.ObserverIndex != nil || got.Packet.Hash != original.Hash || got.Packet.RawHex[:4] != original.RawHex[:4] { + t.Fatal("path correction changed observer, payload identity, route or hop/hash shape") + } + if got.PathVariant != ordinal { + corrections++ + if got.Packet.PathJSON == original.PathJSON { + t.Fatal("retry did not change the real decoded path") + } + } + missing++ + } + } + if corrections == 0 || missing != 3*shape.nodes*6/10 { + t.Fatalf("NULL collision coverage: corrections=%d NULLs=%d", corrections, missing) + } + }) + } +} + +func TestCoreScopeBenchmarkObservationRetryBound(t *testing.T) { + c := benchConfig{Corpus: "B", Seed: 20261008, Epoch: 1791451200, Shape: benchShape{Nodes: 2000, Observers: 128}} + f := benchFixtureFor(c) + used := make(map[string]bool) + for prefix := 0; prefix < 256; prefix++ { + used[fmt.Sprintf(`|["%02X"]`, prefix)] = true + } + if _, err := f.observation(9, 16, c.Epoch, used); err == nil { + t.Fatal("exhausted path space must fail rather than drop an observation or loop forever") + } +} + +func TestCoreScopeBenchmarkProtocol(t *testing.T) { + c := benchConfig{Seed: 20261008, Epoch: 1791451200, Shape: benchShape{Nodes: 80, Observers: 8, Transmissions: 100, Days: 8}} + f := benchFixtureFor(c) + a, _ := f.packet(0, 0, c.Epoch) + repeated, _ := f.packet(18000, 0, c.Epoch) + if a.Hash == repeated.Hash { + t.Fatal("repeated advertiser and emission time collapsed distinct corpus transmissions") + } + seen := map[string]bool{} + for i := 0; i < 100; i++ { + a, e := f.packet(i, 0, c.Epoch-int64(i)) + if e != nil { + t.Fatal(e) + } + b, e := f.packet(i, 1000, c.Epoch-int64(i)) + if e != nil { + t.Fatal(e) + } + if a.Hash != b.Hash { + t.Fatal("path change altered content identity") + } + if seen[a.Hash] { + t.Fatal("generator accidentally deduplicated distinct transmissions") + } + if a.Score == nil || *a.Score != math.Trunc(*a.Score) { + t.Fatal("timed score corpus must stay integral so the unmodified baseline loads the same rows") + } + seen[a.Hash] = true + if i%10 != 0 && !strings.HasPrefix(a.ChannelHash, "#bench-") { + t.Fatal("generated channel message did not decrypt") + } + } + for _, name := range []string{"S", "B", "L"} { + c.Corpus = name + total := 0 + for i := 0; i < 100; i++ { + total += benchFanout(c, i) + } + want := 1600 + if name == "S" { + want = 300 + } + if total != want { + t.Fatalf("fanout %s=%d", name, total) + } + } +} + +func TestCoreScopeBenchmarkEventStream(t *testing.T) { + dir := t.TempDir() + c := benchConfig{Corpus: "S", Seed: 20261008, Epoch: 1791454800, WireEpoch: 1791451200, Shape: benchShape{Nodes: 200, Observers: 16, Transmissions: 100, Observations: 300, Days: 2}, Output: dir, EventsFile: filepath.Join(dir, "one.jsonl"), Seconds: 4, IngestRate: 10} + benchMakeEvents(t, c) + first, e := os.ReadFile(c.EventsFile) + if e != nil { + t.Fatal(e) + } + c.EventsFile = filepath.Join(dir, "two.jsonl") + benchMakeEvents(t, c) + second, e := os.ReadFile(c.EventsFile) + if e != nil { + t.Fatal(e) + } + if !bytes.Equal(first, second) { + t.Fatal("same seed/epochs produced different event streams") + } + dec := json.NewDecoder(bytes.NewReader(first)) + counts := map[string]int{} + var additional *PacketData + for i := 0; i < 40; i++ { + var input benchInput + if e := dec.Decode(&input); e != nil { + t.Fatal(e) + } + if input.Sequence != i { + t.Fatal("sequence drift") + } + counts[input.Class]++ + if i == 10 { + additional = input.Packet + } + if i == 16 && (input.Packet.Hash != additional.Hash || input.Packet.PathJSON != additional.PathJSON) { + t.Fatal("duplicate event does not hit the same observation key") + } + } + for class, want := range map[string]int{"new": 20, "additional": 12, "duplicate": 6, "late": 2} { + if counts[class] != want { + t.Fatalf("event class %s=%d want %d", class, counts[class], want) + } + } +} + +func TestCoreScopeBenchmark(t *testing.T) { + c := benchRead(t) + if runtime.GOOS != "linux" { + t.Fatal("measurement requires Linux") + } + if c.Mode == "prepare" { + if err := benchPrepare(c); err != nil { + t.Fatal(err) + } + return + } + if c.Mode == "events" { + benchMakeEvents(t, c) + return + } + s, err := benchOpen(c) + if err != nil { + t.Fatal(err) + } + defer s.Close() + s.WaitForAsyncMigrations() + settings, err := benchSettings(s) + if err != nil { + t.Fatal(err) + } + benchJSON(t, filepath.Join(c.Output, "database-settings.json"), settings) + if err = s.RefreshPrefixIndex(); err != nil { + t.Fatal(err) + } + switch c.Mode { + case "replay": + benchReplay(t, c, s) + case "retention": + benchRetention(t, c, s) + case "handler": + benchHandler(t, c, s, benchFixtureFor(c)) + case "plans": + if e := os.MkdirAll(filepath.Join(c.Output, "plans"), 0700); e != nil { + t.Fatal(e) + } + for _, query := range []struct { + name, sql string + args []any + }{ + {"retention-empty", pruneAgedTransmissionIDs, []any{time.Now().UTC().AddDate(0, 0, -7).Format(time.RFC3339), pruneBatchTransmissions}}, + {"advert-preservation", legacyAdvertObservationSQL, []any{1, 1, "[]"}}, + } { + plan, e := benchExplain(s, query.sql, query.args...) + if e != nil { + t.Fatal(e) + } + benchJSON(t, filepath.Join(c.Output, "plans", query.name+".json"), map[string]any{"sql": query.sql, "args": query.args, "plan": plan, "context": "separate diagnostic after replay and retention; excluded from headline timings"}) + } + default: + t.Fatal("unsupported benchmark mode") + } +} + +type benchEvent struct { + Sequence int `json:"sequence"` + Class string `json:"class"` + Hash string `json:"hash"` + Scheduled int64 `json:"scheduled_ns"` + Started int64 `json:"started_ns"` + Completed int64 `json:"completed_ns"` + Measured bool `json:"measured"` + Dropped bool `json:"dropped"` + New bool `json:"new_transmission"` + Error string `json:"error,omitempty"` +} + +type benchInput struct { + Sequence int `json:"sequence"` + Class string `json:"class"` + Packet *PacketData `json:"packet"` +} + +func benchMakeEvents(t *testing.T, c benchConfig) { + f := benchFixtureFor(c) + file, e := os.OpenFile(c.EventsFile, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600) + if e != nil { + t.Fatal(e) + } + defer file.Close() + enc := json.NewEncoder(file) + for sequence := 0; sequence < (c.Warmup+c.Seconds)*c.IngestRate; sequence++ { + class, index, variant := "new", c.Shape.Transmissions+sequence, 0 + part, block := sequence%20, sequence/20 + stamp := c.Epoch + int64(sequence/c.IngestRate) + if part >= 10 { + stride := c.Shape.Days * 10 + blocks := c.Shape.Transmissions / stride + if blocks < 1 { + blocks = 1 + } + index = (block%blocks)*stride + block%10 + wire := c + if c.WireEpoch != 0 { + wire.Epoch = c.WireEpoch + } + stamp = benchStamp(wire, index) + class = "additional" + variant = 1000 + part + if part >= 16 && part < 19 { + class = "duplicate" + variant = 1010 + } + if part == 19 { + class = "late" + variant = 1019 + } + } + packet, e := f.packet(index, variant, stamp) + if e != nil { + t.Fatal(e) + } + packet.ObserverID = f.public[0] + packet.Timestamp = time.Unix(c.Epoch+int64(sequence/c.IngestRate), 0).UTC().Format(time.RFC3339) + if class == "late" { + packet.Timestamp = time.Unix(benchStamp(c, index)-60, 0).UTC().Format(time.RFC3339) + } + if e := enc.Encode(benchInput{sequence, class, packet}); e != nil { + t.Fatal(e) + } + } +} + +func benchReplay(t *testing.T, c benchConfig, s *Store) { + var initialTx, initialObs int64 + if err := s.db.QueryRow(`SELECT count(*) FROM transmissions`).Scan(&initialTx); err != nil { + t.Fatal(err) + } + if err := s.db.QueryRow(`SELECT count(*) FROM observations`).Scan(&initialObs); err != nil { + t.Fatal(err) + } + file := benchOutput(t, c, "ingest.jsonl") + encoder := json.NewEncoder(file) + var outputMu sync.Mutex + emit := func(e benchEvent) { + outputMu.Lock() + defer outputMu.Unlock() + if err := encoder.Encode(e); err != nil { + t.Error(err) + } + } + if err := os.WriteFile(filepath.Join(c.Output, "replay-ready"), []byte("ready"), 0600); err != nil { + t.Fatal(err) + } + start, err := benchStart(c) + if err != nil { + t.Fatal(err) + } + benchSleep(start) + queue := make(chan benchInput, 1024) + done := make(chan struct{}) + var successful, newTx, newObs, dropped int64 + seenReplay := make(map[string]bool) + var firstError error + go func() { + defer close(done) + for input := range queue { + sequence, class, p := input.Sequence, input.Class, input.Packet + var e error + event := benchEvent{Sequence: sequence, Class: class, Scheduled: start + int64(sequence)*int64(time.Second)/int64(c.IngestRate), Measured: sequence >= c.Warmup*c.IngestRate} + if p != nil { + event.Hash = p.Hash + event.Started = benchMono() + beforeErrors := s.Stats.WriteErrors.Load() + event.New, e = s.InsertTransmission(p) + if e == nil && s.Stats.WriteErrors.Load() != beforeErrors { + e = fmt.Errorf("write-error counter advanced despite handler return") + } + event.Completed = benchMono() + if e == nil && event.New != (class == "new") { + e = fmt.Errorf("event class %s had unexpected transmission creation", class) + } + } + if e != nil { + event.Error = e.Error() + if firstError == nil { + firstError = e + } + } else { + successful++ + if event.New { + newTx++ + } + key := p.Hash + "/" + p.ObserverID + "/" + p.PathJSON + if !seenReplay[key] { + newObs++ + seenReplay[key] = true + } + } + emit(event) + } + }() + retained := make(chan benchPruneResult, 1) + go func() { + benchSleep(start + int64(c.Warmup+c.Seconds*3/4)*int64(time.Second)) + _ = os.WriteFile(filepath.Join(c.Output, "retention-started"), []byte("started"), 0600) + retained <- benchPrune(s) + }() + events := (c.Warmup + c.Seconds) * c.IngestRate + inputFile, err := os.Open(c.EventsFile) + if err != nil { + close(queue) + <-done + t.Fatal(err) + } + defer inputFile.Close() + inputDecoder := json.NewDecoder(inputFile) + var producerError error + for sequence := 0; sequence < events; sequence++ { + var input benchInput + if err := inputDecoder.Decode(&input); err != nil { + producerError = err + break + } + if input.Sequence != sequence || input.Packet == nil { + producerError = fmt.Errorf("invalid pre-generated event sequence") + break + } + scheduled := start + int64(sequence)*int64(time.Second)/int64(c.IngestRate) + benchSleep(scheduled) + select { + case queue <- input: + default: + dropped++ + emit(benchEvent{Sequence: sequence, Class: "schedule", Scheduled: scheduled, Dropped: true, Measured: sequence >= c.Warmup*c.IngestRate}) + } + } + close(queue) + select { + case <-done: + case <-time.After(120 * time.Second): + t.Fatal("bounded ingestion queue did not drain in 120 seconds") + } + retention := <-retained + benchJSON(t, filepath.Join(c.Output, "retention.json"), retention) + if retention.Error != "" { + t.Fatal(retention.Error) + } + var actualTx, actualObs int64 + if err := s.db.QueryRow(`SELECT count(*) FROM transmissions`).Scan(&actualTx); err != nil { + t.Fatal(err) + } + if err := s.db.QueryRow(`SELECT count(*) FROM observations`).Scan(&actualObs); err != nil { + t.Fatal(err) + } + verified := dropped == 0 && successful == int64(events) && producerError == nil && firstError == nil && s.Stats.WriteErrors.Load() == 0 && actualTx == initialTx+newTx-retention.Transmissions && actualObs == initialObs+newObs-retention.Observations + benchJSON(t, filepath.Join(c.Output, "ingest-validation.json"), map[string]any{"verified": verified, "scheduled": events, "completed": successful, "dropped_schedules": dropped, "expected_transmissions": initialTx + newTx - retention.Transmissions, "actual_transmissions": actualTx, "expected_observations": initialObs + newObs - retention.Observations, "actual_observations": actualObs, "write_errors": s.Stats.WriteErrors.Load(), "writer_wait_hold": s.WriterStatsSnapshot(), "finished_ns": benchMono()}) + if !verified { + t.Fatalf("durable ingest effects did not match: dropped=%d completed=%d/%d tx=%d/%d obs=%d/%d first error=%v", dropped, successful, events, actualTx, initialTx+newTx-retention.Transmissions, actualObs, initialObs+newObs-retention.Observations, firstError) + } +} + +type benchPruneResult struct { + Elapsed int64 `json:"elapsed_ns"` + Transmissions int64 `json:"transmissions_deleted"` + Observations int64 `json:"observations_deleted"` + Error string `json:"error,omitempty"` +} + +func benchPrune(s *Store) benchPruneResult { + result := benchPruneResult{} + cutoff := time.Now().UTC().AddDate(0, 0, -7).Format(time.RFC3339) + q := fmt.Sprintf("SELECT count(*) FROM observations o JOIN transmissions t ON t.id=o.transmission_id WHERE t.first_seen < '%s'", cutoff) + if err := s.db.QueryRow(q).Scan(&result.Observations); err != nil { + result.Error = err.Error() + return result + } + started := benchMono() + n, err := s.PruneOldPackets(7) + result.Elapsed = benchMono() - started + result.Transmissions = n + if err != nil { + result.Error = err.Error() + } + return result +} +func benchRetention(t *testing.T, c benchConfig, s *Store) { + r := benchPrune(s) + benchJSON(t, filepath.Join(c.Output, "retention-empty.json"), r) + if r.Error != "" || r.Transmissions != 0 || r.Observations != 0 { + t.Fatal("empty retention pass found unexpected work or failed") + } +} + +func benchHandler(t *testing.T, c benchConfig, s *Store, f *benchFixture) { + var before, after int64 + if err := s.db.QueryRow(`SELECT count(*) FROM transmissions`).Scan(&before); err != nil { + t.Fatal(err) + } + samples := make([]int64, 0, 200) + for i := 0; i < 200; i++ { + p, err := f.packet(c.Shape.Transmissions+10000000+i, 0, c.Epoch+int64(i)) + if err != nil { + t.Fatal(err) + } + payload, err := json.Marshal(map[string]any{"raw": p.RawHex, "origin": "Synthetic bench", "SNR": 6.5, "RSSI": -95.5}) + if err != nil { + t.Fatal(err) + } + msg := &mockMessage{topic: "meshcore/AAA/" + f.public[0] + "/packets", payload: payload} + started := benchMono() + handleMessage(s, "bench", MQTTSource{Name: "bench"}, msg, f.channels, nil, &Config{}) + samples = append(samples, benchMono()-started) + } + if err := s.db.QueryRow(`SELECT count(*) FROM transmissions`).Scan(&after); err != nil { + t.Fatal(err) + } + valid := after-before == 200 && s.Stats.WriteErrors.Load() == 0 + benchJSON(t, filepath.Join(c.Output, "handler.json"), map[string]any{"verified": valid, "samples_ns": samples, "new_transmissions": after - before, "classification": "separate 200-envelope mechanism control; not mixed-load throughput"}) + if !valid { + t.Fatal("real MQTT handler dropped valid synthetic envelopes") + } +} diff --git a/scripts/postgres-benchmark/overlays/postgres_adapter_test.go b/scripts/postgres-benchmark/overlays/postgres_adapter_test.go new file mode 100644 index 000000000..6d8870393 --- /dev/null +++ b/scripts/postgres-benchmark/overlays/postgres_adapter_test.go @@ -0,0 +1,48 @@ +package main + +import ( + "encoding/json" + "fmt" + "net/url" + "os" + "strings" +) + +func benchExplain(s *Store, query string, args ...any) (any, error) { + var raw []byte + if e := s.db.QueryRow("EXPLAIN (ANALYZE,BUFFERS,SETTINGS,FORMAT JSON) "+query, args...).Scan(&raw); e != nil { + return nil, e + } + if !json.Valid(raw) { + return nil, fmt.Errorf("invalid PostgreSQL plan JSON") + } + return json.RawMessage(raw), nil +} + +func benchOpen(c benchConfig) (*Store, error) { + dsn := os.Getenv("CORESCOPE_WRITER_DATABASE_URL") + u, e := url.Parse(dsn) + if e != nil || u.Hostname() != "127.0.0.1" || !strings.HasPrefix(strings.TrimPrefix(u.Path, "/"), "bench_") { + return nil, fmt.Errorf("refusing non-disposable PostgreSQL benchmark target") + } + return OpenStoreWithState(dsn, c.StateDir, 300) +} +func benchPrepare(c benchConfig) error { + return fmt.Errorf("canonical corpus must be generated by the unmodified SQLite baseline") +} +func benchSettings(s *Store) (map[string]string, error) { + out := map[string]string{} + for _, name := range []string{"server_version", "fsync", "synchronous_commit", "full_page_writes", "shared_buffers", "work_mem", "maintenance_work_mem", "effective_cache_size", "max_connections", "jit", "max_parallel_workers_per_gather", "checkpoint_timeout", "max_wal_size", "autovacuum"} { + var v string + if e := s.db.QueryRow("SHOW " + name).Scan(&v); e != nil { + return nil, e + } + out[name] = v + } + for _, key := range []string{"fsync", "synchronous_commit", "full_page_writes"} { + if out[key] != "on" { + return nil, fmt.Errorf("PostgreSQL %s is not durable", key) + } + } + return out, nil +} diff --git a/scripts/postgres-benchmark/overlays/server_test.go b/scripts/postgres-benchmark/overlays/server_test.go new file mode 100644 index 000000000..c415cd5b3 --- /dev/null +++ b/scripts/postgres-benchmark/overlays/server_test.go @@ -0,0 +1,355 @@ +package main + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path/filepath" + "runtime" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/gorilla/websocket" +) + +type benchCorpusInfo struct { + Node string `json:"node"` + Dense string `json:"dense_observer"` + Sparse string `json:"sparse_observer"` + NewPacket string `json:"new_packet"` + OldPacket string `json:"old_packet"` +} + +func benchInfo(c benchConfig) (benchCorpusInfo, error) { + var info benchCorpusInfo + b, e := os.ReadFile(c.CorpusInfo) + if e == nil { + e = json.Unmarshal(b, &info) + } + return info, e +} + +func TestCoreScopeBenchmark(t *testing.T) { + c := benchRead(t) + if runtime.GOOS != "linux" { + t.Fatal("measurement requires Linux") + } + switch c.Mode { + case "queries": + benchQueries(t, c) + case "client": + benchClient(t, c) + default: + t.Fatal("unsupported server benchmark mode") + } +} + +type benchRequest struct { + Class string `json:"class"` + Scheduled int64 `json:"scheduled_ns"` + Dispatched int64 `json:"dispatched_ns"` + Completed int64 `json:"completed_ns"` + Status int `json:"status"` + ExpectedStatus int `json:"expected_status"` + Bytes int `json:"bytes"` + SHA256 string `json:"response_sha256,omitempty"` + Error string `json:"error,omitempty"` + Measured bool `json:"measured"` + Cache string `json:"cache"` +} + +func benchQueries(t *testing.T, c benchConfig) { + target := os.Getenv("CORESCOPE_READER_DATABASE_URL") + if target == "" { + target = c.SQLite + } + db, e := OpenDB(target) + if e != nil { + t.Fatal(e) + } + defer db.Close() + info, e := benchInfo(c) + if e != nil { + t.Fatal(e) + } + out := json.NewEncoder(benchOutput(t, c, "sql-queries.jsonl")) + for i := 0; i < 200; i++ { + sample := benchRequest{Measured: true, Status: 200, Cache: "explicit SQL miss"} + var value any + if i%2 == 0 { + db.msgCacheMu.Lock() + db.msgCache = nil + db.msgCacheMu.Unlock() + sample.Class = "channel_messages_sql" + sample.Dispatched = benchMono() + rows, total, err := db.GetChannelMessages("#bench-00", 50, (i%4)*500) + sample.Completed = benchMono() + e = err + value = rows + if e == nil && (len(rows) == 0 || total < 50) { + e = fmt.Errorf("channel SQL query returned no meaningful data") + } + } else { + sample.Class = "observer_metrics_sql" + sample.Dispatched = benchMono() + rows, _, err := db.GetObserverMetrics(info.Dense, time.Unix(c.Epoch-86400, 0).UTC().Format(time.RFC3339), time.Unix(c.Epoch, 0).UTC().Format(time.RFC3339), "5m", 300) + sample.Completed = benchMono() + e = err + value = rows + if e == nil && len(rows) == 0 { + e = fmt.Errorf("metrics SQL query returned no meaningful data") + } + } + if e != nil { + sample.Error = e.Error() + } + b, _ := json.Marshal(value) + sum := sha256.Sum256(b) + sample.SHA256 = hex.EncodeToString(sum[:]) + sample.Bytes = len(b) + if e := out.Encode(sample); e != nil { + t.Fatal(e) + } + if sample.Error != "" { + t.Fatal(sample.Error) + } + } +} + +type benchEndpoint struct { + Class string `json:"class"` + Path string `json:"path"` +} + +func benchEndpoints(c benchConfig, info benchCorpusInfo) []benchEndpoint { + since24 := url.QueryEscape(time.Unix(c.Epoch-86400, 0).UTC().Format(time.RFC3339)) + since7 := url.QueryEscape(time.Unix(c.Epoch-7*86400, 0).UTC().Format(time.RFC3339)) + until := url.QueryEscape(time.Unix(c.Epoch, 0).UTC().Format(time.RFC3339)) + groups := []struct { + weight int + items []benchEndpoint + }{ + {20, []benchEndpoint{{"nodes_50", "/api/nodes?limit=50"}, {"nodes_2000", "/api/nodes?limit=2000"}, {"nodes_region", "/api/nodes?limit=50®ion=AAA&search=Synthetic"}}}, + {20, []benchEndpoint{{"channels", "/api/channels"}, {"messages_0", "/api/channels/%23bench-00/messages?limit=50&offset=0"}, {"messages_1000", "/api/channels/%23bench-00/messages?limit=50&offset=1000"}}}, + {10, []benchEndpoint{{"observers", "/api/observers"}}}, + {10, []benchEndpoint{{"metrics_dense_24h", "/api/observers/" + info.Dense + "/metrics?since=" + since24 + "&until=" + until + "&resolution=5m"}, {"metrics_sparse_7d", "/api/observers/" + info.Sparse + "/metrics?since=" + since7 + "&until=" + until + "&resolution=1h"}}}, + {10, []benchEndpoint{{"reach", "/api/nodes/" + info.Node + "/reach"}, {"rx_coverage", "/api/nodes/" + info.Node + "/rx-coverage?bbox=19,29,22,32&z=14"}}}, + {15, []benchEndpoint{{"packets_memory", "/api/packets?limit=50"}, {"packet_detail_memory", "/api/packets/" + info.NewPacket}, {"packet_detail_sql", "/api/packets/" + info.OldPacket}}}, + {10, []benchEndpoint{{"analytics_rf", "/api/analytics/rf"}, {"analytics_topology", "/api/analytics/topology"}, {"analytics_channels", "/api/analytics/channels"}, {"analytics_rf_filtered", "/api/analytics/rf?region=AAA"}}}, + {5, []benchEndpoint{{"stats", "/api/stats"}}}, + } + var result []benchEndpoint + for _, group := range groups { + for i := 0; i < group.weight; i++ { + result = append(result, group.items[i%len(group.items)]) + } + } + return result +} + +// Ignore timestamp/version metadata when deciding whether a response carries +// useful data. A successful empty response must not masquerade as fast SQL. +func benchNonempty(value any) bool { + switch v := value.(type) { + case []any: + return len(v) > 0 + case map[string]any: + if _, bad := v["error"]; bad { + return false + } + if hash, ok := v["hash"].(string); ok && hash != "" { + return true + } + for key, child := range v { + switch key { + case "total", "totalNodes", "nodeCount", "packetCount", "totalPackets", "totalTransmissions", "totalMessages", "count", "mobile_receptions", "totalLoaded": + if n, ok := child.(float64); ok && n > 0 { + return true + } + } + switch child.(type) { + case []any, map[string]any: + if benchNonempty(child) { + return true + } + } + } + } + return false +} + +func benchArrayCount(v any) int { + if rows, ok := v.([]any); ok { + return len(rows) + } + if obj, ok := v.(map[string]any); ok { + for _, key := range []string{"nodes", "packets", "messages", "metrics", "samples", "observers", "features", "data"} { + if child, found := obj[key]; found { + if n := benchArrayCount(child); n > 0 { + return n + } + } + } + } + return 0 +} + +func benchClient(t *testing.T, c benchConfig) { + info, e := benchInfo(c) + if e != nil { + t.Fatal(e) + } + endpoints := benchEndpoints(c, info) + benchJSON(t, filepath.Join(c.Output, "http-workload.json"), endpoints) + requests := json.NewEncoder(benchOutput(t, c, "requests.jsonl")) + messages := json.NewEncoder(benchOutput(t, c, "websocket.jsonl")) + conn, _, e := websocket.DefaultDialer.Dial("ws"+strings.TrimPrefix(c.BaseURL, "http")+"/ws", nil) + if e != nil { + t.Fatal(e) + } + defer conn.Close() + conn.SetReadLimit(8 << 20) + var closing atomic.Bool + var wsErr atomic.Pointer[string] + wsDone := make(chan struct{}) + go func() { + defer close(wsDone) + for { + _, raw, err := conn.ReadMessage() + if err != nil { + if !closing.Load() { + value := err.Error() + wsErr.Store(&value) + } + return + } + var frame struct { + Type string `json:"type"` + Data struct { + Hash string `json:"hash"` + ID int64 `json:"id"` + } `json:"data"` + } + if err = json.Unmarshal(raw, &frame); err != nil { + value := err.Error() + wsErr.Store(&value) + return + } + if frame.Type == "packet" && frame.Data.Hash != "" { + if err := messages.Encode(map[string]any{"hash": frame.Data.Hash, "id": frame.Data.ID, "received_ns": benchMono(), "bytes": len(raw)}); err != nil { + value := err.Error() + wsErr.Store(&value) + return + } + } + } + }() + if e := os.WriteFile(filepath.Join(c.Output, "client-ready"), []byte("ready"), 0600); e != nil { + t.Fatal(e) + } + start, e := benchStart(c) + if e != nil { + t.Fatal(e) + } + client := &http.Client{Timeout: 15 * time.Second} + defer client.CloseIdleConnections() + limit := make(chan struct{}, 64) + var wg sync.WaitGroup + var mu sync.Mutex + var measuredErrors int64 + emit := func(sample benchRequest) { + mu.Lock() + defer mu.Unlock() + if sample.Measured && sample.Error != "" { + measuredErrors++ + } + if e := requests.Encode(sample); e != nil { + t.Error(e) + } + } + for i := 0; i < (c.Warmup+c.Seconds)*c.HTTPRate; i++ { + endpoint := endpoints[i%len(endpoints)] + sample := benchRequest{Class: endpoint.Class, Scheduled: start + int64(i)*int64(time.Second)/int64(c.HTTPRate), Measured: i >= c.Warmup*c.HTTPRate, Cache: "production cache; individual hit/miss unobserved"} + benchSleep(sample.Scheduled) + select { + case limit <- struct{}{}: + default: + sample.Error = "load generator concurrency cap reached" + sample.Completed = benchMono() + emit(sample) + continue + } + wg.Add(1) + go func(endpoint benchEndpoint, sample benchRequest) { + defer wg.Done() + defer func() { <-limit }() + sample.Dispatched = benchMono() + response, err := client.Get(c.BaseURL + endpoint.Path) + if err != nil { + sample.Error = err.Error() + } else { + sample.Status = response.StatusCode + raw, readErr := io.ReadAll(io.LimitReader(response.Body, (8<<20)+1)) + response.Body.Close() + sample.Bytes = len(raw) + sum := sha256.Sum256(raw) + sample.SHA256 = hex.EncodeToString(sum[:]) + var value any + switch { + case readErr != nil: + sample.Error = readErr.Error() + case len(raw) > 8<<20: + sample.Error = "response exceeded bounded client limit" + case sample.Status == 404 && endpoint.Class == "packet_detail_sql": + if _, err := os.Stat(filepath.Join(c.Output, "retention-started")); err != nil { + sample.Error = "history packet missing before retention" + } else { + sample.Class = "packet_detail_pruned" + sample.ExpectedStatus = 404 + } + case sample.Status != 200: + sample.Error = fmt.Sprintf("HTTP %d", sample.Status) + case json.Unmarshal(raw, &value) != nil: + sample.Error = "invalid JSON response" + case (endpoint.Class == "nodes_50" || endpoint.Class == "messages_0" || endpoint.Class == "messages_1000" || endpoint.Class == "packets_memory") && benchArrayCount(value) != 50: + sample.Error = "bounded page did not return the expected 50 real records" + case endpoint.Class == "nodes_2000" && benchArrayCount(value) != 2000: + sample.Error = "node directory did not return the expected 2000 records" + case !benchNonempty(value): + sample.Error = "empty or invalid benchmark result" + } + } + sample.Completed = benchMono() + emit(sample) + }(endpoint, sample) + } + wg.Wait() + // The writer may still be draining its bounded queue. The completion marker + // is authoritative; allow the unchanged poller to catch up afterward. + deadline := time.Now().Add(150 * time.Second) + for time.Now().Before(deadline) { + if _, e := os.Stat(filepath.Join(c.Output, "ingest-validation.json")); e == nil { + break + } + time.Sleep(100 * time.Millisecond) + } + time.Sleep(10 * time.Second) + closing.Store(true) + conn.Close() + <-wsDone + benchJSON(t, filepath.Join(c.Output, "client-validation.json"), map[string]any{"verified": measuredErrors == 0 && wsErr.Load() == nil, "measured_errors": measuredErrors, "websocket_error": wsErr.Load(), "finished_ns": benchMono()}) + if measuredErrors > 0 { + t.Fatalf("%d measured HTTP requests failed validation", measuredErrors) + } + if e := wsErr.Load(); e != nil { + t.Fatal(*e) + } +} diff --git a/scripts/postgres-benchmark/overlays/sqlite_adapter_test.go b/scripts/postgres-benchmark/overlays/sqlite_adapter_test.go new file mode 100644 index 000000000..e84cede6a --- /dev/null +++ b/scripts/postgres-benchmark/overlays/sqlite_adapter_test.go @@ -0,0 +1,337 @@ +package main + +import ( + "database/sql" + "encoding/json" + "fmt" + "os" + "path/filepath" + "testing" + "time" + + "github.com/meshcore-analyzer/packetpath" +) + +func TestCoreScopeBenchmarkSmallCorpus(t *testing.T) { + dir := t.TempDir() + c := benchConfig{Corpus: "S", Seed: 20261008, Epoch: 1791451200, SQLite: filepath.Join(dir, "corpus.sqlite"), Output: dir, Shape: benchShape{Transmissions: 100, Observations: 300, Nodes: 200, Observers: 16, Days: 2}} + if e := benchPrepare(c); e != nil { + t.Fatal(e) + } + db, e := sql.Open("sqlite3", c.SQLite+"?mode=ro") + if e != nil { + t.Fatal(e) + } + defer db.Close() + for table, want := range map[string]int{"transmissions": 100, "observations": 300, "nodes": 200, "observers": 16} { + var n int + if e := db.QueryRow("SELECT count(*) FROM " + table).Scan(&n); e != nil { + t.Fatal(e) + } + if n != want { + t.Fatalf("%s=%d want %d", table, n, want) + } + } + var invalid int + if e := db.QueryRow(`SELECT count(*) FROM observations o LEFT JOIN transmissions t ON t.id=o.transmission_id WHERE t.id IS NULL`).Scan(&invalid); e != nil { + t.Fatal(e) + } + if invalid != 0 { + t.Fatal("generator created orphans") + } + if e := db.QueryRow(`SELECT count(*) FROM observations WHERE score IS NOT NULL AND typeof(score) != 'integer'`).Scan(&invalid); e != nil { + t.Fatal(e) + } + if invalid != 0 { + t.Fatalf("%d fractional scores would make the unchanged baseline silently discard corpus rows", invalid) + } + // The handler returns void, so prove its durable effects rather than + // treating delivery to the callback as a successful ingestion. + db.Close() + store, e := OpenStore(c.SQLite) + if e != nil { + t.Fatal(e) + } + defer store.Close() + store.WaitForAsyncMigrations() + f := benchFixtureFor(c) + packet, e := f.packet(1_000_001, 0, c.Epoch) + if e != nil { + t.Fatal(e) + } + body, _ := json.Marshal(map[string]any{"raw": packet.RawHex, "origin": "Synthetic bench", "SNR": 6.5, "RSSI": -95.5}) + handleMessage(store, "bench", MQTTSource{Name: "bench"}, &mockMessage{topic: "meshcore/AAA/" + f.public[0] + "/packets", payload: body}, f.channels, nil, &Config{}) + var inserted int + if e := store.db.QueryRow(`SELECT count(*) FROM transmissions WHERE hash=?`, packet.Hash).Scan(&inserted); e != nil { + t.Fatal(e) + } + if inserted != 1 || store.Stats.WriteErrors.Load() != 0 { + t.Fatal("handler callback did not produce a durable synthetic transmission") + } +} + +func TestCoreScopeBenchmarkNullableObservationIdentity(t *testing.T) { + dir := t.TempDir() + // This includes the first collision in the real B key space without + // requiring a two-million-row fixture for the regression itself. + c := benchConfig{Corpus: "B", Seed: 20261008, Epoch: 1791451200, SQLite: filepath.Join(dir, "corpus.sqlite"), Output: dir, + Shape: benchShape{Transmissions: 610, Observations: 9760, Nodes: 2000, Observers: 128, Days: 8}} + if err := benchPrepare(c); err != nil { + t.Fatal(err) + } + db, err := sql.Open("sqlite3", c.SQLite+"?mode=ro") + if err != nil { + t.Fatal(err) + } + defer db.Close() + var rows, missing, duplicateKeys int + if err := db.QueryRow(`SELECT count(*),sum(observer_idx IS NULL) FROM observations`).Scan(&rows, &missing); err != nil { + t.Fatal(err) + } + if rows != 9760 || missing != 366 { + t.Fatalf("generator changed fanout/NULL coverage: observations=%d, NULL observers=%d", rows, missing) + } + if err := db.QueryRow(`SELECT count(*) FROM ( + SELECT o.transmission_id,coalesce(obs.id,''),o.path_json FROM observations o + LEFT JOIN observers obs ON obs.rowid=o.observer_idx + GROUP BY o.transmission_id,coalesce(obs.id,''),o.path_json HAVING count(*)>1 + )`).Scan(&duplicateKeys); err != nil { + t.Fatal(err) + } + if duplicateKeys != 0 { + t.Fatalf("%d generated observer/path keys would be dropped by the unchanged upstream loader", duplicateKeys) + } +} + +func benchOpen(c benchConfig) (*Store, error) { return OpenStore(c.SQLite) } +func benchSettings(s *Store) (map[string]string, error) { + out := map[string]string{} + for _, name := range []string{"journal_mode", "synchronous", "foreign_keys", "busy_timeout", "cache_size", "page_size", "mmap_size", "cache_spill", "journal_size_limit", "wal_autocheckpoint"} { + var v string + if e := s.db.QueryRow("PRAGMA " + name).Scan(&v); e != nil { + return nil, e + } + out[name] = v + } + var version string + if e := s.db.QueryRow(`SELECT sqlite_version()`).Scan(&version); e != nil { + return nil, e + } + out["sqlite_version"] = version + if out["journal_mode"] != "wal" || out["synchronous"] != "2" || out["foreign_keys"] != "1" { + return nil, fmt.Errorf("SQLite durability differs from WAL/FULL/FK contract") + } + return out, nil +} + +func benchExplain(s *Store, query string, args ...any) (any, error) { + rows, e := s.db.Query("EXPLAIN QUERY PLAN "+query, args...) + if e != nil { + return nil, e + } + defer rows.Close() + type step struct { + ID int `json:"id"` + Parent int `json:"parent"` + Detail string `json:"detail"` + } + var result []step + for rows.Next() { + var item step + var unused int + if e := rows.Scan(&item.ID, &item.Parent, &unused, &item.Detail); e != nil { + return nil, e + } + result = append(result, item) + } + return result, rows.Err() +} + +func benchPrepare(c benchConfig) error { + if _, e := os.Stat(c.SQLite); !os.IsNotExist(e) { + return fmt.Errorf("corpus path must be new") + } + s, e := OpenStore(c.SQLite) + if e != nil { + return e + } + defer s.Close() + s.WaitForAsyncMigrations() + f := benchFixtureFor(c) + tx, e := s.db.Begin() + if e != nil { + return e + } + defer tx.Rollback() + stamp := time.Unix(c.Epoch, 0).UTC().Format(time.RFC3339) + exec := func(q string, args ...any) error { _, e := tx.Exec(q, args...); return e } + for i, pk := range f.public { + role := []string{"repeater", "companion", "room", "sensor"}[i%4] + if e := exec(`INSERT INTO nodes(public_key,name,role,lat,lon,first_seen,last_seen,advert_count) VALUES(?,?,?,?,?,?,?,0)`, pk, fmt.Sprintf("Synthetic %04d", i), role, 20+float64(i%2000)/10000, 30+float64(i%2000)/10000, stamp, stamp); e != nil { + return e + } + } + for i := 0; i < c.Shape.Observers; i++ { + if e := exec(`INSERT INTO observers(rowid,id,name,iata,last_seen,first_seen,packet_count,noise_floor,can_relay,can_relay_seen) VALUES(?,?,?,?,?,?,0,-110.5,1,1)`, i+1, f.public[i], fmt.Sprintf("Synthetic observer %d", i), []string{"AAA", "BBB", "CCC", "DDD"}[i%4], stamp, stamp); e != nil { + return e + } + } + transmission, e := tx.Prepare(`INSERT INTO transmissions(id,raw_hex,hash,first_seen,route_type,payload_type,payload_version,decoded_json,channel_hash,from_pubkey,last_seen,created_at) VALUES(?,?,?,?,?,?,?,?,?,?,?,?)`) + if e != nil { + return e + } + defer transmission.Close() + observation, e := tx.Prepare(`INSERT INTO observations(transmission_id,observer_idx,direction,snr,rssi,score,path_json,timestamp,raw_hex,resolved_path) VALUES(?,?,?,?,?,?,?,?,?,?)`) + if e != nil { + return e + } + defer observation.Close() + evidence, e := tx.Prepare(`INSERT INTO advert_route_evidence(tx_id,bit) VALUES(?,?)`) + if e != nil { + return e + } + defer evidence.Close() + var observationCount int64 + countsByDay := make([]int, c.Shape.Days) + var newest, oldest string + for i := 0; i < c.Shape.Transmissions; i++ { + at := benchStamp(c, i) + p, e := f.packet(i, 0, at) + if e != nil { + return e + } + count := benchFanout(c, i) + if _, e = transmission.Exec(i+1, p.RawHex, p.Hash, p.Timestamp, p.RouteType, p.PayloadType, p.PayloadVersion, p.DecodedJSON, p.ChannelHash, nilIfEmpty(p.FromPubkey), at+int64(count-1), stamp); e != nil { + return e + } + bits := packetpath.AdvertRouteEvidence(p.RawHex) + observationKeys := make(map[string]bool, count) + for o := 0; o < count; o++ { + generated, e := f.observation(i, o, at, observationKeys) + if e != nil { + return e + } + seen, observer := generated.Packet, generated.ObserverIndex + var snr any = float64((i+o)%20) - 10.5 + if o%11 == 10 { + snr = nil + } + var resolved any + if i%3 != 0 && seen.PathJSON != "[]" { + b, _ := json.Marshal([]string{f.public[(i+generated.PathVariant*7+1)%len(f.public)]}) + resolved = string(b) + } + if _, e = observation.Exec(i+1, observer, "rx", snr, -100.5+float64(o%25), p.Score, seen.PathJSON, at+int64(o), seen.RawHex, resolved); e != nil { + return e + } + observationCount++ + bits |= packetpath.AdvertRouteEvidence(seen.RawHex) + } + if p.PayloadType == 4 { + for _, bit := range []uint8{1, 2} { + if bits&bit != 0 { + if _, e = evidence.Exec(i+1, bit); e != nil { + return e + } + } + } + } + countsByDay[(i/10)%c.Shape.Days]++ + if i == 0 { + newest = p.Hash + } + if i == 70 { + oldest = p.Hash + } + } + if observationCount != int64(c.Shape.Observations) { + return fmt.Errorf("fanout generated %d observations, expected %d", observationCount, c.Shape.Observations) + } + if e := exec(`UPDATE nodes SET advert_count=(SELECT count(*) FROM transmissions t WHERE t.payload_type=4 AND t.from_pubkey=nodes.public_key)`); e != nil { + return e + } + for i := 0; i < c.Shape.Nodes; i++ { + if e := exec(`INSERT INTO neighbor_edges(node_a,node_b,count,last_seen) VALUES(?,?,?,?)`, f.public[i], f.public[(i+1)%len(f.public)], 1+i%100, stamp); e != nil { + return e + } + } + metric, e := tx.Prepare(`INSERT INTO observer_metrics(observer_id,timestamp,noise_floor,tx_air_secs,rx_air_secs,recv_errors,battery_mv,packets_sent,packets_recv) VALUES(?,?,?,?,?,?,?,?,?)`) + if e != nil { + return e + } + defer metric.Close() + for observer := 0; observer < c.Shape.Observers; observer++ { + for sample := 0; sample < c.Shape.Days*288; sample++ { + if observer == c.Shape.Observers-1 && sample%48 != 0 { + continue + } + at := time.Unix(c.Epoch-int64(c.Shape.Days*86400)+int64(sample*300), 0).UTC().Format(time.RFC3339) + if _, e := metric.Exec(f.public[observer], at, -110.5+float64(observer%5), sample*2, sample*9, sample%3, 3700+observer%200, sample*3, sample*20); e != nil { + return e + } + } + } + // Every enabled mobile feature has deterministic data; this is the base + // corpus extension, not a claim to have run the optional million-RX corpus. + reception, e := tx.Prepare(`INSERT INTO client_receptions(rx_pubkey,heard_key,heard_keylen,rssi,snr,lat,lon,pos_acc_m,rx_at,ingested_at,src) VALUES(?,?,?,?,?,?,?,?,?,?,?)`) + if e != nil { + return e + } + defer reception.Close() + for i := 0; i < c.Shape.Transmissions; i++ { + key := f.public[i%len(f.public)] + size := []int{32, 8, 3, 2}[i%4] + at := time.Unix(benchStamp(c, i), int64(i)).UTC().Format("2006-01-02T15:04:05.000000000Z") + if _, e := reception.Exec(f.public[(i+100)%len(f.public)], key[:size*2], size, -95, 6.5, 20+float64(i%2000)/10000, 30+float64(i%2000)/10000, 5, at, stamp, "synthetic"); e != nil { + return e + } + } + for i := 0; i < 100; i++ { + pk := f.public[i%len(f.public)] + at := time.Unix(c.Epoch-int64(i*300), 0).UTC().Format(time.RFC3339) + if e := exec(`INSERT INTO client_observers(pubkey,name,last_seen) VALUES(?,?,?)`, pk, fmt.Sprintf("Synthetic mobile %d", i), stamp); e != nil { + return e + } + if e := exec(`INSERT INTO node_declared_regions(target,rx_pubkey,observed_at,ingested_at,regions_csv,truncated,lat,lon) VALUES(?,?,?,?,?,0,20,30)`, pk, pk, at, stamp, "#bench"); e != nil { + return e + } + if e := exec(`INSERT INTO client_rf_samples(rx_pubkey,sampled_at,ingested_at,lat,lon,stationary,uptime_secs,noise_floor,last_snr) VALUES(?,?,?,20,30,0,1000,-110,6.5)`, pk, at, stamp); e != nil { + return e + } + if e := exec(`INSERT INTO dropped_packets(hash,raw_hex,reason,observer_id,dropped_at) VALUES(?,?,?,?,?)`, fmt.Sprintf("synthetic-drop-%d", i), "", "synthetic-rejection", pk, stamp); e != nil { + return e + } + } + if e := exec(`INSERT INTO client_rx_observations(rx_pubkey,rx_at,ingested_at,pkt_hash,route_type,payload_type,hash_size,hop_count,lat,lon) SELECT ?,first_seen,?,hash,route_type,payload_type,1,0,20,30 FROM transmissions WHERE id<=100`, f.public[0], stamp); e != nil { + return e + } + if e := exec(`INSERT INTO inactive_nodes(public_key,name,role,first_seen,last_seen,advert_count) VALUES('ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff','Synthetic inactive','repeater',?,?,0)`, stamp, stamp); e != nil { + return e + } + if e := exec(`INSERT INTO advert_evidence_backfill(id,tx_cursor,obs_cursor) VALUES(1,?,?)`, c.Shape.Transmissions, c.Shape.Observations); e != nil { + return e + } + if e := exec(`UPDATE _async_migrations SET started_at=?,ended_at=?`, stamp, stamp); e != nil { + return e + } + if e := exec(`INSERT OR REPLACE INTO _async_migrations(name,status,started_at,ended_at) VALUES('advert_route_evidence_v1','done',?,?)`, stamp, stamp); e != nil { + return e + } + if e := exec(`UPDATE scope_match_totals SET since_unix=?,updated_unix=?`, c.Epoch, c.Epoch); e != nil { + return e + } + if e := tx.Commit(); e != nil { + return e + } + if _, e := s.db.Exec(`ANALYZE`); e != nil { + return e + } + summary := map[string]any{"transmissions": c.Shape.Transmissions, "observations": observationCount, "transmissions_by_day": countsByDay, "nodes": c.Shape.Nodes, "observers": c.Shape.Observers, "channels": 20, "rx_rows": c.Shape.Transmissions, "dense_observer": f.public[0], "sparse_observer": f.public[c.Shape.Observers-1], "node": f.public[0], "new_packet": newest, "old_packet": oldest, "epoch": c.Epoch, "protocol_source": "meshcore-dev/MeshCore@a366955cb2f67b8e6842d4f00d2b6a554dddd88a"} + b, e := json.MarshalIndent(summary, "", " ") + if e != nil { + return e + } + return os.WriteFile(filepath.Join(c.Output, "corpus.json"), b, 0600) +} + +var _ *sql.DB // database/sql remains the same application boundary on both sides. diff --git a/scripts/postgres-benchmark/run.py b/scripts/postgres-benchmark/run.py new file mode 100644 index 000000000..5a045deef --- /dev/null +++ b/scripts/postgres-benchmark/run.py @@ -0,0 +1,1140 @@ +#!/usr/bin/env python3 +"""Isolated, serial, source-pinned CoreScope application comparison. + +No production service is contacted. A successful invocation means the requested +measurements and validation completed, not that PostgreSQL is faster. +""" +import argparse +import csv +import decimal +import datetime +import errno +import hashlib +import json +import math +import os +from pathlib import Path +import platform +import re +import shutil +import signal +import socket +import sqlite3 +import subprocess +import sys +import tarfile +import threading +import time +import urllib.error +import urllib.parse +import urllib.request + +BASELINE = "3e21b179ecccae005aa5f6f45bc095353c94b343" +FIRMWARE = "a366955cb2f67b8e6842d4f00d2b6a554dddd88a" +HERE = Path(__file__).resolve().parent +PROCESSES = [] +RESOURCE_COUNTERS = ("cgroup.procs", "memory.current", "memory.events", "cpu.stat", "io.stat") +# Common to both engines: B itself accounts for about 1,222 MiB before replay. +SERVER_MEMORY_MIB = 3072 +PACKET_STORE_MIB = 2048 +TABLES = ["nodes", "inactive_nodes", "observers", "transmissions", "observations", "observer_metrics", "neighbor_edges", "dropped_packets", "client_receptions", "client_observers", "client_rx_observations", "client_rf_samples", "node_declared_regions", "scope_match_totals", "_migrations", "_async_migrations", "advert_route_evidence", "advert_evidence_backfill"] +# These are operational receipt/creation times, not packet timestamps. Full +# pre-run parity includes them; only post-replay logical parity excludes them. +VOLATILE = {"transmissions": {"created_at"}, "observers": {"last_seen", "last_packet_at", "first_seen"}, "_async_migrations": {"started_at", "ended_at"}, "scope_match_totals": {"updated_unix", "since_unix"}} +TEXT_TIMES = {"first_seen", "last_seen", "created_at", "last_packet_at", "clock_last_naive_at", "configured_scope_at", "timestamp", "dropped_at", "rx_at", "ingested_at", "sampled_at", "observed_at", "started_at", "ended_at"} + + +def shift_timestamp(value, delta): + if value is None or value == "": + return value + if not re.fullmatch(r"\d{4}-\d\d-\d\d[T ]\d\d:\d\d:\d\d(?:\.\d+)?(?:Z|[+-]\d\d:\d\d)?", value): + raise ValueError("unsupported synthetic timestamp") + head = datetime.datetime.strptime(value[:19].replace("T", " "), "%Y-%m-%d %H:%M:%S") + datetime.timedelta(seconds=delta) + return head.strftime("%Y-%m-%d" + value[10] + "%H:%M:%S") + value[19:] + + +def freshen(path, delta): + """One recorded receipt-time offset; signed wire bytes remain unchanged.""" + db = sqlite3.connect(path) + try: + db.create_function("bench_shift", 1, lambda value: shift_timestamp(value, delta)) + for table in TABLES: + columns = list(db.execute(f'PRAGMA table_info("{table}")')) + updates = [] + for _, name, kind, *_ in columns: + if name not in TEXT_TIMES and not (table == "scope_match_totals" and name in ("since_unix", "updated_unix")): + continue + if kind.upper() in ("INTEGER", "INT", "BIGINT"): + updates.append(f'"{name}"="{name}"+{int(delta)}') + elif kind.upper() in ("TEXT", "DATETIME"): + updates.append(f'"{name}"=bench_shift("{name}")') + if updates: + db.execute(f'UPDATE "{table}" SET ' + ",".join(updates)) + db.commit() + db.execute("PRAGMA wal_checkpoint(TRUNCATE)") + finally: + db.close() + + +def corpus(name): + return {"S": dict(transmissions=30_000, observations=90_000, nodes=2000, observers=32, days=8), "B": dict(transmissions=128_000, observations=2_048_000, nodes=2000, observers=128, days=8), "L": dict(transmissions=1_000_000, observations=16_000_000, nodes=5000, observers=512, days=30)}[name].copy() + + +def fanout(name, index): + return index % 5 + 1 if name == "S" else (4 if index % 10 < 5 else 22 if index % 10 < 7 else 23 if index % 10 < 9 else 50) + + +def pair_order(pairs): + return [(i, backend) for i in range(pairs) for backend in (("sqlite", "postgres") if i % 2 == 0 else ("postgres", "sqlite"))] + + +def percentiles(values): + values = sorted(values) + def q(p): + return values[max(0, math.ceil(p * len(values)) - 1)] if values else None + return dict(samples=len(values), p50=q(.50), p95=q(.95), p99=q(.99) if len(values) >= 1000 else None, maximum=q(1)) + + +def visibility(ingest, messages): + sent = {e["hash"]: e for e in ingest if e.get("new_transmission") and not e.get("error") and not e.get("dropped")} + measured = {key: event for key, event in sent.items() if event.get("measured")} + received, duplicate_receipts, duplicate_receipts_all = {}, 0, 0 + for message in messages: + key = message.get("hash") + if key not in sent: + continue + if key in received: + duplicate_receipts_all += 1 + duplicate_receipts += int(key in measured) + received[key] = min(received.get(key, message["received_ns"]), message["received_ns"]) + missing = sorted(set(sent) - set(received)) + lags = [received[key] - event["completed_ns"] for key, event in measured.items() if key in received] + return dict(verified=not missing, verification_scope="full stream including warmup", sample_scope="measured events only", + sentinels=len(sent), received=len(received), missing=len(missing), missing_hashes=missing[:100], + measured_sentinels=len(measured), measured_received=len(measured.keys() & received.keys()), measured_missing=len(measured.keys() - received.keys()), + duplicate_hash_receipts=duplicate_receipts, duplicate_hash_receipts_all=duplicate_receipts_all, + visible_before_ingest_return=sum(x < 0 for x in lags), + visible_before_ingest_return_all=sum(received[key] < event["completed_ns"] for key, event in sent.items() if key in received), + lag_ns=percentiles(lags), samples_ns=lags) + + +def json_lines(path): + with open(path) as stream: + for line in stream: + yield json.loads(line) + + +def canonical_row(row): + out = [] + for value in row: + if value is None: + out.append(["null"]) + elif isinstance(value, (int, float, decimal.Decimal)) and not isinstance(value, bool): + number = decimal.Decimal(str(value)) + if not number.is_finite(): + raise ValueError("non-finite number in corpus") + out.append(["number", str(number.normalize()) if number else "0"]) + elif isinstance(value, str): + out.append(["text", value]) + else: + raise ValueError("unsupported corpus cell type") + return (json.dumps(out, ensure_ascii=False, separators=(",", ":")) + "\n").encode() + + +def safe_output(path): + path = Path(path).absolute() + if not path.name.startswith("corescope-bench-") or path.exists() or path.is_symlink(): + raise ValueError("output must be a new directory named corescope-bench-") + if not path.parent.is_dir(): + raise ValueError("output parent must already exist") + return path.resolve() + + +def check_revisions(baseline, candidate): + if baseline != BASELINE or not re.fullmatch(r"[0-9a-f]{40}", candidate): + raise ValueError("use the fixed upstream baseline and a complete 40-character candidate commit SHA") + + +def validate_profile(profile, pairs, cgroup): + if profile == "primary" and (pairs != 5 or not cgroup): + raise ValueError("primary requires five paired runs and a delegated CPU/memory cgroup") + if not 1 <= pairs <= 10: + raise ValueError("pairs must be between one and ten") + + +def write_json(path, value): + Path(path).parent.mkdir(parents=True, exist_ok=True) + temporary = Path(str(path) + ".tmp") + temporary.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n") + temporary.replace(path) + + +def safe_diagnostics(text): + """Keep diagnostic lines only; never publish arbitrary child log tails.""" + out = [] + for line in text.splitlines(): + line = line.strip() + if line.startswith("[migrate] "): + line = "migrate: " + line[len("[migrate] "):] + if not re.match(r"(?:--- FAIL:|FAIL\b|panic:|fatal error:|[^\s]*\.go:\d+(?::\d+)?:|ERROR:|FATAL:|error:|benchmark:|migrate:)", line): + codes = re.findall(r"SQLSTATE [0-9A-Z]{5}", line) + if codes: + out.extend(codes) + continue + if line.startswith("panic:") and not line.startswith("panic: runtime error:"): + line = "panic: " + elif re.search(r"\b(?:host|hostaddr|user|dbname|password|sslmode)\s*=", line, re.I): + line = "database connection error: " + else: + line = re.sub(r"[a-zA-Z][a-zA-Z0-9+.-]*://\S+", "", line) + line = re.sub(r"\b[A-Za-z]:[\\/]\S+|(?", line) + line = re.sub(r"\{.*\}|\[.*\]", "", line) + line = re.sub(r'"[^"\n]*"|\x27[^\x27\n]*\x27', "", line) + line = re.sub(r"\b(?:raw|payload|packet|message|body|authorization|cookie|password|passwd|secret|token|dsn|database_url|credentials?)\s*[:=].*", "", line, flags=re.I) + line = re.sub(r"\b[0-9a-fA-F]{16,}\b|(?", line) + out.append(line[:240]) + return out[-10:] + + +def postgres_startup_errors(text): + # Only fixed error kinds and known SQLSTATE codes are public. Server log + # text, directories, connection strings and arbitrary values stay private. + codes = {"08001", "08006", "22023", "42501", "53100", "53200", "53300", "53400", + "55000", "58000", "58030", "58P01", "58P02", "F0000", "F0001", "XX000", "XX001", "XX002"} + errors = [] + for line in text.splitlines(): + match = re.search(r"(?:^|\]\s+)(FATAL|PANIC|ERROR):\s*(?:([0-9A-Z]{5}):\s*)?(.*)", line) + if not match: + continue + severity, code, message = match.groups() + message = message.lower() + kind = "unclassified_postgres_startup_error" + if "could not create lock file" in message and ".s.pgsql." in message and "permission denied" in message: + kind = "unix_socket_lock_permission_denied" + elif "could not bind" in message and "address already in use" in message: + kind = "socket_address_in_use" + elif "could not create any tcp/ip sockets" in message: + kind = "tcp_listener_unavailable" + elif "could not create any unix-domain sockets" in message: + kind = "unix_socket_unavailable" + elif "shared memory" in message: + kind = "shared_memory_unavailable" + elif "invalid value for parameter" in message or "configuration file" in message: + kind = "invalid_server_configuration" + elif "data directory" in message: + kind = "data_directory_unavailable" + errors.append(dict(severity=severity, sqlstate=code if code in codes else None, kind=kind)) + return errors[-10:] or [dict(severity="UNKNOWN", sqlstate=None, kind="unclassified_postgres_startup_error")] + + +def record_failure(log, exit_code): + log = Path(log) + with open(log, "rb") as stream: + stream.seek(max(0, log.stat().st_size - 8192)) + tail = stream.read(8192).decode("utf-8", errors="replace") + result = dict(log=log.name, exit_code=exit_code, diagnostics=safe_diagnostics(tail), policy="At most 10 sanitized diagnostic lines from the final 8 KiB; raw logs and event bodies excluded") + if log.name == "postgres.log": + result["postgres_errors"] = postgres_startup_errors(tail) + result["diagnostics"] = ["{severity}: {sqlstate} {kind}".format(**error) for error in result["postgres_errors"]] + result["policy"] = "Only allowlisted PostgreSQL startup error kinds and SQLSTATE codes; raw messages excluded" + write_json(log.with_name("failure-" + log.stem + ".json"), result) + return result + + +def record_resource_failure(path, error, **context): + filename = getattr(error, "filename", None) + name = Path(filename).name if filename else None + known = (*RESOURCE_COUNTERS, "memory.peak", "smaps_rollup", "comm", "resources.csv", "processes.csv", "resource-final.json", "resource-error.json") + detail = dict(error_type=type(error).__name__, errno=getattr(error, "errno", None), + counter=name if name in known else None, **context) + try: + write_json(path, detail) + except OSError as write_error: + # A full/unwritable output filesystem must not replace the original + # workload exception with another cleanup exception. + detail["evidence_write_error_type"] = type(write_error).__name__ + detail["evidence_write_errno"] = write_error.errno + return detail + + +def public_bundle(output, repository=None): + """Explicit whitelist: never export databases, event bodies or controls.""" + output = Path(output).resolve() + public = output / "public" + public.mkdir(exist_ok=True) + patterns = ["manifest.json", "resource-preflight.json", "postgres-preflight.json", "summary*.csv", "paired-summary.json", "report.md", "logs/failure-*.json", "corpus/*.json", "pair-*/corpus-offset.json", "pair-*/events.json", "pair-*/*/failure-*.json", "pair-*/*/*.jsonl", "pair-*/*/resources.csv", "pair-*/*/processes.csv", "pair-*/*/resource-final.json", "pair-*/*/resource-error.json", "pair-*/*/*validation*.json", "pair-*/*/startup*.json", "pair-*/*/migration.json", "pair-*/*/retention*.json", "pair-*/*/handler.json", "pair-*/*/database-settings.json", "pair-*/*/reader-settings.json", "pair-*/*/http-workload.json", "pair-*/*/visibility.json", "pair-*/*/plans/*"] + for pattern in patterns: + for source in sorted(output.glob(pattern)): + if not source.is_file() or source.is_symlink(): + continue + destination = public / source.relative_to(output) + destination.parent.mkdir(parents=True, exist_ok=True) + text = source.read_text() + text = text.replace(str(output), "") + if repository: + text = text.replace(str(Path(repository).resolve()), "") + destination.write_text(text) + + +def sha256(path): + h = hashlib.sha256() + with open(path, "rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + h.update(block) + return h.hexdigest() + + +def remove_working_directory(path, runtime_root): + path, runtime_root = Path(path).resolve(), Path(runtime_root).resolve() + if runtime_root.name != ".runtime" or not path.is_relative_to(runtime_root) or not re.fullmatch(r"config-\d+-(sqlite|postgres)", path.name): + raise ValueError("refusing cleanup outside an owned benchmark run") + shutil.rmtree(path) + + +def command(args, *, cwd=None, env=None, log=None, input=None, timeout=900, budget=None): + args = [str(x) for x in args] + proc = subprocess.Popen(budget.arguments(args) if budget else args, cwd=cwd, env=env, stdin=subprocess.PIPE, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, start_new_session=True) + PROCESSES.append(proc) + try: + result, _ = proc.communicate(input, timeout=timeout) + except BaseException: + stop(proc) + raise + if log: + Path(log).write_text(result) + if proc.returncode: + if not log: + diagnostic="; ".join(safe_diagnostics(result[-8192:])) or "no recognized safe diagnostic" + raise RuntimeError(f"{Path(str(args[0])).name} failed (exit {proc.returncode}): {diagnostic}") + record_failure(log, proc.returncode) + raise RuntimeError(f"{Path(str(args[0])).name} failed (exit {proc.returncode}); see failure-{Path(log).stem}.json") + return result + + +def production_hash(root): + h = hashlib.sha256() + for folder in ("cmd", "internal", "public"): + for path in sorted((Path(root) / folder).rglob("*")): + if path.is_file() and not path.name.endswith("_test.go"): + h.update(path.relative_to(root).as_posix().encode() + b"\0") + h.update(bytes.fromhex(sha256(path))) + return h.hexdigest() + + +def archive(repo, revision, destination): + actual = command(["git", "rev-parse", revision + "^{commit}"], cwd=repo).strip() + if actual != revision: + raise ValueError("revision did not resolve to the exact requested commit") + destination.mkdir() + proc = subprocess.Popen(["git", "archive", "--format=tar", revision], cwd=repo, stdout=subprocess.PIPE) + try: + with tarfile.open(fileobj=proc.stdout, mode="r|") as archive_file: + archive_file.extractall(destination, filter="data") + finally: + proc.stdout.close() + if proc.wait() != 0: + raise RuntimeError("git archive failed") + + +class Budget: + def __init__(self, parent, required): + self.path = None + self._memory_peak_file = None + self.required = required + self.info = {"enforced": False, "cpu_limit": None, "memory_limit_bytes": None} + if not parent: + return + parent = Path(parent).resolve() + if not str(parent).startswith("/sys/fs/cgroup/") or parent == Path("/sys/fs/cgroup"): + raise ValueError("cgroup must be a delegated subtree, not the global cgroup root") + self.path = parent / ("corescope-bench-" + str(os.getpid())) + self.path.mkdir() + try: + (self.path / "cpu.max").write_text("300000 100000") + (self.path / "memory.max").write_text(str(6 * 1024 ** 3)) + if (self.path / "memory.swap.max").exists(): + (self.path / "memory.swap.max").write_text("0") + cpu, memory = 3.0, 6 * 1024 ** 3 + for ancestor in (self.path, *self.path.parents): + if ancestor == Path("/sys/fs"): + break + if (ancestor / "cpu.max").exists(): + q, period = (ancestor / "cpu.max").read_text().split() + if q != "max": + cpu = min(cpu, int(q) / int(period)) + if (ancestor / "memory.max").exists(): + value = (ancestor / "memory.max").read_text().strip() + if value != "max": + memory = min(memory, int(value)) + if cpu < 3 or memory < 6 * 1024 ** 3: + raise ValueError("ancestor cgroup limits are below the shared 3-CPU/6-GiB budget") + cpuset = self.path / "cpuset.cpus.effective" + allowed = set(os.sched_getaffinity(0)) + if cpuset.exists(): + effective = set() + for part in cpuset.read_text().strip().split(","): + bounds = [int(x) for x in part.split("-")] + effective.update(range(bounds[0], bounds[-1] + 1)) + allowed &= effective + if len(allowed) < 3: + raise ValueError("effective CPU affinity provides fewer than three application CPUs") + self.info = dict(enforced=True, cpu_limit=cpu, memory_limit_bytes=memory, allowed_cpus=sorted(allowed)) + except Exception: + self.path.rmdir() + self.path = None + raise + + def preflight(self): + if self.path: + for name in RESOURCE_COUNTERS: + (self.path / name).read_text() + self.info["resource_counters"] = list(RESOURCE_COUNTERS) + + def arguments(self, args): + if not self.path: + return args + # No Python preexec_fn: the resource sampler is threaded. Arguments are + # positional shell parameters, never interpolated into shell source. + return ["/bin/sh", "-c", 'printf "%s" "$$" > "$1/cgroup.procs" || exit 125; shift; exec "$@"', "corescope-bench", str(self.path), *args] + + def verify(self, pid): + if self.path: + for _ in range(100): + if str(pid) in (self.path / "cgroup.procs").read_text().split(): + return + time.sleep(.01) + raise RuntimeError("application/database process failed to enter the benchmark cgroup") + + def reset(self): + self.close_memory_peak() + self.info["memory_peak_reset"] = False + if not self.path: + return + if (self.path / "cgroup.procs").read_text().strip(): + raise RuntimeError("previous backend is still active; serial comparison refused") + reclaim = self.path / "memory.reclaim" + if not reclaim.exists(): + raise RuntimeError("primary cgroup needs memory.reclaim to remove inactive prior-backend charges") + amount = int((self.path / "memory.current").read_text()) + if amount: + try: + reclaim.write_text(str(amount)) + except OSError as error: + if error.errno != errno.EAGAIN: + raise + remaining = int((self.path / "memory.current").read_text()) + if remaining > 64 * 1024 ** 2: + raise RuntimeError("more than 64 MiB of inactive prior-backend cgroup memory remains") + self.info["inactive_charge_reset_bytes"] = remaining + try: + # Linux resets memory.peak only for subsequent reads through this + # same open file description, not later opens of the counter. + self._memory_peak_file = (self.path / "memory.peak").open("r+b", buffering=0) + self._memory_peak_file.write(b"0") + self.info["memory_peak_reset"] = True + except OSError: + self.close_memory_peak() + + def memory_peak(self): + if self._memory_peak_file is None: + return None + self._memory_peak_file.seek(0) + return int(self._memory_peak_file.read()) + + def close_memory_peak(self): + if self._memory_peak_file is not None: + self._memory_peak_file.close() + self._memory_peak_file = None + + def close(self): + self.close_memory_peak() + if self.path: + if (self.path / "cgroup.procs").read_text().strip(): + raise RuntimeError("owned benchmark processes remain; refusing cgroup removal") + self.path.rmdir() + + +def spawn(args, *, cwd, env, log, budget=None): + stream = open(log, "w") + args = [str(x) for x in args] + proc = subprocess.Popen(budget.arguments(args) if budget else args, cwd=cwd, env=env, stdout=stream, stderr=subprocess.STDOUT, start_new_session=True) + PROCESSES.append(proc) + proc.bench_log = stream + if budget: + budget.verify(proc.pid) + return proc + + +def finish(proc, timeout=900): + try: + result = proc.wait(timeout=timeout) + if result: + record_failure(proc.bench_log.name, result) + raise RuntimeError(f"benchmark child exited {result}; see failure-{Path(proc.bench_log.name).stem}.json") + finally: + if proc.poll() is None: + os.killpg(proc.pid, signal.SIGTERM) + try: + proc.wait(timeout=15) + except subprocess.TimeoutExpired: + os.killpg(proc.pid, signal.SIGKILL) + proc.wait() + proc.bench_log.close() + + +def stop(proc): + if proc and proc.poll() is None: + os.killpg(proc.pid, signal.SIGTERM) + try: + proc.wait(timeout=30) + except subprocess.TimeoutExpired: + os.killpg(proc.pid, signal.SIGKILL) + proc.wait() + if proc and getattr(proc, "bench_log", None): + proc.bench_log.close() + + +def free_port(): + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + return sock.getsockname()[1] + + +class Postgres: + def __init__(self, runtime, bin_dir, budget, logs): + self.root, self.bin, self.budget, self.logs = runtime, Path(bin_dir), budget, logs + self.data, self.port = runtime / "pgdata", free_port() + self.proc = None + command([self.bin / "initdb", "-D", self.data, "--no-locale", "--encoding=UTF8", "--auth-local=trust", "--auth-host=trust", "-U", "bench_admin"], log=logs / "initdb.log", budget=budget) + self.env = dict({k:v for k,v in os.environ.items() if not k.startswith("PG")}, PGHOST="127.0.0.1", PGPORT=str(self.port), PGUSER="bench_admin", PGDATABASE="postgres") + + def start(self): + self.proc = spawn([self.bin / "postgres", "-D", self.data, "-h", "127.0.0.1", "-p", self.port, "-c", "unix_socket_directories=", "-c", "log_line_prefix=", "-c", "log_error_verbosity=verbose", "-c", "fsync=on", "-c", "synchronous_commit=on", "-c", "full_page_writes=on", "-c", "shared_buffers=256MB", "-c", "max_connections=32"], cwd=self.root, env=self.env, log=self.logs / "postgres.log", budget=self.budget) + for _ in range(200): + if self.proc.poll() is not None: + failure = record_failure(self.proc.bench_log.name, self.proc.returncode) + raise RuntimeError("private PostgreSQL failed to start: " + "; ".join(failure["diagnostics"])) + if subprocess.run([str(self.bin / "pg_isready"), "-q"], env=self.env).returncode == 0: + self.sql("SELECT 1") + return + time.sleep(.1) + record_failure(self.proc.bench_log.name, None) + raise TimeoutError("private PostgreSQL readiness timeout; see failure-postgres.json") + + def stop(self): + stop(self.proc) + self.proc = None + + def sql(self, query, database="postgres"): + return command([self.bin / "psql", "-X", "-qAt", "-v", "ON_ERROR_STOP=1"], env=dict(self.env, PGDATABASE=database), input=query, log=None) + + def create(self, name): + if not re.fullmatch(r"bench_[a-z0-9_]+", name): + raise ValueError("non-benchmark database name") + self.sql(f"CREATE ROLE {name}_owner LOGIN; CREATE ROLE {name}_writer LOGIN; CREATE ROLE {name}_reader LOGIN; CREATE DATABASE {name} OWNER {name}_owner TEMPLATE template0 LC_COLLATE 'C' LC_CTYPE 'C';") + return {role: f"postgres://{name}_{role}@127.0.0.1:{self.port}/{name}?sslmode=disable" for role in ("owner", "writer", "reader")} + + def grant(self, name): + self.sql(f"GRANT USAGE ON SCHEMA public TO {name}_writer,{name}_reader; GRANT SELECT ON ALL TABLES IN SCHEMA public TO {name}_writer,{name}_reader; GRANT INSERT,UPDATE,DELETE ON ALL TABLES IN SCHEMA public TO {name}_writer; GRANT USAGE,SELECT ON ALL SEQUENCES IN SCHEMA public TO {name}_writer; DO $$DECLARE t record;BEGIN FOR t IN SELECT tablename FROM pg_tables WHERE schemaname='public' AND tablename LIKE 'corescope_%' LOOP EXECUTE format('REVOKE INSERT,UPDATE,DELETE,TRUNCATE,TRIGGER ON %I FROM {name}_writer',t.tablename);END LOOP;END$$; ANALYZE;", name) + + def drop(self, name): + if not re.fullmatch(r"bench_[a-z0-9_]+", name): + raise ValueError("non-benchmark database name") + self.sql(f"DROP DATABASE {name} WITH (FORCE); DROP ROLE {name}_reader,{name}_writer,{name}_owner;") + + +def table_manifest(source): + with sqlite3.connect(f"file:{source}?mode=ro", uri=True) as db: + manifest = [] + for table in TABLES: + columns = [row[1] for row in db.execute(f'PRAGMA table_info("{table}")')] + if not columns: + raise ValueError("corpus lacks canonical table " + table) + if table == "observers": + columns.insert(0, "rowid") + manifest.append(dict(name=table, columns=columns)) + return manifest + + +def validate(source, manifest, pg=None, database=None, logical=False): + result = {} + db = sqlite3.connect(f"file:{source}?mode=ro", uri=True) if pg is None else None + try: + for table in manifest: + name = table["name"] + columns = [c for c in table["columns"] if not logical or c not in VOLATILE.get(name, set())] + select = ",".join('"' + c + '"' for c in columns) + # Ordering by the complete row is deterministic even with optional + # nullable keys. Diagnostic sorting is outside headline timing. + if pg: + # SQLite sorts NULL before values; PostgreSQL's default is last. + order = ",".join('"' + c + '" NULLS FIRST' for c in columns) + query = f'SELECT json_build_array({select})::text FROM "{name}" ORDER BY {order};' + proc = subprocess.Popen([str(pg.bin / "psql"), "-X", "-qAt", "-v", "ON_ERROR_STOP=1"], env=dict(pg.env, PGDATABASE=database), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, start_new_session=True) + PROCESSES.append(proc) + proc.stdin.write(query) + proc.stdin.close() + rows = (json.loads(line, parse_float=decimal.Decimal) for line in proc.stdout) + else: + rows = db.execute(f'SELECT {select} FROM "{name}" ORDER BY {select}') + digest, count = hashlib.sha256(), 0 + for row in rows: + digest.update(canonical_row(row)) + count += 1 + if pg: + diagnostic = proc.stderr.read() + if proc.wait() != 0: + raise RuntimeError("PostgreSQL corpus validation query failed: " + "; ".join(safe_diagnostics(diagnostic))) + result[name] = dict(rows=count, sha256=digest.hexdigest()) + finally: + if db: + db.close() + return result + + +class Resources: + def __init__(self, path, budget, data_paths): + self.path, self.budget, self.data_paths = path, budget, data_paths + self.phase="prepare" + self.done = threading.Event() + self.error = None + self.thread = threading.Thread(target=self.sample, daemon=True) + + def sample(self): + try: + self.sample_rows() + except BaseException as error: + self.error = error + + def sample_rows(self): + with open(self.path, "w", newline="") as stream, open(self.path.parent/"processes.csv","w",newline="") as processes: + process_writer=csv.DictWriter(processes,fieldnames=["mono_ns","phase","pid","command","rss_bytes","pss_bytes"]) + process_writer.writeheader() + writer = csv.DictWriter(stream, fieldnames=["mono_ns", "phase", "filesystem_free_bytes", "cgroup_memory_bytes", "cgroup_memory_peak_bytes", "cgroup_cpu_usec", "rss_bytes", "pss_bytes", "storage_bytes", "wal_bytes", "io_stat", "memory_events"]) + writer.writeheader() + while not self.done.is_set(): + row = dict(mono_ns=time.monotonic_ns(), phase=self.phase, filesystem_free_bytes=shutil.disk_usage(self.path.parent).free, cgroup_memory_bytes="not measured", cgroup_memory_peak_bytes="not measured", cgroup_cpu_usec="not measured", rss_bytes="not measured", pss_bytes="not measured", storage_bytes=0, wal_bytes=0, io_stat="not measured", memory_events="not measured") + if self.budget.path: + c = self.budget.path + row["cgroup_memory_bytes"] = (c / "memory.current").read_text().strip() + peak = self.budget.memory_peak() + if peak is not None: + row["cgroup_memory_peak_bytes"] = peak + cpu = dict(line.split() for line in (c / "cpu.stat").read_text().splitlines()) + row["cgroup_cpu_usec"] = cpu.get("usage_usec", "not measured") + row["io_stat"] = (c / "io.stat").read_text().strip() + row["memory_events"] = (c / "memory.events").read_text().strip() + rss = pss = 0 + for pid in (c / "cgroup.procs").read_text().split(): + try: + process_rss=process_pss=0 + for line in Path("/proc", pid, "smaps_rollup").read_text().splitlines(): + if line.startswith("Rss:"): + process_rss += int(line.split()[1]) * 1024 + if line.startswith("Pss:"): + process_pss += int(line.split()[1]) * 1024 + rss+=process_rss;pss+=process_pss + process_writer.writerow(dict(mono_ns=row["mono_ns"],phase=self.phase,pid=pid,command=Path("/proc",pid,"comm").read_text().strip(),rss_bytes=process_rss,pss_bytes=process_pss)) + except (FileNotFoundError, PermissionError, ProcessLookupError): + pass + row.update(rss_bytes=rss, pss_bytes=pss) + for root in self.data_paths: + for file in root.rglob("*") if root.is_dir() else [root, Path(str(root) + "-wal")]: + try: + if file.is_file(): + size = file.stat().st_size + row["storage_bytes"] += size + if file.name.endswith("-wal") or "pg_wal" in file.parts: + row["wal_bytes"] += size + except FileNotFoundError: + pass + writer.writerow(row) + stream.flush() + processes.flush() + self.done.wait(1) + + def __enter__(self): + self.thread.start() + return self + + def __exit__(self, exc_type, exc_value, traceback): + try: + self.done.set() + self.thread.join(timeout=10) + if self.thread.is_alive(): + raise TimeoutError("resource sampler failed to stop") + if self.error: + raise self.error + if self.budget.path: + c=self.budget.path + write_json(self.path.parent/"resource-final.json",dict(memory_peak_bytes=self.budget.memory_peak(),cpu_stat=(c/"cpu.stat").read_text(),io_stat=(c/"io.stat").read_text(),memory_events=(c/"memory.events").read_text(),budget=self.budget.info)) + except BaseException as error: + detail = record_resource_failure(self.path.parent/"resource-error.json", error, phase=self.phase, + workload_error_type=exc_type.__name__ if exc_type else None) + if exc_value is None: + raise RuntimeError("resource sampler failed: " + json.dumps(detail, sort_keys=True)) from error + finally: + self.budget.close_memory_peak() + return False + + +def wait_file(path, processes, seconds=120): + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + if path.exists(): + return + if any(p.poll() is not None for p in processes): + for proc in processes: + if proc.poll() is not None and getattr(proc, "bench_log", None): + record_failure(proc.bench_log.name, proc.returncode) + raise RuntimeError("benchmark worker exited before the start barrier") + time.sleep(.05) + raise TimeoutError("benchmark start barrier timeout") + + +def server_settings(backend, sqlite_path, state_dir, run_env): + config = dict(hashChannels=[f"#bench-{i:02d}" for i in range(20)], dbPath=str(sqlite_path), stateDir=str(state_dir), port=free_port(), packetStore=dict(retentionHours=168, maxMemoryMB=PACKET_STORE_MIB, hotStartupHours=0), runtime=dict(maxMemoryMB=SERVER_MEMORY_MIB), clientRxCoverage=dict(enabled=True), userManagement=dict(enabled=False)) + if backend == "postgres": + config.pop("dbPath") + env = dict(run_env, GOMEMLIMIT=f"{SERVER_MEMORY_MIB}MiB", CORESCOPE_DATABASE_URL=run_env.get("CORESCOPE_READER_DATABASE_URL", "")) + return config, env + + +def record_postgres_selection(binaries, config_dir, state_dir, run_env, owner_url, log, budget): + """Adopt the verified imported target before measuring runtime startup.""" + command([binaries / "migrate", "-storage-action=adopt", "-backend=postgres", "-offline", + "-selection-file", state_dir / "storage-selection.json", "-config-dir", config_dir], + cwd=config_dir, env=dict(run_env, CORESCOPE_DATABASE_URL=owner_url), + log=log, timeout=180, budget=budget) + + +def retained_counts(source, since_epoch): + db = sqlite3.connect(f"file:{source}?mode=ro", uri=True) + try: + tx = db.execute("SELECT count(*) FROM transmissions WHERE last_seen >= ?", (since_epoch,)).fetchone()[0] + obs = db.execute("SELECT count(*) FROM observations o JOIN transmissions t ON t.id=o.transmission_id WHERE t.last_seen >= ?", (since_epoch,)).fetchone()[0] + return dict(transmissions=tx, observations=obs) + finally: + db.close() + + +def check_loaded_rows(health, expected): + if health.get("loadedTx") != expected["transmissions"] or health.get("loadedObs") != expected["observations"]: + raise RuntimeError("startup did not load the complete retained corpus: " + json.dumps(dict(actual=health, expected=expected))) + + +def startup(server, base_url, output, expected, launched_ns, hot_hours, retained_expected=None): + start, first, ready = launched_ns, None, None + deadline = time.monotonic() + 600 + last = None + while time.monotonic() < deadline: + if server.poll() is not None: + record_failure(server.bench_log.name, server.returncode) + raise RuntimeError("server exited during startup") + try: + with urllib.request.urlopen(base_url + "/api/healthz", timeout=5) as response: + body = json.load(response) + first = first or time.monotonic_ns() + last = body + if response.status == 200 and response.headers.get("X-CoreScope-Load-Status") == "ready": + ready = time.monotonic_ns() + break + except urllib.error.HTTPError as error: + first = first or time.monotonic_ns() + last = {"ready":False,"status":error.code} + except (urllib.error.URLError, TimeoutError, json.JSONDecodeError): + pass + time.sleep(.1) + if ready is None: + raise TimeoutError("server did not reach full health/readiness") + full = ready + perf = None + while time.monotonic() < deadline: + with urllib.request.urlopen(base_url + "/api/perf", timeout=30) as response: + perf = json.load(response) + store = perf.get("packetStore", {}) + if store.get("backgroundLoadFailed"): + raise RuntimeError("background startup load failed") + if not hot_hours or store.get("backgroundLoadComplete") is True: + full = time.monotonic_ns() + break + time.sleep(.1) + else: + raise TimeoutError("background startup did not complete") + initial_health=last + with urllib.request.urlopen(base_url+"/api/healthz",timeout=30) as response: + last=json.load(response) + if not last or last.get("loadedTx",0) <= 0 or last.get("loadedObs",0) <= 0: + raise RuntimeError("ready server did not load a meaningful corpus") + if retained_expected is not None: + check_loaded_rows(last, retained_expected) + with urllib.request.urlopen(base_url + "/api/stats", timeout=30) as response: + stats = json.load(response) + result = dict(first_http_ns=first-start, ready_ns=ready-start, full_ready_ns=full-start, hot_startup_hours=hot_hours, initial_health=initial_health, health=last, stats=stats, perf=perf, durable_expected=expected, retained_expected=retained_expected, loaded_count_status="complete retained corpus verified" if retained_expected is not None else "bounded capacity load; actual counts recorded under common memory/retention limits") + write_json(output, result) + return result + + +def summarize(output): + rows=[] + def add(pair,backend,scenario,name,values,errors=0,drops=0,throughput=None): + stats=percentiles(values) + rows.append(dict(pair=pair,backend=backend,scenario=scenario,event_class=name,samples=stats["samples"],p50_ns=stats["p50"],p95_ns=stats["p95"],p99_ns=stats["p99"],maximum_ns=stats["maximum"],errors=errors,drops=drops,completed_per_second_including_drain=throughput)) + for directory in sorted(output.glob("pair-*/*")): + if not directory.is_dir():continue + pair,backend=directory.parent.name,directory.name + for file,scenario in (("requests.jsonl","mixed-http"),("sql-queries.jsonl","sql-miss"),("ingest.jsonl","durable-ingest")): + path=directory/file + if not path.exists():continue + groups={} + for item in json_lines(path): + if not item.get("measured"):continue + group=groups.setdefault(item["class"],dict(values=[],errors=0,drops=0,first=None,last=None)) + if item.get("dropped"): + group["drops"]+=1;continue + if item.get("error") or item.get("status",200)!=(item.get("expected_status") or 200): + group["errors"]+=1;continue + begun=item.get("dispatched_ns",item.get("started_ns")) + group["values"].append(item["completed_ns"]-begun) + scheduled=item.get("scheduled_ns") or begun + group["first"]=min(group["first"] or scheduled,scheduled) + group["last"]=max(group["last"] or item["completed_ns"],item["completed_ns"]) + for name,group in sorted(groups.items()): + elapsed=(group["last"] or 0)-(group["first"] or 0) + rate=len(group["values"])/(elapsed/1e9) if elapsed>0 else None + add(pair,backend,scenario,name,group["values"],group["errors"],group["drops"],rate) + for filename in ("startup.json","startup-hot1.json"): + if (directory/filename).exists(): + obj=json.loads((directory/filename).read_text()) + for phase in ("first_http_ns","ready_ns","full_ready_ns"): + add(pair,backend,filename.removesuffix(".json"),phase,[obj[phase]]) + for filename,key in (("migration.json","migration_tool_ns"),("retention.json","elapsed_ns"),("retention-empty.json","elapsed_ns")): + if (directory/filename).exists(): + obj=json.loads((directory/filename).read_text());add(pair,backend,filename.removesuffix(".json"),key,[obj[key]]) + if (directory/"handler.json").exists(): + obj=json.loads((directory/"handler.json").read_text());add(pair,backend,"handler-control","200-valid-envelopes",obj["samples_ns"]) + if (directory/"visibility.json").exists(): + obj=json.loads((directory/"visibility.json").read_text());add(pair,backend,"websocket","new-transmission-sentinels",obj["samples_ns"],obj["measured_missing"]) + if rows: + with open(output/"summary.csv","w",newline="") as stream: + writer=csv.DictWriter(stream,fieldnames=list(rows[0]));writer.writeheader();writer.writerows(rows) + resource_rows=[] + for path in sorted(output.glob("pair-*/*/resources.csv")): + groups={} + with open(path,newline="") as stream: + for row in csv.DictReader(stream):groups.setdefault(row["phase"],[]).append(row) + for phase,samples in groups.items(): + def numbers(key):return [int(r[key]) for r in samples if str(r.get(key,"")).isdigit()] + resource_rows.append(dict(pair=path.parts[-3],backend=path.parts[-2],phase=phase,samples=len(samples),observed_cgroup_memory_peak_bytes=max(numbers("cgroup_memory_bytes"),default=None),observed_pss_peak_bytes=max(numbers("pss_bytes"),default=None),observed_storage_peak_bytes=max(numbers("storage_bytes"),default=None),observed_wal_peak_bytes=max(numbers("wal_bytes"),default=None),minimum_filesystem_free_bytes=min(numbers("filesystem_free_bytes"),default=None))) + if resource_rows: + with open(output/"summary-resources.csv","w",newline="") as stream: + writer=csv.DictWriter(stream,fieldnames=list(resource_rows[0]));writer.writeheader();writer.writerows(resource_rows) + paired=[] + keys=sorted({(r["scenario"],r["event_class"]) for r in rows}) + for scenario,name in keys: + for metric in ("p50_ns","p95_ns","p99_ns","completed_per_second_including_drain"): + ratios=[];absolute=[] + for pair in sorted({r["pair"] for r in rows}): + values={r["backend"]:r[metric] for r in rows if (r["pair"],r["scenario"],r["event_class"])==(pair,scenario,name)} + if set(values)=={"sqlite","postgres"} and values["sqlite"] not in (None,0) and values["postgres"] is not None: + ratios.append(values["postgres"]/values["sqlite"]);absolute.append(dict(pair=pair,**values)) + if ratios: + paired.append(dict(scenario=scenario,event_class=name,metric=metric,pairs=len(ratios),candidate_over_baseline_median=percentiles(ratios)["p50"],minimum=min(ratios),maximum=max(ratios),absolute_runs=absolute)) + write_json(output/"paired-summary.json",paired) + return rows + + +def build(root, backend, binaries, env, logs): + before = production_hash(root) + replacements = {} + for package in ("ingestor", "server"): + templates = ["common_test.go", "clock_linux_test.go", "clock_other_test.go", package + "_test.go"] + if package == "ingestor": + templates.append(backend + "_adapter_test.go") + for template in templates: + replacements[str(root / "cmd" / package / ("zz_benchmark_" + template))] = str(HERE / "overlays" / template) + overlay = binaries / (backend + "-overlay.json") + write_json(overlay, {"Replace": replacements}) + for package in ("ingestor", "server"): + command(["go", "test", "-overlay", overlay, "-c", "-o", binaries / (backend + "-" + package + ".test"), "."], cwd=root / "cmd" / package, env=env, log=logs / (backend + "-" + package + "-build.log")) + if package == "ingestor": + command([binaries / (backend + "-ingestor.test"), "-test.run=^TestCoreScopeBenchmark.+", "-test.count=1", "-test.timeout=3m"], + cwd=root / "cmd" / package, env=env, log=logs / (backend + "-generator-controls.log")) + command(["go", "build", "-trimpath", "-o", binaries / (backend + "-server"), "."], cwd=root / "cmd/server", env=env, log=logs / (backend + "-server-binary.log")) + if backend == "postgres": + command(["go", "build", "-trimpath", "-o", binaries / "migrate", "."], cwd=root / "cmd/migrate", env=env, log=logs / "migrate-build.log") + if production_hash(root) != before: + raise RuntimeError("build changed production sources") + return before + + +def worker(binary, config, mode, env, log, budget=None): + cfg = dict(config, mode=mode) + path = Path(config["control_dir"]) / (mode + "-config.json") + write_json(path, cfg) + return spawn([binary, "-test.run=^TestCoreScopeBenchmark$", "-test.count=1", "-test.timeout=30m", "-test.v"], cwd=path.parent, env=dict(env, CORESCOPE_BENCH_CONFIG=str(path)), log=log, budget=budget) + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repo", type=Path, default=Path.cwd()) + parser.add_argument("--baseline-sha", default=BASELINE) + parser.add_argument("--candidate-sha", required=True) + parser.add_argument("--corpus", choices=("S", "B", "L"), default="B") + parser.add_argument("--pairs", type=int, default=5) + parser.add_argument("--profile", choices=("primary", "smoke"), default="primary") + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--cgroup", default=os.environ.get("CORESCOPE_BENCH_CGROUP")) + parser.add_argument("--postgres-bin", type=Path, default=Path("/usr/lib/postgresql/18/bin")) + parser.add_argument("--postgres-version", default="18.6") + parser.add_argument("--go-version", default="1.27.2") + parser.add_argument("--seed", type=int, default=20261008) + parser.add_argument("--warmup", type=int) + parser.add_argument("--seconds", type=int) + parser.add_argument("--ingest-rate", type=int, default=50) + parser.add_argument("--http-rate", type=int, default=20) + args = parser.parse_args(argv) + check_revisions(args.baseline_sha, args.candidate_sha) + validate_profile(args.profile, args.pairs, args.cgroup) + if sys.version_info < (3,12): + raise ValueError("Python 3.12 or newer is required for safe archive extraction") + if sys.platform != "linux": + raise ValueError("measured runs require one isolated Linux host; use portable unit/overlay checks here") + if os.geteuid() == 0: + raise ValueError("run as an unprivileged account with delegated cgroup permissions; initdb refuses root") + if args.profile == "primary" and (len(os.sched_getaffinity(0)) < 4 or os.sysconf("SC_PHYS_PAGES") * os.sysconf("SC_PAGE_SIZE") < 8 * 1024 ** 3): + raise ValueError("primary requires at least four host CPUs and 8 GiB RAM") + for executable in ("git", "go", "cc"): + if not shutil.which(executable): + raise ValueError("missing prerequisite: " + executable) + go_version = command(["go", "version"]).strip() + if not re.search(r"\bgo" + re.escape(args.go_version) + r"\b", go_version): + raise ValueError("Go version differs from the common toolchain pin") + for executable in ("postgres", "initdb", "psql", "pg_isready"): + if not (args.postgres_bin / executable).is_file(): + raise ValueError("missing PostgreSQL prerequisite: " + executable) + version = command([args.postgres_bin / "postgres", "--version"]).strip() + if not re.search(r"\b" + re.escape(args.postgres_version) + r"\b", version): + raise ValueError("PostgreSQL patch version differs from requested benchmark pin") + output = safe_output(args.output) + minimum = {"S": 4, "B": 12, "L": 128}[args.corpus] * 1024 ** 3 + if shutil.disk_usage(output.parent).free < minimum: + raise ValueError("insufficient free space for the declared corpus safety guard (S:4/B:12/L:128 GiB); these are capacity guards, not measured dataset sizes") + output.mkdir(mode=0o700) + runtime, logs, binaries = output / ".runtime", output / "logs", output / ".runtime/binaries" + for path in (runtime, logs, binaries): + path.mkdir(exist_ok=True) + budget = Budget(args.cgroup, args.profile == "primary") + env = {k:v for k,v in os.environ.items() if not k.startswith(("CORESCOPE_", "PG")) and k not in ("DB_PATH","MQTT_BROKER","MQTT_TOPIC","GOFLAGS")} + env.update(CGO_ENABLED="1",GOMAXPROCS="3",GOMEMLIMIT="512MiB",GOTOOLCHAIN="local",GOFLAGS="") + pg, server = None, None + manifest = dict(status="incomplete", baseline_sha=BASELINE, candidate_sha=args.candidate_sha, seed=args.seed, corpus=args.corpus, shape=corpus(args.corpus), firmware_reference=FIRMWARE, harness_sha256=hashlib.sha256(b"".join(bytes.fromhex(sha256(p)) for p in sorted(HERE.rglob("*")) if p.is_file() and "__pycache__" not in p.parts)).hexdigest(), host=dict(system=platform.system(), kernel=platform.release(), architecture=platform.machine(), cpus=os.cpu_count(), ram_bytes=os.sysconf("SC_PHYS_PAGES") * os.sysconf("SC_PAGE_SIZE")), postgres_version=version, go_version=command(["go", "version"]).strip(), compiler=command(["cc", "--version"]).splitlines()[0], resource_budget=budget.info, cache_policy=dict(startup="new application process after full validation; OS cache warm", mixed="warmup then application-warm measurement", sql="channel application cache explicitly cleared; OS cache warm"), os_cold="not measured", comparison_eligible=False, extended_120s_600s_profile="not run unless explicitly requested by duration flags", order=pair_order(args.pairs)) + write_json(output / "manifest.json", manifest) + try: + try: + budget.preflight() + except OSError as error: + detail = record_resource_failure(output / "resource-preflight.json", error, status="failed") + raise RuntimeError("resource preflight failed: " + json.dumps(detail, sort_keys=True)) from error + write_json(output / "resource-preflight.json", dict(status="passed", enforced=bool(budget.path), counters=list(RESOURCE_COUNTERS) if budget.path else [])) + try: + pg = Postgres(runtime, args.postgres_bin, budget, logs) + pg.start() + pg.stop() + except BaseException as error: + write_json(output / "postgres-preflight.json", dict(status="failed", error_type=type(error).__name__)) + raise + write_json(output / "postgres-preflight.json", dict(status="passed", transport="loopback_tcp", readiness="SELECT 1", stopped=True)) + roots = {} + for backend, revision in (("sqlite", BASELINE), ("postgres", args.candidate_sha)): + root = runtime / backend + archive(args.repo.resolve(), revision, root) + roots[backend] = root + manifest[backend + "_production_hash_before"] = build(root, backend, binaries, env, logs) + epoch = int(time.time()) + canonical = runtime / "corpus.sqlite" + prepare_dir = output / "corpus" + prepare_dir.mkdir() + base_config = dict(control_dir=str(runtime/"prepare-controls"), corpus_info=str(prepare_dir / "corpus.json"), corpus=args.corpus, shape=corpus(args.corpus), seed=args.seed, epoch=epoch, sqlite=str(canonical), output=str(prepare_dir), warmup=0, seconds=0, ingest_rate=args.ingest_rate, http_rate=args.http_rate) + finish(worker(binaries / "sqlite-ingestor.test", base_config, "prepare", env, logs / "prepare.log"), timeout=1800) + tables = table_manifest(canonical) + canonical_validation = validate(canonical, tables) + if canonical_validation["transmissions"]["rows"] != corpus(args.corpus)["transmissions"] or canonical_validation["observations"]["rows"] != corpus(args.corpus)["observations"]: + raise RuntimeError("canonical corpus counts differ from the declared shape") + write_json(prepare_dir / "validation.json", canonical_validation) + manifest.update(epoch=epoch, canonical_file_sha256=sha256(canonical), table_manifest=tables) + manifest["binary_sha256"] = {p.name: sha256(p) for p in binaries.iterdir() if p.is_file() and not p.name.endswith(".json")} + warmup = args.warmup if args.warmup is not None else (60 if args.profile == "primary" else 2) + seconds = args.seconds if args.seconds is not None else (180 if args.profile == "primary" else 10) + if warmup < 0 or seconds < 1 or args.ingest_rate < 1 or args.http_rate < 1 or (warmup+seconds)*max(args.ingest_rate,args.http_rate)>1_000_000: + raise ValueError("invalid workload duration or offered rate") + manifest["workload"] = dict(warmup_seconds=warmup, measured_seconds=seconds, ingest_rate=args.ingest_rate, http_rate=args.http_rate, queue_limit=1024, http_concurrency=64, server_memory_mib=SERVER_MEMORY_MIB, ingestor_memory_mib=512, packet_store_mib=PACKET_STORE_MIB, retention_hours=168, sql_reader_pool=4, sql_writer_pool=1) + write_json(output / "manifest.json", manifest) + after_by_pair = {} + paired_sources = {} + for pair, backend in pair_order(args.pairs): + budget.reset() + run_dir = output / f"pair-{pair:02d}" / backend + run_dir.mkdir(parents=True) + if shutil.disk_usage(output).free < max(2 * 1024 ** 3, canonical.stat().st_size * 8): + raise RuntimeError("insufficient measured-corpus headroom before the next restore/import") + config_dir = runtime / f"config-{pair:02d}-{backend}" + config_dir.mkdir() + state_dir = config_dir / "state" + state_dir.mkdir() + sqlite_path = config_dir / "telemetry.sqlite" + if pair not in paired_sources: + pair_source=runtime/f"pair-{pair:02d}.sqlite" + shutil.copy2(canonical,pair_source) + delta=int(time.time())-epoch + freshen(pair_source,delta) + paired_validation=validate(pair_source,tables) + paired_sources[pair]=(pair_source,delta,paired_validation) + write_json(output/f"pair-{pair:02d}"/"corpus-offset.json",dict(receipt_epoch_offset_seconds=delta,wire_epoch_unchanged=epoch,sha256=sha256(pair_source),validation=paired_validation)) + pair_source,delta,paired_validation=paired_sources[pair] + command(["cp","--reflink=never",pair_source,sqlite_path],budget=budget,log=run_dir/"restore-copy.log") + config = dict(base_config, control_dir=str(config_dir/"controls"), events_file=str(runtime/f"pair-{pair:02d}-events.jsonl"), epoch=epoch+delta, wire_epoch=epoch, output=str(run_dir), sqlite=str(sqlite_path), state_dir=str(state_dir), warmup=warmup, seconds=seconds, start_file=str(run_dir / "start.json")) + if not Path(config["events_file"]).exists(): + finish(worker(binaries/"sqlite-ingestor.test",config,"events",env,run_dir/"event-generation.log"),timeout=600) + write_json(output/f"pair-{pair:02d}"/"events.json",dict(sha256=sha256(config["events_file"]),rows=(warmup+seconds)*args.ingest_rate,source="one identical baseline-generated stream for both engines")) + event_hash = json.loads((output/f"pair-{pair:02d}"/"events.json").read_text())["sha256"] + if sha256(config["events_file"]) != event_hash: + raise RuntimeError("paired event stream changed before replay") + run_env = dict(env,CORESCOPE_INGESTOR_STATS=str(state_dir/"ingestor-stats.json")) + database = f"bench_p{pair:02d}" + with Resources(run_dir / "resources.csv", budget, [pg.data] if backend == "postgres" else [sqlite_path]) as resources: + if backend == "postgres": + pg.start() + urls = pg.create(database) + run_env.update(CORESCOPE_DATABASE_URL=urls["writer"], CORESCOPE_WRITER_DATABASE_URL=urls["writer"], CORESCOPE_READER_DATABASE_URL=urls["reader"]) + before_source = sha256(sqlite_path) + resources.phase="migration" + migration_start = time.monotonic_ns() + command([binaries / "migrate", "-offline", "-from-sqlite", sqlite_path, "-state-dir", config_dir / "migration"], env=dict(env, CORESCOPE_DATABASE_URL=urls["owner"]), log=run_dir / "migration.log", timeout=1800, budget=budget) + tool_finished=time.monotonic_ns() + pg.grant(database) + grants_finished=time.monotonic_ns() + migrated = validate(sqlite_path, tables, pg, database) + valid = migrated == paired_validation and sha256(sqlite_path) == before_source + write_json(run_dir / "migration.json", dict(elapsed_ns=time.monotonic_ns()-migration_start, migration_tool_ns=tool_finished-migration_start, role_grants_ns=grants_finished-tool_finished, harness_validation_ns=time.monotonic_ns()-grants_finished, verified=valid, source_unchanged=sha256(sqlite_path) == before_source, tables=migrated)) + if not valid: + raise RuntimeError("migration parity/source immutability failed") + else: + with sqlite3.connect(sqlite_path) as db: + db.execute("ANALYZE") + pre = validate(sqlite_path, tables, pg if backend == "postgres" else None, database) + if pre != paired_validation: + raise RuntimeError("initial backend state differs from the canonical corpus") + write_json(run_dir / "validation-before.json", pre) + retained_expected = retained_counts(sqlite_path, int(time.time())-168*3600) if args.corpus in ("S", "B") else None + app_config, server_env = server_settings(backend, sqlite_path, state_dir, run_env) + write_json(config_dir / "config.json", app_config) + if backend == "postgres": + record_postgres_selection(binaries, config_dir, state_dir, run_env, urls["owner"], run_dir / "selection.log", budget) + resources.phase="startup_full" + launched_ns=time.monotonic_ns() + server = spawn([binaries / (backend + "-server"), "-config-dir", config_dir, "-public", roots[backend] / "public", "-port", app_config["port"], "-poll-ms", 1000], cwd=roots[backend], env=server_env, log=run_dir / "server.log", budget=budget) + config["base_url"] = f"http://127.0.0.1:{app_config['port']}" + startup(server,config["base_url"],run_dir/"startup.json",pre,launched_ns,0,retained_expected) + stop(server) + app_config["packetStore"]["hotStartupHours"]=1 + write_json(config_dir/"config.json",app_config) + resources.phase="startup_hot" + launched_ns=time.monotonic_ns() + server=spawn([binaries/(backend+"-server"),"-config-dir",config_dir,"-public",roots[backend]/"public","-port",app_config["port"],"-poll-ms",1000],cwd=roots[backend],env=server_env,log=run_dir/"server-hot.log",budget=budget) + startup(server,config["base_url"],run_dir/"startup-hot1.json",pre,launched_ns,1,retained_expected) + resources.phase="sql_queries" + finish(worker(binaries / (backend + "-server.test"), config, "queries", server_env, run_dir / "queries.log", budget), timeout=180) + resources.phase="mixed_with_retention" + client = worker(binaries / (backend + "-server.test"), config, "client", server_env, run_dir / "client.log") + replay = worker(binaries / (backend + "-ingestor.test"), config, "replay", run_env, run_dir / "replay.log", budget) + try: + wait_file(run_dir / "client-ready", [client, replay, server]) + wait_file(run_dir / "replay-ready", [client, replay, server]) + write_json(run_dir / "start.json", dict(mono_ns=time.monotonic_ns() + 2_000_000_000)) + finish(replay, timeout=warmup + seconds + 180) + finish(client, timeout=warmup + seconds + 180) + finally: + stop(replay) + stop(client) + observed=visibility(json_lines(run_dir/"ingest.jsonl"),json_lines(run_dir/"websocket.jsonl")) + write_json(run_dir/"visibility.json",observed) + if not observed["verified"]: + raise RuntimeError("durable new-transmission sentinels were not observed on the real WebSocket") + resources.phase="retention_empty" + finish(worker(binaries / (backend + "-ingestor.test"), config, "retention", run_env, run_dir / "retention.log", budget), timeout=600) + stop(server) + server = None + resources.phase="plans" + finish(worker(binaries/(backend+"-ingestor.test"),config,"plans",run_env,run_dir/"plans.log",budget),timeout=180) + resources.phase="post_validation" + post = validate(sqlite_path, tables, pg if backend == "postgres" else None, database, logical=True) + write_json(run_dir / "validation-after.json", post) + # Separate real-handler control follows parity, so its deliberate + # receipt-time timestamps do not weaken the deterministic replay digest. + resources.phase="handler_control" + finish(worker(binaries / (backend + "-ingestor.test"), config, "handler", run_env, run_dir / "handler.log", budget), timeout=180) + after_by_pair.setdefault(pair, {})[backend] = post + if len(after_by_pair[pair])==2: + coverage={} + for kind in ("sqlite","postgres"): + coverage[kind]=[json.loads((output/f"pair-{pair:02d}"/kind/file).read_text())["health"] for file in ("startup.json","startup-hot1.json")] + for position in (0,1): + for field in ("loadedTx","loadedObs"): + if coverage["sqlite"][position][field]!=coverage["postgres"][position][field]: + raise RuntimeError("startup memory coverage differs under equal retention/memory settings") + if len(after_by_pair[pair]) == 2 and after_by_pair[pair]["sqlite"] != after_by_pair[pair]["postgres"]: + raise RuntimeError("post-replay/retention logical parity failed; retain both validation artifacts") + if backend == "postgres": + pg.drop(database) + pg.stop() + remove_working_directory(config_dir,runtime) + if sha256(config["events_file"]) != event_hash: + raise RuntimeError("paired event stream changed during replay") + if len(after_by_pair[pair])==2: + pair_source.unlink() + Path(config["events_file"]).unlink() + if sha256(canonical) != manifest["canonical_file_sha256"]: + raise RuntimeError("canonical corpus was modified") + for backend, root in roots.items(): + after = production_hash(root) + manifest[backend + "_production_hash_after"] = after + if after != manifest[backend + "_production_hash_before"]: + raise RuntimeError("production source changed during measurements") + summary = summarize(output) + manifest.update(status="completed", comparison_eligible=args.profile=="primary", measured_rows=len(summary), completed_pairs=args.pairs) + write_json(output / "manifest.json", manifest) + (output / "report.md").write_text("# CoreScope paired database measurements\n\nResults are scoped to this exact manifest, corpus, host, cache state and conversion.\nSee summary.csv and the raw per-run artifacts. p99 is omitted below 1,000 samples.\nOS-cold behavior and the optional 65-minute maintenance soak were not measured.\nNo engine-wide speed claim follows from these results.\n") + except BaseException as error: + for child in reversed(PROCESSES): + stop(child) + try: + summarize(output) + except (OSError,ValueError,KeyError) as summary_error: + manifest["partial_summary_error"]=str(summary_error) + manifest.update(status="failed", comparison_eligible=False, error=str(error)) + write_json(output / "manifest.json", manifest) + raise + finally: + for child in reversed(PROCESSES): + stop(child) + PROCESSES.clear() + stop(server) + if pg: + pg.stop() + try: + budget.close() + except Exception as cleanup_error: + manifest.update(status="failed",comparison_eligible=False,cleanup_error=str(cleanup_error)) + write_json(output/"manifest.json",manifest) + raise + finally: + public_bundle(output,args.repo) + + +if __name__ == "__main__": + try: + main() + except (ValueError, RuntimeError, OSError, subprocess.SubprocessError) as error: + print("benchmark:", error, file=sys.stderr) + sys.exit(1) diff --git a/scripts/postgres-benchmark/test_harness.py b/scripts/postgres-benchmark/test_harness.py new file mode 100644 index 000000000..43179eba2 --- /dev/null +++ b/scripts/postgres-benchmark/test_harness.py @@ -0,0 +1,556 @@ +import importlib.util +import csv +import errno +import io +import json +import pathlib +import re +import tempfile +import types +import unittest +from unittest import mock + +HERE = pathlib.Path(__file__).resolve().parent +spec = importlib.util.spec_from_file_location("bench", HERE / "run.py") +bench = importlib.util.module_from_spec(spec) +spec.loader.exec_module(bench) + + +class MemoryPeakFile(io.BytesIO): + """Model cgroup v2's reset scope: one open file description.""" + def __init__(self, state, mode): + super().__init__() + self.state, self.binary, self.peak = state, "b" in mode, state.lifetime + state.handles.append(self) + + def write(self, value): + if self.state.reject_reset: + raise OSError(errno.EINVAL, "reset unsupported") + self.peak = self.state.current + return len(value) + + def read(self, size=-1): + value = b"" if self.tell() else str(self.peak).encode() + b"\n" + self.seek(len(value), io.SEEK_CUR) + return value if self.binary else value.decode() + + +class HarnessTests(unittest.TestCase): + def test_selected_postgres_setup_is_offline_and_keeps_owner_out_of_argv(self): + owner = "postgres://bench_owner:private@127.0.0.1:25432/bench_p00" + runtime = {"CORESCOPE_WRITER_DATABASE_URL": "writer", "CORESCOPE_READER_DATABASE_URL": "reader"} + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + with mock.patch.object(bench, "command") as command: + bench.record_postgres_selection(root / "bin", root, root / "state", runtime, owner, root / "setup.log", None) + args = [str(value) for value in command.call_args.args[0]] + self.assertIn("-storage-action=adopt", args) + self.assertIn("-backend=postgres", args) + self.assertIn("-offline", args) + self.assertIn(str(root / "state" / "storage-selection.json"), args) + self.assertNotIn(owner, " ".join(args)) + self.assertEqual(command.call_args.kwargs["env"]["CORESCOPE_DATABASE_URL"], owner) + self.assertEqual(command.call_args.kwargs["env"]["CORESCOPE_WRITER_DATABASE_URL"], "writer") + self.assertNotIn("CORESCOPE_DATABASE_URL", runtime) + with mock.patch.object(bench, "command", side_effect=RuntimeError("selection refused")): + with self.assertRaisesRegex(RuntimeError, "selection refused"): + bench.record_postgres_selection(root / "bin", root, root / "state", runtime, owner, root / "setup.log", None) + + def test_default_common_rate_and_emitted_worker_configs(self): + parse = bench.argparse.ArgumentParser.parse_args + for extra, expected_rate in (([], 50), (["--ingest-rate", "100"], 100)): + parsed = [] + def capture(parser, argv): + parsed.append(parse(parser, argv)) + raise RuntimeError("stop after parsing") + with mock.patch.object(bench.argparse.ArgumentParser, "parse_args", capture): + with self.assertRaisesRegex(RuntimeError, "stop after parsing"): + bench.main(["--candidate-sha", "1" * 40, "--output", "corescope-bench-test", *extra]) + args = parsed[0] + self.assertEqual((args.ingest_rate, args.http_rate, args.pairs), (expected_rate, 20, 5)) + self.assertEqual((args.warmup, args.seconds), (None, None)) + # Exercise the actual JSON controls consumed by both compiled workers. + emitted = [] + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + for backend in ("sqlite", "postgres"): + controls = root / backend + controls.mkdir() + config = dict(control_dir=str(controls), warmup=60, seconds=180, + ingest_rate=args.ingest_rate, http_rate=args.http_rate) + with mock.patch.object(bench, "spawn") as spawn: + bench.worker(root / (backend + "-ingestor.test"), config, "replay", {}, root / (backend + ".log")) + written = json.loads(pathlib.Path(spawn.call_args.kwargs["env"]["CORESCOPE_BENCH_CONFIG"]).read_text()) + written.pop("control_dir") + emitted.append(written) + self.assertEqual(emitted[0], emitted[1]) + self.assertEqual(emitted[0], dict(mode="replay", warmup=60, seconds=180, + ingest_rate=expected_rate, http_rate=20)) + self.assertEqual((emitted[0]["warmup"] + emitted[0]["seconds"]) * emitted[0]["ingest_rate"], + 12000 if expected_rate == 50 else 24000) + + def test_workflow_declares_the_common_supplemental_rate(self): + workflow = (HERE.parent.parent / ".github/workflows/postgres-benchmark.yml").read_text() + commands = re.findall(r"(?m)^\s*python3 scripts/postgres-benchmark/run\.py (.+)", workflow.replace("\\\n", " ")) + self.assertEqual(len(commands), 2) + for command in commands: + self.assertIn("--ingest-rate 50", command) + self.assertIn("--http-rate 20", command) + self.assertIn("supplemental", workflow.lower()) + + def test_common_server_memory_profile_retains_b_and_replay_headroom(self): + # Native baseline B qualification tracked 1221.7 MiB. The original + # 100/s envelope conservatively bounds the current 50/s experiment too. + required_store_mib = 1221.7 + 24000 * 8279 / 1048576 + settings = [] + for backend in ("sqlite", "postgres"): + with mock.patch.object(bench, "free_port", return_value=12345): + config, env = bench.server_settings(backend, pathlib.Path("corpus.sqlite"), pathlib.Path("state"), {}) + self.assertGreater(config["packetStore"]["maxMemoryMB"], required_store_mib) + self.assertEqual(config["packetStore"], {"retentionHours": 168, "maxMemoryMB": 2048, "hotStartupHours": 0}) + self.assertEqual(config["runtime"]["maxMemoryMB"], 3072) + self.assertEqual(env["GOMEMLIMIT"], "3072MiB") + self.assertEqual("dbPath" in config, backend == "sqlite") + config.pop("dbPath", None) + settings.append((config, env)) + self.assertEqual(settings[0], settings[1], "capacity settings must be common to both engines") + + def test_build_runs_compiled_generator_controls(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + binaries, logs = root / "binaries", root / "logs" + binaries.mkdir(); logs.mkdir() + with mock.patch.object(bench, "production_hash", return_value="stable"), mock.patch.object(bench, "command") as command: + bench.build(root, "sqlite", binaries, {}, logs) + controls = [call.args[0] for call in command.call_args_list if pathlib.Path(call.args[0][0]).name == "sqlite-ingestor.test"] + self.assertEqual(len(controls), 1, "new Go generator regressions must execute, not only compile") + self.assertIn("-test.run=^TestCoreScopeBenchmark.+", controls[0]) + + def resource_fixture(self, root, missing=None): + group = root / "private-cgroup-path" + group.mkdir() + for name, value in {"memory.current": "0", "memory.events": "oom 0\n", "cpu.stat": "usage_usec 0\n", + "io.stat": "8:0 rbytes=1024 wbytes=2048\n", "cgroup.procs": ""}.items(): + if name != missing: + (group / name).write_text(value) + budget = bench.Budget(None, False) + budget.path = group + budget.info = {"enforced": True, "memory_peak_reset": False} + budget.close = mock.Mock() + run = root / "pair-00" / "sqlite" + run.mkdir(parents=True) + return budget, run + + def memory_peak_counter(self, budget, reject_reset=False): + (budget.path / "memory.reclaim").touch() + state = types.SimpleNamespace(current=0, lifetime=9000, handles=[], reject_reset=reject_reset) + real_open = pathlib.Path.open + def open_file(path, mode="r", *args, **kwargs): + if path == budget.path / "memory.peak": + return MemoryPeakFile(state, mode) + return real_open(path, mode, *args, **kwargs) + return state, mock.patch.object(pathlib.Path, "open", open_file) + + def test_memory_peak_is_reset_per_leg_for_samples_and_final_report(self): + with tempfile.TemporaryDirectory() as directory: + budget, run = self.resource_fixture(pathlib.Path(directory)) + state, counter = self.memory_peak_counter(budget) + def sample_peak(value): + state.current = value + state.lifetime = max(state.lifetime, value) + for handle in state.handles: + if not handle.closed: + handle.peak = max(handle.peak, value) + resource = bench.Resources(run / "resources.csv", budget, []) + with mock.patch.object(resource.done, "wait", side_effect=lambda _: resource.done.set()): + resource.sample() + self.assertIsNone(resource.error) + with open(run / "resources.csv", newline="") as stream: + row = next(csv.DictReader(stream)) + # Use the real final-write path after the sampler has stopped. + with mock.patch.object(resource.thread, "join"): + resource.__exit__(None, None, None) + final = json.loads((run / "resource-final.json").read_text()) + self.assertTrue(final["budget"]["memory_peak_reset"]) + self.assertEqual(int(row["cgroup_memory_peak_bytes"]), value) + self.assertEqual(final["memory_peak_bytes"], value) + self.assertTrue(all(handle.closed for handle in state.handles)) + with counter: + budget.reset() + sample_peak(3000) + state.current = 0 + budget.reset() + sample_peak(1000) + + def test_memory_peak_descriptor_closes_on_failure_and_unsupported_reset(self): + for unsupported in (False, True): + with self.subTest(unsupported=unsupported), tempfile.TemporaryDirectory() as directory: + budget, run = self.resource_fixture(pathlib.Path(directory)) + state, counter = self.memory_peak_counter(budget, reject_reset=unsupported) + with counter: + budget.reset() + self.assertEqual(budget.info["memory_peak_reset"], not unsupported) + with self.assertRaisesRegex(ValueError, "workload failure"): + with bench.Resources(run / "resources.csv", budget, []): + raise ValueError("workload failure") + self.assertTrue(all(handle.closed for handle in state.handles)) + final = json.loads((run / "resource-final.json").read_text()) + self.assertEqual(final["memory_peak_bytes"], None if unsupported else 0) + + def test_memory_peak_descriptor_closes_if_no_resource_context_is_entered(self): + with tempfile.TemporaryDirectory() as directory: + budget, _ = self.resource_fixture(pathlib.Path(directory)) + state, counter = self.memory_peak_counter(budget) + with counter, mock.patch.object(pathlib.Path, "rmdir"): + budget.reset() + first = state.handles[0] + budget.reset() + self.assertTrue(first.closed) + bench.Budget.close(budget) + self.assertTrue(all(handle.closed for handle in state.handles)) + + def test_sampler_failure_identifies_counter_without_private_path(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + budget, run = self.resource_fixture(root, missing="io.stat") + with self.assertRaisesRegex(RuntimeError, "io.stat"): + with bench.Resources(run / "resources.csv", budget, []) as resource: + resource.thread.join(timeout=2) + evidence = json.loads((run / "resource-error.json").read_text()) + self.assertEqual(evidence["error_type"], "FileNotFoundError") + self.assertEqual(evidence["errno"], errno.ENOENT) + self.assertEqual(evidence["counter"], "io.stat") + self.assertNotIn("private-cgroup-path", json.dumps(evidence)) + bench.public_bundle(root) + self.assertTrue((root / "public/pair-00/sqlite/resource-error.json").is_file()) + + def test_sampler_failure_does_not_replace_workload_exception(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + budget, run = self.resource_fixture(root, missing="io.stat") + with self.assertRaisesRegex(ValueError, "independent workload failure"): + with bench.Resources(run / "resources.csv", budget, []) as resource: + resource.thread.join(timeout=2) + raise ValueError("independent workload failure") + self.assertEqual(json.loads((run / "resource-error.json").read_text())["counter"], "io.stat") + + def test_successful_resource_sample_keeps_io_metrics(self): + with tempfile.TemporaryDirectory() as directory: + budget, run = self.resource_fixture(pathlib.Path(directory)) + resource = bench.Resources(run / "resources.csv", budget, []) + with mock.patch.object(resource.done, "wait", side_effect=lambda _: resource.done.set()): + resource.sample() + self.assertIsNone(resource.error) + with open(run / "resources.csv", newline="") as stream: + rows = list(csv.DictReader(stream)) + self.assertEqual(len(rows), 1) + self.assertEqual(rows[0]["io_stat"], "8:0 rbytes=1024 wbytes=2048") + + def test_final_resource_write_failure_does_not_replace_workload_exception(self): + with tempfile.TemporaryDirectory() as directory: + budget, run = self.resource_fixture(pathlib.Path(directory)) + with mock.patch.object(bench, "write_json", side_effect=PermissionError(errno.EACCES, "private detail", str(run / "resource-final.json"))): + with self.assertRaisesRegex(ValueError, "independent workload failure"): + with bench.Resources(run / "resources.csv", budget, []): + raise ValueError("independent workload failure") + + def test_missing_counter_fails_before_archiving_or_building(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + budget, _ = self.resource_fixture(root, missing="io.stat") + postgres_bin = root / "pg-bin" + postgres_bin.mkdir() + for tool in ("postgres", "initdb", "psql", "pg_isready"): + (postgres_bin / tool).touch() + output = root / "corescope-bench-preflight" + def version(args, **kwargs): + return "go version go1.27.2" if args[0] == "go" else "PostgreSQL 18.6" + with mock.patch.object(bench.sys, "platform", "linux"), \ + mock.patch.object(bench.os, "geteuid", return_value=1000, create=True), \ + mock.patch.object(bench.os, "sysconf", return_value=4096, create=True), \ + mock.patch.object(bench.shutil, "which", return_value="tool"), \ + mock.patch.object(bench.shutil, "disk_usage", return_value=types.SimpleNamespace(free=32 * 1024**3)), \ + mock.patch.object(bench, "command", side_effect=version), \ + mock.patch.object(bench, "Budget", return_value=budget), \ + mock.patch.object(bench, "archive") as archive, \ + mock.patch.object(bench, "build", side_effect=AssertionError("must preflight before building")) as build: + with self.assertRaisesRegex(RuntimeError, "io.stat"): + bench.main(["--candidate-sha", "1" * 40, "--profile", "smoke", "--pairs", "1", "--corpus", "S", + "--output", str(output), "--postgres-bin", str(postgres_bin)]) + archive.assert_not_called() + build.assert_not_called() + evidence = json.loads((output / "public/resource-preflight.json").read_text()) + self.assertEqual(evidence["counter"], "io.stat") + self.assertFalse(json.loads((output / "public/manifest.json").read_text())["comparison_eligible"]) + + def test_workflow_delegates_every_required_resource_controller(self): + workflow = (HERE.parents[1] / ".github/workflows/postgres-benchmark.yml").read_text() + command = next(line for line in workflow.splitlines() if "printf '+" in line and '"$delegated/cgroup.subtree_control"' in line) + self.assertTrue({"cpu", "memory", "io"}.issubset(set(re.findall(r"\+([a-z]+)", command)))) + + def test_private_postgres_launch_uses_only_loopback_tcp_and_durable_settings(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + pg = bench.Postgres.__new__(bench.Postgres) + pg.root, pg.bin, pg.data, pg.logs = root, root / "bin", root / "pgdata", root + pg.port, pg.env, pg.budget = 25433, {"PGHOST": "127.0.0.1"}, object() + pg.sql = mock.Mock(return_value="1\n") + with mock.patch.object(bench, "spawn", return_value=types.SimpleNamespace(poll=lambda: None)) as spawn, \ + mock.patch.object(bench.subprocess, "run", return_value=types.SimpleNamespace(returncode=0)): + pg.start() + args = spawn.call_args.args[0] + settings = dict(args[i+1].split("=", 1) for i, arg in enumerate(args[:-1]) if arg == "-c") + self.assertEqual(settings.get("unix_socket_directories"), "") + self.assertEqual(args[args.index("-h")+1], "127.0.0.1") + for setting in ("fsync", "synchronous_commit", "full_page_writes"): + self.assertEqual(settings[setting], "on") + self.assertEqual(settings["shared_buffers"], "256MB") + self.assertEqual(settings["max_connections"], "32") + self.assertEqual(settings.get("log_error_verbosity"), "verbose") + pg.sql.assert_called_once_with("SELECT 1") + + def test_postgres_timestamped_failure_keeps_only_allowlisted_diagnostics(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + logs = root / "logs" + logs.mkdir() + path = logs / "postgres.log" + path.write_text('2026-10-09 05:08:18.001 UTC [123] FATAL: 42501: could not create lock file "/private/socket/.s.PGSQL.5432.lock": Permission denied\n' + '2026-10-09 05:08:18.002 UTC [123] DETAIL: postgres://private:secret@private.example/data\n' + '2026-10-09 05:08:18.003 UTC [123] FATAL: arbitrary private body\n') + result = bench.record_failure(path, 1) + self.assertIn("42501", json.dumps(result)) + self.assertEqual(result["postgres_errors"][0], {"severity": "FATAL", "sqlstate": "42501", "kind": "unix_socket_lock_permission_denied"}) + for private in ("/private", "secret", "private.example", "arbitrary private body"): + self.assertNotIn(private, json.dumps(result)) + bench.public_bundle(root) + self.assertTrue((root / "public/logs/failure-postgres.json").is_file()) + + def test_failed_postgres_launch_precedes_archiving_and_building(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + budget, _ = self.resource_fixture(root) + postgres_bin = root / "pg-bin" + postgres_bin.mkdir() + for tool in ("postgres", "initdb", "psql", "pg_isready"): + (postgres_bin / tool).touch() + output = root / "corescope-bench-pg-preflight" + pg = mock.Mock() + pg.start.side_effect = RuntimeError("private PostgreSQL failed to start") + def version(args, **kwargs): + return "go version go1.27.2" if args[0] == "go" else "PostgreSQL 18.6" + with mock.patch.object(bench.sys, "platform", "linux"), \ + mock.patch.object(bench.os, "geteuid", return_value=1000, create=True), \ + mock.patch.object(bench.os, "sysconf", return_value=4096, create=True), \ + mock.patch.object(bench.shutil, "which", return_value="tool"), \ + mock.patch.object(bench.shutil, "disk_usage", return_value=types.SimpleNamespace(free=32 * 1024**3)), \ + mock.patch.object(bench, "command", side_effect=version), \ + mock.patch.object(bench, "Budget", return_value=budget), \ + mock.patch.object(bench, "Postgres", return_value=pg), \ + mock.patch.object(bench, "archive") as archive, \ + mock.patch.object(bench, "build", side_effect=AssertionError("must launch PostgreSQL before building")) as build: + with self.assertRaisesRegex(RuntimeError, "private PostgreSQL failed"): + bench.main(["--candidate-sha", "1" * 40, "--profile", "smoke", "--pairs", "1", "--corpus", "S", + "--output", str(output), "--postgres-bin", str(postgres_bin)]) + archive.assert_not_called() + build.assert_not_called() + pg.start.assert_called_once() + pg.stop.assert_called() + self.assertEqual(json.loads((output / "public/postgres-preflight.json").read_text())["status"], "failed") + + def test_corpus_counts_are_exact(self): + for name, transmissions, observations in [("S", 30_000, 90_000), ("B", 128_000, 2_048_000), ("L", 1_000_000, 16_000_000)]: + shape = bench.corpus(name) + self.assertEqual(shape["transmissions"], transmissions) + self.assertEqual(sum(bench.fanout(name, i) for i in range(transmissions)), observations) + + def test_pair_order_is_serial_and_alternates(self): + self.assertEqual(bench.pair_order(3), [(0, "sqlite"), (0, "postgres"), (1, "postgres"), (1, "sqlite"), (2, "sqlite"), (2, "postgres")]) + + def test_sparse_p99_is_not_reported_as_zero(self): + self.assertIsNone(bench.percentiles([1, 2, 3])["p99"]) + self.assertIsNone(bench.percentiles([])["p50"]) + self.assertEqual(bench.percentiles(range(1, 1001))["p99"], 990) + + def test_logical_digest_distinguishes_null_and_empty_and_normalizes_numbers(self): + self.assertNotEqual(bench.canonical_row([None]), bench.canonical_row([""])) + self.assertEqual(bench.canonical_row([1]), bench.canonical_row([1.0])) + self.assertNotEqual(bench.canonical_row(["1"]), bench.canonical_row([1])) + + def test_destinations_fail_closed(self): + with self.assertRaises(ValueError): + bench.safe_output(pathlib.Path("/tmp")) + with tempfile.TemporaryDirectory() as directory: + target = pathlib.Path(directory) / "corescope-bench-fixture" + self.assertEqual(bench.safe_output(target), target.resolve()) + target.mkdir() + with self.assertRaises(ValueError): + bench.safe_output(target) + + def test_floating_or_wrong_baseline_is_rejected(self): + with self.assertRaises(ValueError): + bench.check_revisions("master", "1" * 40) + with self.assertRaises(ValueError): + bench.check_revisions("2" * 40, "1" * 40) + bench.check_revisions(bench.BASELINE, "1" * 40) + + def test_primary_requires_five_pairs_and_resource_controls(self): + with self.assertRaises(ValueError): + bench.validate_profile("primary", 1, None) + with self.assertRaises(ValueError): + bench.validate_profile("primary", 5, None) + bench.validate_profile("smoke", 1, None) + + def test_epoch_shift_preserves_nanoseconds_and_nulls(self): + self.assertEqual(bench.shift_timestamp("2026-10-08T12:00:00.123456789Z", 3600), "2026-10-08T13:00:00.123456789Z") + self.assertEqual(bench.shift_timestamp("2026-10-08 12:00:00", -1), "2026-10-08 11:59:59") + self.assertIsNone(bench.shift_timestamp(None, 1)) + with self.assertRaises(ValueError): + bench.shift_timestamp("unexpected", 1) + + def test_public_bundle_excludes_databases_payloads_and_controls(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + (root / ".runtime").mkdir() + (root / ".runtime" / "events.jsonl").write_text("private packet payload") + (root / "manifest.json").write_text('{"path":"' + str(root).replace('\\', '\\\\') + '"}') + run = root / "pair-00" / "sqlite" + run.mkdir(parents=True) + (run / "replay-config.json").write_text("private control") + (run / "requests.jsonl").write_text('{"class":"nodes"}\n') + bench.public_bundle(root) + files = {p.relative_to(root / "public").as_posix() for p in (root / "public").rglob("*") if p.is_file()} + self.assertEqual(files, {"manifest.json", "pair-00/sqlite/requests.jsonl"}) + + def test_visibility_is_bound_to_durable_sent_hashes(self): + sent = [{"hash":"a", "completed_ns":100, "new_transmission":True, "measured":True}, {"hash":"b", "completed_ns":200, "new_transmission":True, "measured":True}] + result = bench.visibility(sent,[{"hash":"a","received_ns":90},{"hash":"a","received_ns":110},{"hash":"unrelated","received_ns":300}]) + self.assertFalse(result["verified"]) + self.assertEqual(result["missing_hashes"],["b"]) + self.assertEqual(result["duplicate_hash_receipts"],1) + self.assertEqual(result["visible_before_ingest_return"],1) + self.assertIsNone(result["lag_ns"]["p99"]) + + def test_visibility_headline_excludes_warmup_but_keeps_earliest_negative_lags(self): + sent = [{"hash": "warm", "completed_ns": 100, "new_transmission": True, "measured": False}, + {"hash": "measured", "completed_ns": 200, "new_transmission": True, "measured": True}] + received = [{"hash": "warm", "received_ns": 90}, {"hash": "warm", "received_ns": 95}, + {"hash": "warm", "received_ns": 99}, {"hash": "measured", "received_ns": 190}, + {"hash": "measured", "received_ns": 220}] + result = bench.visibility(sent, received) + self.assertTrue(result["verified"]) + self.assertEqual(result["samples_ns"], [-10]) + self.assertEqual(result["lag_ns"]["samples"], 1) + self.assertEqual(result["duplicate_hash_receipts"], 1) + self.assertEqual(result["duplicate_hash_receipts_all"], 3) + self.assertEqual(result["visible_before_ingest_return"], 1) + self.assertEqual(result["visible_before_ingest_return_all"], 2) + + def test_missing_warmup_sentinel_still_fails_verification(self): + sent = [{"hash": "warm", "completed_ns": 100, "new_transmission": True, "measured": False}, + {"hash": "measured", "completed_ns": 200, "new_transmission": True, "measured": True}] + result = bench.visibility(sent, [{"hash": "measured", "received_ns": 230}]) + self.assertFalse(result["verified"]) + self.assertEqual(result["missing_hashes"], ["warm"]) + self.assertEqual(result["measured_missing"], 0) + self.assertEqual(result["samples_ns"], [30]) + + def test_warmup_does_not_make_sparse_visibility_p99_reportable(self): + sent = [{"hash": str(i), "completed_ns": 100, "new_transmission": True, "measured": i == 1000} for i in range(1001)] + result = bench.visibility(sent, [{"hash": str(i), "received_ns": 110} for i in range(1001)]) + self.assertEqual(result["lag_ns"]["samples"], 1) + self.assertIsNone(result["lag_ns"]["p99"]) + + def test_cgroup_launch_preserves_arguments_without_shell_interpolation(self): + budget = bench.Budget.__new__(bench.Budget) + budget.path = pathlib.Path("/sys/fs/cgroup/bench-test") + arguments = ["/tmp/application", "$(must-not-execute)", "a b", "a'b"] + launched = budget.arguments(arguments) + self.assertEqual(launched[-len(arguments):], arguments) + self.assertEqual(launched[-len(arguments)-1], str(budget.path)) + self.assertNotIn("must-not-execute", launched[2]) + budget.path = None + self.assertEqual(budget.arguments(arguments), arguments) + + def test_failure_diagnostic_keeps_errors_but_excludes_sensitive_log_data(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + log = root / "replay.log" + log.write_text('ordinary packet body: do not publish me\n' + ' zz_benchmark_ingestor_test.go:459: ERROR: missing relation "private_table" (SQLSTATE 42P01)\n' + ' zz_benchmark_ingestor_test.go:460: failed postgres://writer:topsecret@private.example/db\n' + ' zz_benchmark_ingestor_test.go:461: failed at /home/private/state/config.json and C:\\Private\\state.json\n' + ' zz_benchmark_ingestor_test.go:462: payload={"text":"private body"} raw=00112233445566778899aabbccddeeff\n' + ' zz_benchmark_ingestor_test.go:463: host=private.example password=topsecret\n' + '--- FAIL: TestCoreScopeBenchmark (1.20s)\n') + diagnostic = bench.record_failure(log, 1) + artifact = root / "failure-replay.json" + text = artifact.read_text() + self.assertEqual(diagnostic["log"], "replay.log") + self.assertEqual(diagnostic["exit_code"], 1) + self.assertIn("SQLSTATE 42P01", text) + self.assertIn("TestCoreScopeBenchmark", text) + for private in ("do not publish me", "topsecret", "private.example", "/home/private", "Private", "private body", "00112233", "private_table"): + self.assertNotIn(private, text) + self.assertLess(len(text), 5000) + + def test_public_bundle_includes_only_sanitized_failure_artifact(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + logs = root / "logs" + logs.mkdir() + log = logs / "postgres-build.log" + log.write_text("private raw log\n./main.go:27: undefined: MissingSymbol\n") + bench.record_failure(log, 1) + bench.public_bundle(root) + self.assertTrue((root / "public/logs/failure-postgres-build.json").is_file()) + self.assertFalse((root / "public/logs/postgres-build.log").exists()) + + def test_failure_diagnostic_bounds_long_logs(self): + with tempfile.TemporaryDirectory() as directory: + log = pathlib.Path(directory) / "worker.log" + log.write_text("ignored prefix\n" * 10000 + "panic: " + "x" * 20000 + "\n--- FAIL: TestCoreScopeBenchmark (1s)\n") + result = bench.record_failure(log, 2) + self.assertLessEqual(sum(len(x) for x in result["diagnostics"]), 2400) + self.assertIn("--- FAIL: TestCoreScopeBenchmark (1s)", result["diagnostics"]) + + def test_shipping_migrate_prefix_retains_safe_failure_reason(self): + with tempfile.TemporaryDirectory() as directory: + log = pathlib.Path(directory) / "migration.log" + log.write_text("[migrate] import dropped_packets: column dropped_at: non-text value in a text column\n" + "[migrate] connection failed postgres://owner:private-secret@private.example/db at /private/state/file\n") + result = bench.record_failure(log, 1) + diagnostic = json.dumps(result) + self.assertIn("import dropped_packets: column dropped_at: non-text value in a text column", diagnostic) + for private in ("private-secret", "private.example", "/private/state"): + self.assertNotIn(private, diagnostic) + + def test_finish_records_failed_child_before_raising(self): + with tempfile.TemporaryDirectory() as directory: + log = pathlib.Path(directory) / "replay.log" + log.write_text(" zz_benchmark_ingestor_test.go:460: durable effects did not match\n") + child = types.SimpleNamespace(wait=lambda timeout: 1, poll=lambda: 1, bench_log=open(log, "a")) + with self.assertRaisesRegex(RuntimeError, "failure-replay.json"): + bench.finish(child) + self.assertTrue(child.bench_log.closed) + self.assertIn("durable effects did not match", (log.parent / "failure-replay.json").read_text()) + + def test_retained_load_requires_every_eligible_row(self): + with tempfile.TemporaryDirectory() as directory: + source = pathlib.Path(directory) / "corpus.sqlite" + with bench.sqlite3.connect(source) as db: + db.executescript("CREATE TABLE transmissions(id INTEGER,last_seen INTEGER);" + "CREATE TABLE observations(transmission_id INTEGER);" + "INSERT INTO transmissions VALUES(1,90),(2,100),(3,110);" + "INSERT INTO observations VALUES(1),(2),(2),(3);") + db.close() + expected = bench.retained_counts(source, 100) + self.assertEqual(expected, {"transmissions": 2, "observations": 3}) + bench.check_loaded_rows({"loadedTx": 2, "loadedObs": 3}, expected) + for incomplete in ({"loadedTx": 1, "loadedObs": 3}, {"loadedTx": 2, "loadedObs": 2}): + with self.assertRaisesRegex(RuntimeError, "retained corpus"): + bench.check_loaded_rows(incomplete, expected) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/prepare-postgres-fixture.go b/scripts/prepare-postgres-fixture.go new file mode 100644 index 000000000..85a2c4511 --- /dev/null +++ b/scripts/prepare-postgres-fixture.go @@ -0,0 +1,142 @@ +// Run from cmd/migrate: +// go run ../../scripts/prepare-postgres-fixture.go -source ../../test-fixtures/e2e-fixture.db -destination /private/fixture.sqlite +// This explicitly repairs the historical test fixture on a new disposable copy. +// Production duplicates require a reviewed repair/upgrade with the last SQLite +// release while retaining the raw recovery snapshot; the importer never repairs +// or discards incompatible source records silently. +package main + +import ( + "crypto/sha256" + "database/sql" + "errors" + "flag" + "fmt" + "io" + "net/url" + "os" + "path/filepath" + "runtime" + + _ "github.com/mattn/go-sqlite3" + "github.com/meshcore-analyzer/dbschema/legacy" +) + +func main() { + source := flag.String("source", "", "closed historical SQLite fixture") + destination := flag.String("destination", "", "new disposable destination copy (must not exist)") + flag.Parse() + if err := prepareFixture(*source, *destination); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func prepareFixture(source, destination string) error { + if source == "" || destination == "" { + return errors.New("-source and -destination are required") + } + if info, err := os.Stat(source + "-wal"); err == nil && info.Size() > 0 { + return errors.New("fixture has WAL data; close/checkpoint its writer before preparing a disposable test copy") + } else if err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + before, err := fileHash(source) + if err != nil { + return err + } + in, err := os.Open(source) + if err != nil { + return err + } + defer in.Close() + if err := os.MkdirAll(filepath.Dir(destination), 0o700); err != nil { + return err + } + out, err := os.OpenFile(destination, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if err != nil { + return err + } + _, copyErr := io.Copy(out, in) + syncErr := out.Sync() + closeErr := out.Close() + if err := errors.Join(copyErr, syncErr, closeErr); err != nil { + return err + } + abs, err := filepath.Abs(destination) + if err != nil { + return err + } + abs = filepath.ToSlash(abs) + if runtime.GOOS == "windows" { + abs = "/" + abs + } + u := url.URL{Scheme: "file", Path: abs} + q := url.Values{"mode": {"rw"}} + u.RawQuery = q.Encode() + db, err := sql.Open("sqlite3", u.String()) + if err != nil { + return err + } + defer db.Close() + db.SetMaxOpenConns(1) + if _, err := db.Exec(`PRAGMA trusted_schema=OFF`); err != nil { + return err + } + beforeRows, err := counts(db) + if err != nil { + return err + } + // This step is deliberate and visible in the before/after counts. It is + // kept outside the production import command and runs only on destination. + if err := legacy.Apply(db, nil); err != nil { + return err + } + if err := legacy.Normalize(db, nil); err != nil { + return err + } + afterRows, err := counts(db) + if err != nil { + return err + } + if err := db.Close(); err != nil { + return err + } + after, err := fileHash(source) + if err != nil { + return err + } + if after != before { + return errors.New("original fixture changed during preparation; discard the disposable destination") + } + if info, err := os.Stat(source + "-wal"); err == nil && info.Size() > 0 { + return errors.New("fixture writer became active during preparation; discard the disposable destination") + } + fmt.Printf("transmissions: %d -> %d; observations: %d -> %d; original fixture unchanged\n", beforeRows[0], afterRows[0], beforeRows[1], afterRows[1]) + return nil +} + +func counts(db *sql.DB) ([2]int64, error) { + var result [2]int64 + for i, table := range []string{"transmissions", "observations"} { + if err := db.QueryRow(`SELECT count(*) FROM ` + table).Scan(&result[i]); err != nil { + return result, err + } + } + return result, nil +} + +func fileHash(path string) ([32]byte, error) { + var result [32]byte + file, err := os.Open(path) + if err != nil { + return result, err + } + defer file.Close() + h := sha256.New() + if _, err := io.Copy(h, file); err != nil { + return result, err + } + copy(result[:], h.Sum(nil)) + return result, nil +} diff --git a/scripts/prune-nodes-outside-geo-filter.py b/scripts/prune-nodes-outside-geo-filter.py index 6f34eb08e..fd8e1bb0e 100644 --- a/scripts/prune-nodes-outside-geo-filter.py +++ b/scripts/prune-nodes-outside-geo-filter.py @@ -1,10 +1,13 @@ #!/usr/bin/env python3 """ +Offline legacy SQLite tooling; use the Admin Geo Filter workflow for PostgreSQL. +Operate only on a stopped, disposable recovery copy. + Delete nodes from the database that fall outside the configured geo_filter polygon + bufferKm. Nodes with no GPS coordinates are always kept. Usage: - python3 prune-nodes-outside-geo-filter.py [db_path] [--config config.json] [--dry-run] + python3 prune-nodes-outside-geo-filter.py --legacy-sqlite-offline [db_path] [--config config.json] [--dry-run] db_path Path to meshcore.db (default: /app/data/meshcore.db) --config PATH Path to config.json (default: /app/config.json) @@ -165,4 +168,7 @@ def main(): if __name__ == '__main__': + if '--legacy-sqlite-offline' not in sys.argv: + sys.exit('Offline legacy SQLite tooling only. Use the Admin Geo Filter workflow for PostgreSQL. Pass --legacy-sqlite-offline only for a stopped, disposable recovery copy.') + sys.argv.remove('--legacy-sqlite-offline') main() diff --git a/scripts/test-postgres-compose.sh b/scripts/test-postgres-compose.sh new file mode 100755 index 000000000..25dc1dec6 --- /dev/null +++ b/scripts/test-postgres-compose.sh @@ -0,0 +1,366 @@ +#!/usr/bin/env bash +# CI-only packaged startup/ingestion smoke. No image publication or host ports. +set +x +set -euo pipefail +[[ ${GITHUB_ACTIONS:-} == true && $(uname -s) == Linux ]] || { echo 'This smoke requires a disposable Linux GitHub Actions runner.' >&2; exit 1; } +[[ ${GITHUB_SHA:-} =~ ^[0-9a-f]{40}$ && ${1:-} == "corescope-pg-smoke:$GITHUB_SHA" ]] || { echo 'An exact locally built candidate image is required.' >&2; exit 1; } +repo=$(cd "$(dirname "$0")/.." && pwd -P) +temp_root=$(realpath -e "${RUNNER_TEMP:?RUNNER_TEMP is required}") +[[ -x "$repo/docker/postgres-init.sh" ]] || { echo 'PostgreSQL initialization must be executable, not sourced by the vendor entrypoint.' >&2; exit 1; } +image=$1 +backend=${2:-postgres} +case "$backend" in sqlite|postgres) ;; *) echo 'Choose sqlite or postgres for the packaged smoke.' >&2; exit 1 ;; esac +image_id=$(timeout 15s docker image inspect --format '{{.Id}}' "$image") +umask 077 +work=$(mktemp -d "$temp_root/corescope-pg-smoke-XXXXXXXX") +project=$(basename "$work" | tr '[:upper:]' '[:lower:]') +owned_project=false +phase=configuration +dc=(docker compose --ansi never --project-name "$project" --env-file "$work/.env" + -f "$repo/docker-compose.example.yml") +if [ "$backend" = postgres ]; then dc+=(-f "$repo/docker-compose.example.postgres.yml"); fi +dc+=(-f "$work/override.yml") + +project_resources() { + timeout 10s docker container ls -aq --filter "label=com.docker.compose.project=$project" || return 1 + timeout 10s docker network ls -q --filter "label=com.docker.compose.project=$project" || return 1 + timeout 10s docker volume ls -q --filter "label=com.docker.compose.project=$project" +} + +finish() { + local result=$? cleanup_ok=true + trap - EXIT INT TERM + set +e + if [ "$result" -ne 0 ] && $owned_project; then + timeout 20s "${dc[@]}" logs --no-color --tail 100 > "$work/container.raw.log" 2>&1 + fi + if $owned_project; then + timeout 60s "${dc[@]}" down --volumes --remove-orphans --timeout 20 >> "$work/actions.raw.log" 2>&1 || cleanup_ok=false + remaining=$(project_resources) || cleanup_ok=false + [ -z "$remaining" ] || cleanup_ok=false + fi + if ! $cleanup_ok; then result=1; phase=cleanup; fi + # Persist only bounded, redacted diagnostics; never print Compose's full config. + python3 - "$work" <<'PY' || result=1 +import os,re,sys +from pathlib import Path +root=Path(sys.argv[1]) +text="" +for name in ("actions.raw.log","container.raw.log"): + path=root/name + if path.exists(): + with path.open("rb") as source: + text += source.read(262144).decode("utf-8",errors="replace")+"\n" +for key,value in os.environ.items(): + if key.endswith("_PASSWORD") and value: + text=text.replace(value,"[redacted]") +text=re.sub(r"postgres(?:ql)?://[^\s'\"<>]+","[redacted database URL]",text) +(root/"failure.log").write_text(text,encoding="utf-8") +PY + # Remove only this mktemp directory's private data, after the project is gone. + if [[ $(realpath -e "$work") == "$temp_root"/corescope-pg-smoke-* && ! -L "$work" ]]; then + if $cleanup_ok; then + sudo rm -rf -- "$work/state" "$work/postgres" || result=1 + rm -f -- "$work/.env" "$work/compose.json" "$work/override.yml" "$work/packet.json" "$work/stats.json" "$work/states.json" "$work/health.json" "$work/proxy-health.json" "$work/actions.raw.log" "$work/container.raw.log" "$work/recreate.yml" "$work/counts.txt" || result=1 + fi + if [ "$result" -eq 0 ]; then + rm -f -- "$work/failure.log" + rmdir -- "$work" || result=1 + else + echo "Compose smoke failed during $phase; sanitized diagnostics: $work/failure.log" >&2 + tail -n 120 "$work/failure.log" >&2 + fi + else + echo 'Refusing cleanup outside the owned smoke directory.' >&2 + result=1 + fi + exit "$result" +} +trap finish EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +touch "$work/.env" +if [ "$backend" = postgres ]; then +for key in POSTGRES_ADMIN_PASSWORD CORESCOPE_OWNER_PASSWORD CORESCOPE_READER_PASSWORD CORESCOPE_WRITER_PASSWORD CORESCOPE_ACCOUNTS_PASSWORD CORESCOPE_CHANNELS_PASSWORD; do + password=$(openssl rand -hex 32) + export "$key=$password" + printf '%s=%s\n' "$key" "${!key}" >> "$work/.env" +done +unset password +else + unset POSTGRES_ADMIN_PASSWORD CORESCOPE_OWNER_PASSWORD CORESCOPE_READER_PASSWORD CORESCOPE_WRITER_PASSWORD CORESCOPE_ACCOUNTS_PASSWORD CORESCOPE_CHANNELS_PASSWORD +fi +export CORESCOPE_STATE_DIR=/app/data/state +export CORESCOPE_IMAGE="$image" DATA_DIR="$work/state" POSTGRES_DATA_DIR="$work/postgres" +export DISABLE_MOSQUITTO=false +# Exercise the example's default Caddy/port-80 path, including when the runner +# environment has its own Caddy preference. The owned .env contains only secrets. +unset DISABLE_CADDY +mkdir "$work/state" +python3 - "$repo" "$work" <<'PY' +import json,re,sys +from pathlib import Path +repo,work=map(Path,sys.argv[1:]) +config={"siteName":"CI packaged native storage smoke","port":3000,"stateDir":"/app/data/state", + "mqttSources":[{"name":"ci-smoke","broker":"mqtt://localhost:1883","topics":["meshcore/SJC/ci-smoke/packets"]}], + "userManagement":{"enabled":False},"autoRegionKeys":{"enabled":False}} +(work/"state/config.json").write_text(json.dumps(config),encoding="utf-8") +# Reuse the existing ingestor regression's packet, not invented wire bytes. +test=(repo/"cmd/ingestor/main_test.go").read_text(encoding="utf-8").split("func TestHandleMessageRawPacket(t *testing.T) {",1)[1].split("\n}",1)[0] +raw=re.search(r'rawHex := "([0-9A-Fa-f]+)"',test).group(1) +bytes.fromhex(raw) +(work/"packet.json").write_text(json.dumps({"raw":raw,"SNR":5.5,"RSSI":-100.0,"origin":"ci-smoke"})+"\n",encoding="utf-8") +PY +cat > "$work/override.yml" <<'YAML' +services: + corescope: + pull_policy: never + restart: "no" + ports: !reset [] +networks: + default: + internal: true +YAML +if [ "$backend" = postgres ]; then + cat >> "$work/override.yml" <<'YAML' +volumes: + postgres-smoke: +YAML + python3 - "$work/override.yml" <<'PY' +from pathlib import Path +import sys +p=Path(sys.argv[1]); text=p.read_text() +text=text.replace('services:\n','services:\n postgres:\n restart: "no"\n volumes:\n - postgres-smoke:/var/lib/postgresql\n bootstrap:\n pull_policy: never\n',1) +p.write_text(text) +PY +fi + +timeout 30s "${dc[@]}" config --format json > "$work/compose.json" 2>> "$work/actions.raw.log" +python3 - "$repo" "$work" "$backend" <<'PY' +import json,sys +from pathlib import Path +repo,work=map(Path,sys.argv[1:3]); backend=sys.argv[3] +config=json.loads((work/"compose.json").read_text()) +assert config["name"]==work.name.lower() +assert set(config["services"])==({"postgres","bootstrap","corescope"} if backend=="postgres" else {"corescope"}) +assert config["services"]["corescope"]["environment"]["CORESCOPE_STATE_DIR"]=="/app/data/state" +assert set(config["networks"])=={"default"} +assert config["networks"]["default"]["internal"] is True +assert config["services"]["corescope"]["environment"]["DISABLE_CADDY"] == "false", "smoke must use the default Caddy listener" +allowed={(work/"state").resolve(),(repo/"docker/postgres-init.sh").resolve()} +for service in config["services"].values(): + assert not service.get("ports"), "smoke must publish no ports" + assert not service.get("network_mode") and set(service["networks"])=={"default"} + for mount in service.get("volumes",[]): + if mount["type"]=="bind": + assert Path(mount["source"]).resolve() in allowed, "unexpected host mount" +for volume in config.get("volumes",{}).values(): + assert not volume.get("external") and volume["name"].startswith(config["name"]+"_") +PY +existing=$(project_resources) +[[ -z "$existing" ]] || { echo 'Smoke project already exists; refusing to touch it.' >&2; exit 1; } +owned_project=true +if [ "$backend" = postgres ]; then + phase=nil-account-adoption + # Generate a tiny native telemetry fixture with seed/account state outside the + # active selection, then adopt it as a legacy telemetry-only installation. + # No selection JSON is authored or edited by this test. + timeout 60s "${dc[@]}" run --rm --no-deps --entrypoint /bin/sh corescope -eu -c ' + /app/corescope-migrate -storage-action=init -backend=sqlite -offline -config-dir /app \ + -selection-file /app/data/generator/storage-selection.json \ + -sqlite-path /app/data/meshcore.db -users-sqlite-path /app/data/generator/users.db + /app/corescope-migrate -storage-action=adopt -backend=sqlite -offline -config-dir /app \ + -selection-file /app/data/state/storage-selection.json \ + -sqlite-path /app/data/meshcore.db -users-sqlite-path /app/data/uninitialized-users.db + ' >> "$work/actions.raw.log" 2>&1 + timeout 90s "${dc[@]}" up -d --wait --wait-timeout 60 postgres >> "$work/actions.raw.log" 2>&1 + timeout 120s "${dc[@]}" run --rm --no-deps --entrypoint /app/storage.sh bootstrap switch -backend=postgres >> "$work/actions.raw.log" 2>&1 +fi +phase=startup +timeout 210s "${dc[@]}" up -d --wait --wait-timeout 180 >> "$work/actions.raw.log" 2>&1 +application=$(timeout 10s "${dc[@]}" ps -q corescope) +phase=bootstrap-order +[[ $(timeout 10s docker inspect --format '{{.Image}}' "$application") == "$image_id" ]] +if [ "$backend" = postgres ]; then +bootstrap=$(timeout 10s "${dc[@]}" ps -a -q bootstrap) +timeout 10s docker inspect --format '{{json .State}}' "$bootstrap" "$application" > "$work/states.json" +python3 - "$work/states.json" <<'PY' +import datetime,json,sys +bootstrap,app=map(json.loads,open(sys.argv[1],encoding="utf-8")) +assert bootstrap["Status"]=="exited" and bootstrap["ExitCode"]==0 +assert app["Running"] +stamp=lambda value: datetime.datetime.fromisoformat(value.replace("Z","+00:00")) +assert stamp(app["StartedAt"]) >= stamp(bootstrap["FinishedAt"]), "application preceded bootstrap completion" +PY +fi +timeout 10s "${dc[@]}" exec -T corescope pg_dump --version | grep -Eq '^pg_dump .* 18[.]' + +phase=schema-readiness +[[ $(timeout 10s "${dc[@]}" exec -T corescope /app/storage.sh field backend) == "$backend" ]] +[[ $(timeout 10s "${dc[@]}" exec -T corescope /app/storage.sh field state_dir) == /app/data/state ]] +if [ "$backend" = postgres ]; then +owner_sql() { + timeout 10s "${dc[@]}" exec -T postgres sh -eu -c ' + export PGHOST=127.0.0.1 PGUSER=corescope_owner PGPASSWORD="$CORESCOPE_OWNER_PASSWORD" PGDATABASE="$1" + exec psql -X -A -t -v ON_ERROR_STOP=1 + ' sh "$1" 2>> "$work/actions.raw.log" +} +[[ $(owner_sql corescope_telemetry <<'SQL' +SELECT version=2 AND ready AND current_setting('server_version_num')='180006' FROM corescope_schema WHERE kind='telemetry'; +SQL +) == t ]] +[[ $(timeout 10s "${dc[@]}" exec -T corescope /app/storage.sh field has_accounts) == false ]] +# An explicit test inspection proves bootstrap did not initialize the absent +# account target. Runtime bootstrap/readiness/grants must select telemetry only. +[[ $(owner_sql corescope_accounts <<'SQL' +SELECT count(*)=0 FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='public' AND c.relkind IN ('r','p','v','m','S','f'); +SQL +) == t ]] + +else + timeout 10s "${dc[@]}" exec -T corescope sqlite3 -readonly /app/data/meshcore.db "SELECT count(*) FROM transmissions;" | grep -qx 0 +fi + +phase=mqtt-ingestion +timeout 10s "${dc[@]}" exec -T corescope mosquitto_pub -h 127.0.0.1 -q 1 -r -t meshcore/SJC/ci-smoke/packets -s < "$work/packet.json" >> "$work/actions.raw.log" 2>&1 +for attempt in $(seq 1 30); do + if timeout 5s "${dc[@]}" exec -T corescope wget -qO- http://127.0.0.1:3000/api/healthz > "$work/health.json" 2>> "$work/actions.raw.log" && + timeout 5s "${dc[@]}" exec -T corescope wget -qO- http://127.0.0.1:80/api/healthz > "$work/proxy-health.json" 2>> "$work/actions.raw.log" && + timeout 5s "${dc[@]}" exec -T corescope wget -qO- http://127.0.0.1:80/api/stats > "$work/stats.json" 2>> "$work/actions.raw.log" && + python3 - "$work" "$GITHUB_SHA" <<'PY' +import json,sys +from pathlib import Path +root=Path(sys.argv[1]) +health=json.loads((root/"health.json").read_text()) +proxy_health=json.loads((root/"proxy-health.json").read_text()) +stats=json.loads((root/"stats.json").read_text()) +sys.exit(0 if health.get("ready") is True and proxy_health.get("ready") is True and stats.get("totalTransmissions")==1 and stats.get("totalObservations")==1 and stats.get("commit")==sys.argv[2][:7] else 1) +PY + then break; fi + [ "$attempt" -lt 30 ] || { echo 'Published fixture did not become visible through the packaged API.' >&2; exit 1; } + sleep 1 +done +phase=runtime-roles +if [ "$backend" = postgres ]; then +[[ $(owner_sql corescope_telemetry <<'SQL' +SELECT (SELECT count(*)=1 FROM transmissions) + AND (SELECT count(*)=1 FROM observations) + AND EXISTS(SELECT 1 FROM pg_stat_activity WHERE datname=current_database() AND usename='corescope_reader') + AND EXISTS(SELECT 1 FROM pg_stat_activity WHERE datname=current_database() AND usename='corescope_writer') + AND NOT has_table_privilege('corescope_reader','transmissions','INSERT,UPDATE,DELETE') + AND NOT has_schema_privilege('corescope_reader','public','CREATE') + AND NOT has_schema_privilege('corescope_writer','public','CREATE'); +SQL +) == t ]] +else + timeout 10s "${dc[@]}" exec -T corescope sqlite3 -readonly /app/data/meshcore.db 'SELECT count(*) FROM transmissions; SELECT count(*) FROM observations;' > "$work/counts.txt" + [ "$(cat "$work/counts.txt")" = $'1\n1' ] +fi +if [ "$backend" = postgres ]; then + # Reconstruct the previous schema only inside this owned disposable project. + # Existing telemetry is retained; the three new-feature tables were not + # part of v1. No runtime process may be alive while fixture DDL runs. + previous_telemetry_layout() { + owner_sql corescope_telemetry <<'SQL' >> "$work/actions.raw.log" +BEGIN; +DROP TABLE tx_rewrite_feed,key_backfill_state,key_backfill_done; +UPDATE corescope_schema SET version=1 WHERE kind='telemetry'; +COMMIT; +SQL + } + selection_digest() { + timeout 10s "${dc[@]}" exec -T corescope sha256sum /app/data/state/storage-selection.json + } + + phase=recorded-telemetry-upgrade-without-accounts + before=$(selection_digest) + timeout 60s "${dc[@]}" stop corescope >> "$work/actions.raw.log" 2>&1 + previous_telemetry_layout + timeout 210s "${dc[@]}" up -d --force-recreate --wait --wait-timeout 180 >> "$work/actions.raw.log" 2>&1 + [[ $(selection_digest) == "$before" ]] + [[ $(timeout 10s "${dc[@]}" exec -T corescope /app/storage.sh field has_accounts) == false ]] + [[ $(owner_sql corescope_telemetry <<'SQL' +SELECT version=2 AND ready AND (SELECT count(*)=1 FROM transmissions) FROM corescope_schema WHERE kind='telemetry'; +SQL +) == t ]] + [[ $(owner_sql corescope_accounts <<'SQL' +SELECT count(*)=0 FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='public' AND c.relkind IN ('r','p','v','m','S','f'); +SQL +) == t ]] + + phase=first-account-setup + timeout 60s "${dc[@]}" stop corescope >> "$work/actions.raw.log" 2>&1 + timeout 90s "${dc[@]}" run --rm --no-deps --entrypoint /app/storage.sh bootstrap setup -backend=postgres >> "$work/actions.raw.log" 2>&1 + timeout 210s "${dc[@]}" up -d --force-recreate --wait --wait-timeout 180 >> "$work/actions.raw.log" 2>&1 + [[ $(timeout 10s "${dc[@]}" exec -T corescope /app/storage.sh field has_accounts) == true ]] + [[ $(owner_sql corescope_accounts <<'SQL' +SELECT version=1 AND ready AND (SELECT version=6 FROM schema_version) FROM corescope_schema WHERE kind='accounts'; +SQL +) == t ]] + + phase=recorded-v1-v5-upgrade + before=$(selection_digest) + timeout 60s "${dc[@]}" stop corescope >> "$work/actions.raw.log" 2>&1 + previous_telemetry_layout + # Account UI remains disabled: initialized accounts still need upgrading. + owner_sql corescope_accounts <<'SQL' >> "$work/actions.raw.log" +BEGIN; +-- Build the genuine v5 physical layout: dropping columns would leave catalog +-- slots and produce an unsupported synthetic layout after re-adding them. +DO $$ BEGIN + IF EXISTS(SELECT 1 FROM sessions) THEN RAISE EXCEPTION 'fixture expected no sessions before seeding'; END IF; +END $$; +DROP VIEW linked_companions; +DROP TABLE companion_links,link_challenges,sessions; +CREATE TABLE sessions ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + token_hash TEXT COLLATE "C" NOT NULL UNIQUE, + user_id BIGINT NOT NULL REFERENCES users(id) ON DELETE CASCADE, + csrf_token TEXT COLLATE "C" NOT NULL, + created_at BIGINT NOT NULL, + expires_at BIGINT NOT NULL, + last_seen_at BIGINT NOT NULL, + user_agent TEXT COLLATE "C" NOT NULL DEFAULT '' +); +CREATE INDEX sessions_user ON sessions(user_id); +INSERT INTO users(id,email,display_name,password_hash,status,created_at) VALUES(4000000000,'upgrade@example.invalid','Retained fixture','opaque-hash','active',1); +INSERT INTO sessions(id,token_hash,user_id,csrf_token,created_at,expires_at,last_seen_at) VALUES(42,'retained-session-hash',4000000000,'retained-csrf',1,9999999999,2); +SELECT setval(pg_get_serial_sequence('users','id'),4000000100,true); +SELECT setval(pg_get_serial_sequence('sessions','id'),42,true); +UPDATE schema_version SET version=5; +COMMIT; +SQL + timeout 210s "${dc[@]}" up -d --force-recreate --wait --wait-timeout 180 >> "$work/actions.raw.log" 2>&1 + [[ $(selection_digest) == "$before" ]] + [[ $(owner_sql corescope_telemetry <<'SQL' +SELECT version=2 AND ready AND (SELECT count(*)=1 FROM transmissions) FROM corescope_schema WHERE kind='telemetry'; +SQL +) == t ]] + [[ $(owner_sql corescope_accounts <<'SQL' +SELECT version=1 AND ready AND (SELECT version=6 FROM schema_version) + AND EXISTS(SELECT 1 FROM users WHERE id=4000000000 AND password_hash='opaque-hash' AND display_name='Retained fixture') + AND EXISTS(SELECT 1 FROM sessions WHERE id=42 AND token_hash='retained-session-hash' AND csrf_token='retained-csrf' AND user_id=4000000000 AND kind='web' AND label='' AND scopes='') + AND (SELECT last_value=4000000100 FROM users_id_seq) + AND has_table_privilege('corescope_channels','linked_companions','SELECT') + AND NOT has_table_privilege('corescope_channels','companion_links','SELECT') + AND NOT has_table_privilege('corescope_channels','link_challenges','SELECT') + FROM corescope_schema WHERE kind='accounts'; +SQL +) == t ]] +fi +phase=recreation +before=$(timeout 10s "${dc[@]}" exec -T corescope /app/storage.sh field generation) +# Exercise stale bootstrap hints and persistent custom state on actual recreate. +other=sqlite; [ "$backend" != sqlite ] || other=postgres +printf 'services:\n corescope:\n environment:\n CORESCOPE_DB_BACKEND: %s\n' "$other" > "$work/recreate.yml" +dc+=(-f "$work/recreate.yml") +timeout 180s "${dc[@]}" up -d --no-deps --force-recreate --wait --wait-timeout 150 corescope >> "$work/actions.raw.log" 2>&1 +[[ $(timeout 10s "${dc[@]}" exec -T corescope /app/storage.sh field generation) == "$before" ]] +[[ $(timeout 10s "${dc[@]}" exec -T corescope /app/storage.sh field backend) == "$backend" ]] +timeout 10s "${dc[@]}" exec -T corescope wget -qO- http://127.0.0.1:80/api/stats > "$work/stats.json" +python3 - "$work/stats.json" <<'PY' +import json,sys +stats=json.load(open(sys.argv[1])); assert stats['totalTransmissions']==1 and stats['totalObservations']>=1 +PY +echo "Packaged $backend setup, recorded-schema upgrades, persistent selection on recreate, native runtime, MQTT ingestion, direct/Caddy readiness, API visibility and native clients passed." diff --git a/scripts/test-storage-compose.js b/scripts/test-storage-compose.js new file mode 100644 index 000000000..370048a53 --- /dev/null +++ b/scripts/test-storage-compose.js @@ -0,0 +1,93 @@ +// Validate actual Compose models without starting a Docker daemon or service. +// Run: node scripts/test-storage-compose.js +// Optional portable executable: CORESCOPE_COMPOSE_BIN=/path/to/docker-compose +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const crypto = require('node:crypto'); +const { spawnSync } = require('node:child_process'); +const root = path.resolve(__dirname, '..'); +const temp = fs.mkdtempSync(path.join(os.tmpdir(), 'corescope-storage-config-')); +const candidates = process.env.CORESCOPE_COMPOSE_BIN + ? [[process.env.CORESCOPE_COMPOSE_BIN, []]] + : [['docker', ['compose']], ['docker-compose', []]]; +const executable = candidates.find(([bin, args]) => spawnSync(bin, [...args, 'version'], { encoding: 'utf8', timeout: 10000 }).status === 0); +assert(executable, 'Docker Compose CLI required for model validation (no daemon needed); set CORESCOPE_COMPOSE_BIN for a portable executable'); +const [bin, prefix] = executable; +const passwordKeys = ['POSTGRES_ADMIN_PASSWORD', 'CORESCOPE_OWNER_PASSWORD', 'CORESCOPE_READER_PASSWORD', 'CORESCOPE_WRITER_PASSWORD', 'CORESCOPE_ACCOUNTS_PASSWORD', 'CORESCOPE_CHANNELS_PASSWORD']; +const clean = { ...process.env }; +for (const key of Object.keys(clean)) if (/^(PG|POSTGRES_|CORESCOPE_)/.test(key)) delete clean[key]; +Object.assign(clean, { + DATA_DIR: path.join(temp, 'example-state'), + PROD_DATA_DIR: path.join(temp, 'prod-state'), + STAGING_DATA_DIR: path.join(temp, 'staging-state'), + POSTGRES_DATA_DIR: path.join(temp, 'example-postgres'), + PROD_POSTGRES_DATA_DIR: path.join(temp, 'prod-postgres'), + STAGING_POSTGRES_DATA_DIR: path.join(temp, 'staging-postgres'), +}); +const credentials = Object.fromEntries(passwordKeys.map(key => [key, crypto.randomBytes(24).toString('hex')])); +const emptyEnv = path.join(temp, 'empty.env'); +fs.writeFileSync(emptyEnv, ''); +const cases = [ + ['docker-compose.yml', 'prod', 'docker-compose.postgres.yml', 'corescope', clean.PROD_POSTGRES_DATA_DIR], + ['docker-compose.no-mosquitto.yml', 'prod', 'docker-compose.postgres.yml', 'corescope', clean.PROD_POSTGRES_DATA_DIR], + ['docker-compose.example.yml', 'corescope', 'docker-compose.example.postgres.yml', 'corescope', clean.POSTGRES_DATA_DIR], + ['docker-compose.staging.yml', 'staging-go', 'docker-compose.staging.postgres.yml', 'corescope-staging', clean.STAGING_POSTGRES_DATA_DIR], + ['docker-compose.staging.no-mosquitto.yml', 'staging-go', 'docker-compose.staging.postgres.yml', 'corescope-staging', clean.STAGING_POSTGRES_DATA_DIR], +]; +function model(files, env) { + return spawnSync(bin, [...prefix, '--env-file', emptyEnv, ...files.flatMap(file => ['-f', path.join(root, file)]), 'config', '--format', 'json'], { cwd: root, env, encoding: 'utf8', timeout: 20000 }); +} +function parsed(result, label) { + // Deliberately never print a rendered model: it contains generated passwords. + assert.equal(result.status, 0, `${label} failed Compose validation: ${result.stderr || result.error?.message}`); + return JSON.parse(result.stdout); +} +try { + for (const [base, appName, override, project, postgresPath] of cases) { + const sqlite = parsed(model([base], clean), `${base} with all PostgreSQL variables unset`); + assert.equal(sqlite.name, project); + assert.deepEqual(Object.keys(sqlite.services), [appName], `${base} started an optional service by default`); + const app = sqlite.services[appName]; + assert.equal(app.environment.CORESCOPE_DB_BACKEND, 'sqlite'); + assert.equal(app.environment.CORESCOPE_STATE_DIR, '/app/data'); + assert(!app.depends_on?.postgres && !app.depends_on?.bootstrap, 'SQLite app depends on PostgreSQL'); + assert(!Object.keys(app.environment).some(key => /DATABASE_URL|PASSWORD/.test(key)), 'default app needs PostgreSQL credentials'); + const missing = model([base, override], clean); + assert.notEqual(missing.status, 0, `${override} accepted missing required credentials`); + const pg = parsed(model([base, override], { ...clean, ...credentials }), `${base} + ${override}`); + assert.equal(pg.name, project); + assert.deepEqual(Object.keys(pg.services).sort(), [appName, 'bootstrap', 'postgres'].sort()); + assert(!pg.services.postgres.ports?.length, 'database port exposed on host'); + assert.equal(pg.services.postgres.image, 'postgres:18.6-alpine3.24'); + const data = pg.services.postgres.volumes.find(volume => volume.target === '/var/lib/postgresql'); + assert.equal(path.resolve(data.source), path.resolve(postgresPath), 'wrong environment persistent database directory'); + const init = pg.services.postgres.volumes.find(volume => volume.target === '/docker-entrypoint-initdb.d/10-corescope.sh'); + assert.equal(path.resolve(init.source), path.resolve(root, 'docker/postgres-init.sh'), 'extends resolved the init script outside this checkout'); + assert(init.read_only); + const runtime = pg.services[appName]; + assert.equal(runtime.environment.CORESCOPE_DB_BACKEND, 'postgres'); + for (const key of passwordKeys) assert(!(key in runtime.environment), 'owner/admin credentials reached runtime'); + const roles = { CORESCOPE_READER_DATABASE_URL: ['corescope_reader','corescope_telemetry'], CORESCOPE_WRITER_DATABASE_URL: ['corescope_writer','corescope_telemetry'], CORESCOPE_USERS_DATABASE_URL: ['corescope_accounts','corescope_accounts'], CORESCOPE_APPROVED_CHANNELS_DATABASE_URL: ['corescope_channels','corescope_accounts'] }; + for (const [key, [user, db]] of Object.entries(roles)) { + const target = new URL(runtime.environment[key]); + assert.equal(target.username, user); assert.equal(target.hostname, 'postgres'); assert.equal(target.port, '5432', 'managed role endpoint must be explicitly pinned'); assert.equal(target.searchParams.get('search_path'), 'public', 'managed schema must be pinned'); assert.equal(target.pathname, '/' + db); + assert(![credentials.POSTGRES_ADMIN_PASSWORD, credentials.CORESCOPE_OWNER_PASSWORD].includes(target.password), 'owner credential in runtime URL'); + } + assert.equal(runtime.depends_on.bootstrap.condition, 'service_completed_successfully'); + assert.equal(pg.services.bootstrap.depends_on.postgres.condition, 'service_healthy'); + assert.equal(pg.services.bootstrap.environment.CORESCOPE_STATE_DIR, runtime.environment.CORESCOPE_STATE_DIR); + const runtimeState = runtime.volumes.find(volume => volume.target === '/app/data'); + const setupState = pg.services.bootstrap.volumes.find(volume => volume.target === '/app/data'); + assert.equal(setupState.source, runtimeState.source); assert(!setupState.read_only, 'bootstrap cannot atomically commit shared selection'); + if (base.includes('no-mosquitto')) assert.equal(runtime.environment.DISABLE_MOSQUITTO, 'true'); + if (base === 'docker-compose.staging.yml') assert(pg.networks['meshcore-net'].external, 'staging lost its existing broker network'); + console.log(`PASS ${base}: SQLite default and explicit PostgreSQL override`); + } + const custom = parsed(model(['docker-compose.example.yml','docker-compose.example.postgres.yml'], { ...clean, ...credentials, CORESCOPE_STATE_DIR: '/app/data/custom-state' }), 'custom shared state directory'); + for (const service of ['corescope','bootstrap']) assert.equal(custom.services[service].environment.CORESCOPE_STATE_DIR, '/app/data/custom-state'); +} finally { + assert(path.resolve(temp).startsWith(path.resolve(os.tmpdir()) + path.sep) && path.basename(temp).startsWith('corescope-storage-config-')); + fs.rmSync(temp, { recursive: true }); +} diff --git a/test-all.sh b/test-all.sh index 758731029..1b586e80b 100755 --- a/test-all.sh +++ b/test-all.sh @@ -220,4 +220,9 @@ node tests/unit/test-warmup-banner.js node tests/unit/test-ws-stale-watchdog-1074.js node tests/unit/test-xss-escape-sinks.js +node tests/unit/test-postgres-ops.js +node tests/unit/test-deployment-readiness.js +node tests/unit/test-storage-entrypoint.js +node tests/unit/test-storage-manage.js + echo "All standalone frontend suites passed" diff --git a/tests/e2e/test-e2e-playwright.js b/tests/e2e/test-e2e-playwright.js index f5c6dafae..88a1f574f 100644 --- a/tests/e2e/test-e2e-playwright.js +++ b/tests/e2e/test-e2e-playwright.js @@ -369,6 +369,9 @@ async function run() { // TODO: requires running server with full customize/home wiring await page.goto(BASE, { waitUntil: 'domcontentloaded' }); await page.waitForSelector('nav, .navbar, .nav, [class*="nav"]'); + // The initial theme response installs SITE_CONFIG asynchronously. Set the + // sentinel only after that completes, so this tests opening the customizer. + await page.waitForFunction(() => window._customizerV2 && window._customizerV2.initDone); await page.evaluate(() => { localStorage.removeItem('cs-theme-overrides'); window.SITE_CONFIG = window.SITE_CONFIG || {}; diff --git a/tests/e2e/test-hop-path-inference-e2e.js b/tests/e2e/test-hop-path-inference-e2e.js index 99bfaf360..762cd63ab 100644 --- a/tests/e2e/test-hop-path-inference-e2e.js +++ b/tests/e2e/test-hop-path-inference-e2e.js @@ -2,7 +2,7 @@ * E2E: the packet detail pane infers an observation's hops by position, and * the map route view draws the same hops. * - * Fixture: packet 57224a4a01c9c682, observation 147 — a 22-hop flood whose + * Fixture: transmission 147, observation 147 — a 22-hop flood whose * path carries the 1-byte prefix "10" three times and "D6" twice, so a * per-prefix resolution would give each repeat the same answer. * @@ -12,7 +12,7 @@ const { chromium } = require('playwright'); const BASE = process.env.BASE_URL || 'http://localhost:13581'; -const HASH = '57224a4a01c9c682'; +const TRANSMISSION_ID = '147'; const OBS_ID = '147'; let passed = 0, failed = 0; @@ -36,9 +36,22 @@ function assert(c, m) { if (!c) throw new Error(m || 'assertion failed'); } console.log(`\n=== Hop path inference E2E against ${BASE} ===`); + // Offline fixture preparation normalizes legacy hashes. Resolve the preserved + // transmission ID through the real API, then exercise its canonical hash links. + const fixtureResponse = await ctx.request.get(`${BASE}/api/packets/${TRANSMISSION_ID}`); + assert(fixtureResponse.ok(), `fixture transmission returned HTTP ${fixtureResponse.status()}`); + const fixture = await fixtureResponse.json(); + const hash = fixture.packet.hash; + assert(/^[a-f0-9]{16}$/.test(hash), 'fixture has no canonical packet hash'); + const observation = fixture.observations.find(o => String(o.id) === OBS_ID); + assert(observation, 'fixture observation is missing'); + const path = JSON.parse(observation.path_json); + assert(path.length === 22 && path.filter(h => h === '10').length === 3 && + path.filter(h => h === 'D6').length === 2, 'fixture no longer has the repeated-prefix route'); + let detailHops = []; await step('the detail pane renders the observation\'s 22 hops', async () => { - await page.goto(`${BASE}/#/packets/${HASH}?obs=${OBS_ID}`, { waitUntil: 'domcontentloaded' }); + await page.goto(`${BASE}/#/packets/${hash}?obs=${OBS_ID}`, { waitUntil: 'domcontentloaded' }); await page.waitForFunction(() => { const dt = [...document.querySelectorAll('.detail-meta dt')].find(d => d.textContent === 'Path'); return dt && dt.nextElementSibling.querySelectorAll('.hop').length > 0; @@ -53,6 +66,7 @@ function assert(c, m) { if (!c) throw new Error(m || 'assertion failed'); } }); await step('every hop is an inferred entry: a likelihood or an unlisted reason', async () => { + assert(detailHops.length === 22, 'expected the complete observation path'); const bad = detailHops.filter(h => !/% likely|no listed node|no known node|no GPS/.test(h.title)); assert(!bad.length, 'hops without an inferred title: ' + JSON.stringify(bad.slice(0, 3))); }); @@ -75,7 +89,7 @@ function assert(c, m) { if (!c) throw new Error(m || 'assertion failed'); } }); await step('the map route view draws the same hops, unlisted ones as placeholders', async () => { - await page.goto(`${BASE}/#/map?packet=${HASH}&obs=${OBS_ID}`, { waitUntil: 'domcontentloaded' }); + await page.goto(`${BASE}/#/map?packet=${hash}&obs=${OBS_ID}`, { waitUntil: 'domcontentloaded' }); await page.waitForFunction(() => document.querySelectorAll('.mc-rt-row').length >= 22, null, { timeout: 15000 }); const rows = await page.evaluate(() => [...document.querySelectorAll('.mc-rt-row')].map(r => r.innerText)); for (let i = 0; i < detailHops.length; i++) { diff --git a/tests/e2e/test-postgres-backup-e2e.js b/tests/e2e/test-postgres-backup-e2e.js new file mode 100644 index 000000000..694437f18 --- /dev/null +++ b/tests/e2e/test-postgres-backup-e2e.js @@ -0,0 +1,128 @@ +/** + * Native SQLite/PostgreSQL backup and restored-browser-session proof. + * Run test-user-management-e2e.js first to create its synthetic accounts. + * + * BASE_URL=http://localhost:13582 E2E_BACKUP_DIR= + * node tests/e2e/test-postgres-backup-e2e.js capture + * Restore telemetry.dump and accounts.dump into fresh PostgreSQL databases, + * grant the restricted runtime roles, and restart the fixture server against + * those databases. Then use the SAME directory and local browser origin: + * node tests/e2e/test-postgres-backup-e2e.js verify + * + * The directory contains account snapshots and session cookies. Keep it private + * and outside the checkout; never publish it as a CI artifact. + */ +'use strict'; +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const { isDeepStrictEqual } = require('node:util'); +const { chromium } = require('playwright'); + +const BASE = process.env.BASE_URL || 'http://localhost:13582'; +const DIR = process.env.E2E_BACKUP_DIR || ''; +const MODE = process.argv[2]; +const BACKEND = process.env.CORESCOPE_TEST_BACKEND || 'postgres'; +assert(['sqlite','postgres'].includes(BACKEND), 'explicit supported backup backend required'); +const EXTENSION = BACKEND === 'sqlite' ? 'db' : 'dump'; +const HASH = 'fae0c9e6d357a814'; +const ACCOUNTS = ['admin@e2e.test', 'sync@e2e.test']; +let passed = 0; + +function pass(name) { passed++; console.log(' ✓ ' + name); } +function save(name, value) { + fs.writeFileSync(path.join(DIR, name), JSON.stringify(value), { mode: 0o600, flag: 'wx' }); +} +async function json(page, route) { + const response = await page.request.get(BASE + route); + assert.equal(response.status(), 200, route + ' status'); + return response.json(); +} +async function telemetry(page) { + const stats = await json(page, '/api/stats'); + const counts = {}; + for (const key of ['totalTransmissions', 'totalObservations', 'totalNodes', 'totalObservers']) { + assert(Number.isInteger(stats[key]) && stats[key] > 0, key + ' must contain fixture data'); + counts[key] = stats[key]; + } + const detail = await json(page, '/api/packets/' + HASH); + assert.equal(detail.packet.hash, HASH); + assert.equal(detail.observations.length, 3, 'seeded observation count'); + return { counts, hash: detail.packet.hash, observations: detail.observations.map(o => ({ + id: o.id, raw_hex: o.raw_hex, path_json: o.path_json, + })).sort((a, b) => a.id - b.id) }; +} + +(async () => { + assert(['capture', 'verify'].includes(MODE), 'use capture or verify'); + assert(['localhost', '127.0.0.1', '[::1]'].includes(new URL(BASE).hostname), 'local fixture server required'); + assert(path.isAbsolute(DIR), 'E2E_BACKUP_DIR must be a private absolute directory'); + const repo = path.resolve(__dirname, '../..'); + const relative = path.relative(repo, DIR); + assert(relative.startsWith('..' + path.sep) || path.isAbsolute(relative), 'backup directory must be outside the checkout'); + fs.mkdirSync(DIR, { recursive: true, mode: 0o700 }); + const expected = MODE === 'verify' ? JSON.parse(fs.readFileSync(path.join(DIR, 'expected.json'), 'utf8')) : { accounts: [] }; + const browser = await chromium.launch({ headless: true, + executablePath: process.env.CHROMIUM_PATH || undefined, + args: ['--no-sandbox', '--disable-gpu', '--disable-dev-shm-usage'] }); + try { + const pages = []; + for (let i = 0; i < ACCOUNTS.length; i++) { + const statePath = path.join(DIR, 'browser-' + i + '.json'); + const context = await browser.newContext(MODE === 'verify' ? { storageState: statePath } : {}); + const page = await context.newPage(); + page.setDefaultTimeout(10000); + if (MODE === 'capture') { + await page.goto(BASE + '/#/account/login', { waitUntil: 'domcontentloaded' }); + await page.fill('#loginEmail', ACCOUNTS[i]); + await page.fill('#loginPassword', 'correct horse battery'); + await page.click('#loginForm button[type="submit"]'); + } else { + // No login request: the pre-backup cookie must resolve the restored session. + await page.goto(BASE + '/#/account', { waitUntil: 'domcontentloaded' }); + } + await page.waitForSelector('#profileForm'); + await page.evaluate(() => window.CSAuth.ready()); + if (MODE === 'capture') await page.evaluate(() => window.CSSettingsSync.syncNow()); + const identity = await json(page, '/api/auth/me'); + assert.equal(identity.email, ACCOUNTS[i]); + const settings = (await json(page, '/api/account/settings')).doc; + if (MODE === 'capture') { + expected.accounts.push({ identity, settings }); + save('browser-' + i + '.json', await context.storageState()); + } else { + // Keep session tokens and account contents out of failure output. + assert(isDeepStrictEqual(identity, expected.accounts[i].identity), 'restored identity and CSRF token'); + assert(isDeepStrictEqual(settings, expected.accounts[i].settings), 'restored account settings'); + } + pages.push(page); + pass(MODE + ': ' + (i === 0 ? 'admin' : 'user') + ' session and settings'); + } + assert.equal(expected.accounts[1].settings.keys['meshcore-time-window'], '180', 'synced fixture setting'); + const actualTelemetry = await telemetry(pages[0]); + if (MODE === 'capture') expected.telemetry = actualTelemetry; + else assert(isDeepStrictEqual(actualTelemetry, expected.telemetry), 'restored telemetry API identity and observation bytes'); + pass(MODE + ': telemetry counts and distinct observation bytes'); + + const anonymous = await (await browser.newContext()).newPage(); + // Telemetry falls back to the API-key gate; account backups use withAdmin. + for (const [route, filename, anonymousStatus] of [['/api/backup', 'telemetry.' + EXTENSION, 403], ['/api/admin/users-backup', 'accounts.' + EXTENSION, 401]]) { + assert.equal((await anonymous.request.get(BASE + route)).status(), anonymousStatus, route + ' anonymous denial'); + assert.equal((await pages[1].request.get(BASE + route)).status(), 403, route + ' non-admin denial'); + const response = await pages[0].request.get(BASE + route); + assert.equal(response.status(), 200, route + ' admin download'); + assert.match(response.headers()['content-disposition'], new RegExp('^attachment; filename="corescope-[^"]+\\.' + EXTENSION + '"$')); + assert.equal(response.headers()['content-type'], 'application/octet-stream'); + assert.equal(response.headers()['cache-control'], 'no-store'); + const body = await response.body(); + const magic = BACKEND === 'sqlite' ? 'SQLite format 3\0' : 'PGDMP'; + assert.equal(body.subarray(0, Buffer.byteLength(magic)).toString(), magic, route + ' native archive magic'); + if (MODE === 'capture') fs.writeFileSync(path.join(DIR, filename), body, { mode: 0o600, flag: 'wx' }); + pass(MODE + ': protected native ' + filename + ' download'); + } + if (MODE === 'capture') save('expected.json', expected); + } finally { + await browser.close(); + } + console.log('\n' + passed + '/' + passed + ' tests passed'); +})().catch(err => { console.error('PostgreSQL backup E2E failed: ' + err.message); process.exitCode = 1; }); diff --git a/tests/e2e/test-user-management-e2e.js b/tests/e2e/test-user-management-e2e.js index 72d47b574..2a779f533 100644 --- a/tests/e2e/test-user-management-e2e.js +++ b/tests/e2e/test-user-management-e2e.js @@ -3,15 +3,18 @@ * BASE_URL server with userManagement on + fake mailer (-tags e2etest build) * BASE_URL_OFF the regular fixture server (feature off) * - * Local run (never point the servers at the tracked fixture; migrate a copy): - * cp test-fixtures/e2e-fixture.db "$TMP/on.db"; cp test-fixtures/e2e-fixture.db "$TMP/off.db" - * corescope-migrate -db "$TMP/on.db"; corescope-migrate -db "$TMP/off.db" + * Local run: prepare/freshen a disposable copy of the tracked SQLite fixture, + * apply the CI synthetic seeds and migrate-fixture-hashes.go, then import it + * with corescope-migrate -offline -from-sqlite . Use owner URLs only for + * migration; grant a read-only telemetry role and a separate account writer. * (cd cmd/server && go build -o ../../corescope-server . && go build -tags e2etest -o ../../corescope-server-e2e .) * # config.json for the on-server (in $CFGDIR): port 13582, userManagement {enabled: true, - * # dbPath "users.db", adminEmails ["admin@e2e.test"], publicBaseUrl "http://localhost:13582", + * # adminEmails ["admin@e2e.test"], publicBaseUrl "http://localhost:13582", * # mail {provider "fake", fromEmail "noreply@e2e.test"}, channelProposals {enabled: true}, notifications {enabled: true}} - * corescope-server -port 13581 -db "$TMP/off.db" -public public & - * (cd "$CFGDIR" && corescope-server-e2e -config-dir . -port 13582 -db "$TMP/on.db" -public /public) & + * # Set CORESCOPE_READER_DATABASE_URL for both processes and + * # CORESCOPE_USERS_DATABASE_URL for the on-server only (fresh account DB). + * corescope-server -port 13581 -state-dir "$TMP/off-state" -public public & + * (cd "$CFGDIR" && corescope-server-e2e -config-dir . -port 13582 -state-dir "$TMP/on-state" -public /public) & * BASE_URL=http://localhost:13582 BASE_URL_OFF=http://localhost:13581 node tests/e2e/test-user-management-e2e.js * Set CHROMIUM_PATH to a Chrome/Chromium binary if Playwright's own is not installed. */ diff --git a/tests/helpers/manage-shell.js b/tests/helpers/manage-shell.js new file mode 100644 index 000000000..a66518e3d --- /dev/null +++ b/tests/helpers/manage-shell.js @@ -0,0 +1,33 @@ +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); + +const root = path.resolve(__dirname, '../..'); +const bash = require('../../scripts/bash-path')(); +const quote = value => "'" + value.replace(/'/g, "'\\''") + "'"; +const posix = value => value.replace(/\\/g, '/'); + +// Source the complete production script from its real path. Only external +// commands supplied by each test are replaced; management functions stay real. +module.exports = function manageFixture() { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'corescope-manage-test-')); + return { + dir, + run(before = '', after = '') { + return spawnSync(bash, ['--noprofile', '--norc'], { + input: `set -e\n${before}\nsource ${quote(posix(path.join(root, 'manage.sh')))} help >/dev/null\n${after}\n`, + cwd: dir, + env: { ...process.env, PROD_DATA_DIR: posix(path.join(dir, 'data')), STAGING_DATA_DIR: posix(path.join(dir, 'staging')) }, + encoding: 'utf8', + timeout: 20000, + }); + }, + cleanup() { + assert(path.resolve(dir).startsWith(path.resolve(os.tmpdir()) + path.sep)); + assert(path.basename(dir).startsWith('corescope-manage-test-')); + fs.rmSync(dir, { recursive: true }); + }, + }; +}; diff --git a/tests/unit/test-deployment-readiness.js b/tests/unit/test-deployment-readiness.js new file mode 100644 index 000000000..144b1f296 --- /dev/null +++ b/tests/unit/test-deployment-readiness.js @@ -0,0 +1,48 @@ +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const manageFixture = require('../helpers/manage-shell'); +const root = path.resolve(__dirname, '../..'); + +const bases = ['docker-compose.yml', 'docker-compose.no-mosquitto.yml', 'docker-compose.example.yml', + 'docker-compose.staging.yml', 'docker-compose.staging.no-mosquitto.yml']; +for (const file of bases) { + const source = fs.readFileSync(path.join(root, file), 'utf8'); + const command = source.match(/healthcheck:\s*\n\s*test:\s*(\[[^\n]+\])/); + assert(command, `${file} has no health command`); + assert.equal(JSON.parse(command[1]).at(-1), 'http://localhost:3000/api/healthz', `${file} accepts a partially loaded server`); +} + +for (const becomesReady of [true, false]) { + const fixture = manageFixture(); + try { + fs.mkdirSync(path.join(fixture.dir, 'caddy-config')); + fs.writeFileSync(path.join(fixture.dir, 'caddy-config/Caddyfile'), 'scope.example.org {\n}\n'); + const result = fixture.run(` +git(){ printf 'test-revision\\n'; } +docker(){ + case "$*" in + 'compose version') : ;; + *'/api/stats') printf 'stats\\n' >> probes; return 0 ;; + *'/api/healthz') + printf 'ready\\n' >> probes + [ ${becomesReady} = true ] && [ "$(wc -l < probes)" -ge 3 ] ;; + logs*) : ;; + *) return 1 ;; + esac +} +sleep(){ :; } +curl(){ printf '%s\\n' "$*" > https-probe; } +`, 'verify_health'); + assert.equal(result.status, becomesReady ? 0 : 1, result.stderr || result.stdout); + const probes = fs.readFileSync(path.join(fixture.dir, 'probes'), 'utf8').trim().split('\n'); + assert.equal(probes.length, becomesReady ? 3 : 45, 'readiness was not polled to success or timeout'); + assert(probes.every(p => p === 'ready'), 'verification used statistics as readiness'); + if (becomesReady) { + assert.match(fs.readFileSync(path.join(fixture.dir, 'https-probe'), 'utf8'), /https:\/\/scope\.example\.org\/api\/healthz/); + } + } finally { + fixture.cleanup(); + } +} +console.log('PASS all Compose healthchecks and management verification wait for readiness'); diff --git a/tests/unit/test-issue-1648-m2-emoji-scan.js b/tests/unit/test-issue-1648-m2-emoji-scan.js index beb98ebaa..a4997b2ef 100644 --- a/tests/unit/test-issue-1648-m2-emoji-scan.js +++ b/tests/unit/test-issue-1648-m2-emoji-scan.js @@ -48,7 +48,7 @@ const MIN_USE_REFS = { 'map.js': 10, 'node-analytics.js': 4, 'traces.js': 1, - 'perf.js': 9, + 'perf.js': 8, // PostgreSQL replaces the SQLite-only WAL-size warning card. 'audio-lab.js': 5, }; diff --git a/tests/unit/test-issue-1956-release-routing.js b/tests/unit/test-issue-1956-release-routing.js index bb520ee64..51c24963a 100644 --- a/tests/unit/test-issue-1956-release-routing.js +++ b/tests/unit/test-issue-1956-release-routing.js @@ -11,13 +11,7 @@ const { spawnSync } = require('node:child_process'); const read = name => fs.readFileSync(path.join(REPO_ROOT, '.github/workflows', name), 'utf8').replace(/\r/g, ''); const fast = read('release-fast-path.yml'); const deploy = read('deploy.yml'); -let bash = process.env.BASH_PATH || 'bash'; -if (!process.env.BASH_PATH && process.platform === 'win32') { - // Prefer Git Bash over Windows' WSL launcher; CI uses the native Linux bash. - const git = spawnSync('git', ['--exec-path'], { encoding: 'utf8' }); - const gitBash = path.resolve((git.stdout || '').trim(), '../../../bin/bash.exe'); - if (git.status === 0 && fs.existsSync(gitBash)) bash = gitBash; -} +const bash = require('../../scripts/bash-path')(); // Extract known YAML blocks, retaining the actual expressions and shell code. // Full YAML syntax is separately checked by actionlint; no YAML dependency here. @@ -45,7 +39,8 @@ const steps = source => source.split(/(?=^ - name:)/m).slice(1); function evaluate(expression, context) { if (!expression) return true; return vm.runInNewContext(expression.replace(/^\$\{\{|\}\}$/g, '').trim(), { - ...context, startsWith: (text, prefix) => text.startsWith(prefix), cancelled: () => false + ...context, startsWith: (text, prefix) => text.startsWith(prefix), cancelled: () => false, + contains: (text, part) => String(text ?? '').toLowerCase().includes(String(part).toLowerCase()) }); } const expand = (script, context) => script.replace(/\$\{\{(.*?)\}\}/g, (_, expression) => String(evaluate(expression, context))); @@ -56,13 +51,13 @@ function runSteps(source, context, edge, mutateFails = false) { const output = path.join(dir, 'output'); const log = path.join(dir, 'commands'); fs.writeFileSync(log, ''); - fs.mkdirSync(path.join(dir, 'cmd/decrypt'), { recursive: true }); + for (const name of ['decrypt', 'migrate']) fs.mkdirSync(path.join(dir, 'cmd', name), { recursive: true }); fs.mkdirSync(path.join(dir, 'docs/release-notes'), { recursive: true }); fs.writeFileSync(path.join(dir, 'docs/release-notes/v9.8.7.md'), 'Fixture release notes\n'); // go() only logs, so nothing real is produced; the release job's own // static/runnable verification step still needs these to exist. - for (const arch of ['amd64', 'arm64']) { - fs.writeFileSync(path.join(dir, `corescope-decrypt-linux-${arch}`), + for (const arch of ['amd64', 'arm64']) for (const name of ['decrypt', 'migrate']) { + fs.writeFileSync(path.join(dir, `corescope-${name}-linux-${arch}`), '#!/bin/sh\necho "corescope-decrypt stub"\n', { mode: 0o755 }); } const stubs = ` @@ -132,8 +127,7 @@ function runSteps(source, context, edge, mutateFails = false) { const result = spawnSync(bash, ['--noprofile', '--norc', '-e', '-o', 'pipefail'], { input: stubs + '\n' + expand(script, context), cwd: dir, encoding: 'utf8', timeout: 15000, env: { - ...process.env, GITHUB_REF: context.github.ref, GITHUB_SHA: context.github.sha, - GITHUB_REF_NAME: context.github.ref_name, + ...process.env, CC: '', GITHUB_REF: context.github.ref, GITHUB_REF_NAME: context.github.ref_name, GITHUB_SHA: context.github.sha, GITHUB_OUTPUT: bashPath(output), COMMAND_LOG: bashPath(log), TMPDIR: bashPath(dir), EDGE_CONFIG: edge === null ? 'missing' : JSON.stringify({ config: { Labels: { 'org.opencontainers.image.revision': edge } } }), // :edge is a two-platform index plus the two buildx attestation @@ -178,7 +172,7 @@ function context(ref = 'refs/tags/v9.8.7', event = 'workflow_dispatch', inputs = // That is what keeps a gate red rather than skipped, which branch protection // would count as passing. function route(ctx, failedJob, ingestor = 'true') { - for (const name of ['changes', 'go-test', 'race-test', 'e2e-shard', 'e2e-test', 'image-check', 'build-and-publish', 'release-artifacts', 'deploy', 'publish']) { + for (const name of ['changes', 'go-backend', 'go-test', 'race-backend', 'race-test', 'e2e-shard', 'e2e-test', 'image-check', 'build-and-publish', 'release-artifacts', 'deploy', 'publish']) { const job = block(deploy, name, 2); assert.ok(job, `missing ${name} job`); const needs = value(job, 'needs', 4).replace(/[\[\]\s]/g, '').split(',').filter(Boolean); @@ -201,6 +195,17 @@ function route(ctx, failedJob, ingestor = 'true') { return ctx.needs; } +// SQLite must have no PostgreSQL service; selected PG must get the pinned one. +const serviceImage = value(block(deploy, 'postgres', 6), 'image', 8); +assert.equal(evaluate(serviceImage.startsWith('${{') ? serviceImage : JSON.stringify(serviceImage), { matrix: { backend: 'sqlite' } }), '', 'default SQLite CI starts a PostgreSQL service'); +assert.equal(evaluate(serviceImage.startsWith('${{') ? serviceImage : JSON.stringify(serviceImage), { matrix: { backend: 'postgres' } }), 'postgres:18.6-alpine3.24'); +for (const [gate, worker] of [['go-test','go-backend'], ['race-test','race-backend']]) { + const definition = block(deploy, worker, 2); + assert.ok(definition.includes('backend: [sqlite, postgres]'), `${worker} must exercise both native backends`); + assert.equal(value(definition, 'CORESCOPE_TEST_BACKEND', 6), '${{ matrix.backend }}'); + assert.ok(value(block(deploy, gate, 2), 'needs', 4).includes(worker), `${gate} does not wait for both backends`); +} + for (const [name, edge] of [['matching', 'a'.repeat(40)], ['missing', null], ['mismatched', 'b'.repeat(40)]]) { const ctx = context(); const { commands } = runSteps(block(fast, 'retag-or-fallback', 2), ctx, edge); @@ -262,7 +267,14 @@ for (const [ref, event] of [['refs/heads/master', 'push'], ['refs/heads/master', // The cross-toolchain gate: since the SQLite driver became cgo, every // event must build both architectures, without publishing, beside the tests. const checkSteps = steps(block(deploy, 'image-check', 2)).filter(step => step.includes('uses: docker/build-push-action')); - assert.equal(checkSteps.length, 1, 'image-check runs exactly one build'); + assert.equal(checkSteps.length, 2, 'image-check has one two-arch build and one cached amd64 load'); + for (const step of checkSteps) assert.equal(value(step, 'push', 10), 'false', 'neither image-check build may publish'); + assert.equal(value(checkSteps[1], 'platforms', 10), 'linux/amd64', 'Compose smoke must load the native runner architecture only'); + assert.equal(value(checkSteps[1], 'load', 10), 'true', 'Compose smoke requires a locally loaded image'); + assert.equal(value(checkSteps[1], 'context', 10), value(checkSteps[0], 'context', 10), 'smoke must build the same source'); + assert.equal(value(checkSteps[1], 'build-args', 10), value(checkSteps[0], 'build-args', 10), 'smoke must reuse identical version/commit/time metadata'); + assert.equal(value(checkSteps[1], 'cache-from', 10), value(checkSteps[0], 'cache-from', 10), 'smoke must reuse the validated build cache'); + assert.equal(value(checkSteps[1], 'if', 8), '', 'packaged smoke must run on every image-check event'); assert.equal(value(checkSteps[0], 'push', 10), 'false', 'the image check must never publish'); assert.equal(value(checkSteps[0], 'platforms', 10), 'linux/amd64,linux/arm64', 'the image check must cover both shipped architectures'); assert.equal(value(checkSteps[0], 'if', 8), '', 'the image check runs on every event'); @@ -283,6 +295,8 @@ for (const [ref, event] of [['refs/heads/master', 'push'], ['refs/heads/master', assert.equal(noRace['build-and-publish'].result, 'success', `${event}: a skipped race-test must not block`); } assert.equal(route(context(), 'go-test')['release-artifacts'].result, 'skipped', 'failed Go validation must block release'); +assert.equal(route(context(), 'go-backend')['go-test'].result, 'failure', 'failed backend matrix must fail the stable Go gate'); +assert.equal(route(context(), 'race-backend')['race-test'].result, 'failure', 'failed race matrix must fail the stable race gate'); const dispatchInput = block(deploy, 'images_published', 6); assert.equal(value(dispatchInput, 'type', 8), 'boolean', 'dispatch flag must retain boolean semantics'); assert.equal(value(dispatchInput, 'default', 8), 'false', 'manual and fallback dispatches must build images by default'); @@ -305,21 +319,26 @@ try { else process.env.GITHUB_REF_NAME = inheritedRefName; } const builds = runSteps(release, context(), null).commands.filter(command => command[0] === 'go'); -// CGO_ENABLED=1 since the SQLite driver became github.com/mattn/go-sqlite3, and -// CC must be zig targeting musl — that is what makes the artifact static and -// cross-buildable. A silent revert to the Go-only toolchain fails here. +// Exercise the actual release shell: both tools support native SQLite and use +// the same static musl cross-compiler for both architectures. assert.deepEqual(builds.map(command => command.slice(1, 5)), [ ['linux', 'amd64', '1', 'zig cc -target x86_64-linux-musl'], + ['linux', 'amd64', '1', 'zig cc -target x86_64-linux-musl'], + ['linux', 'arm64', '1', 'zig cc -target aarch64-linux-musl'], ['linux', 'arm64', '1', 'zig cc -target aarch64-linux-musl'], ]); for (const command of builds) { - assert.ok(command.includes("-ldflags=-s -w -extldflags '-static -Wl,-s' -X main.version=v9.8.7"), 'binary version must come from tag, and the artifact must stay static'); - assert.ok(command.includes('-tags'), 'netgo/osusergo/sqlite_omit_load_extension must survive'); + const importer = command.some(arg => arg.includes('corescope-migrate-linux-')); + assert.ok(command.includes('netgo,osusergo,sqlite_omit_load_extension'), 'SQLite extension loading must be omitted for static linking'); + assert.ok(command.includes(importer + ? "-ldflags=-s -w -extldflags '-static -Wl,-s'" + : "-ldflags=-s -w -extldflags '-static -Wl,-s' -X main.version=v9.8.7"), 'static linking/version flags must match the tool'); + assert.ok(command.includes('-tags'), 'runtime resolver tags must survive'); } const upload = steps(release).filter(step => step.includes('uses: softprops/action-gh-release@v2')); assert.equal(upload.length, 1, 'publish both architectures together, before the release becomes immutable'); assert.equal(value(upload[0], 'fail_on_unmatched_files', 10), 'true', 'missing assets must prevent publication'); -assert.deepEqual(value(upload[0], 'files', 10).trim().split('\n').map(line => line.trim()), ['corescope-decrypt-linux-amd64', 'corescope-decrypt-linux-arm64']); +assert.deepEqual(value(upload[0], 'files', 10).trim().split('\n').map(line => line.trim()), ['corescope-decrypt-linux-amd64', 'corescope-decrypt-linux-arm64', 'corescope-migrate-linux-amd64', 'corescope-migrate-linux-arm64']); assert.equal(value(upload[0], 'draft', 10), '', 'standard release action must finalize after both uploads'); assert.equal(value(upload[0], 'prerelease', 10), '', 'standard release action must upload before publishing'); const checkout = steps(release).find(step => step.includes('uses: actions/checkout@')); @@ -345,3 +364,132 @@ console.log('PASS failed retag/Go gates, branch/PR routes, and complete tagged r assert.deepEqual(commands.filter(command => command[0] === 'crane' && command[1] === 'tag').map(command => command.at(-1)), ['v9.8', 'v9', 'latest']); console.log('PASS dispatched republish: tags rebuilt, no second release dispatch'); } + +// Benchmark dispatches cannot publish or deploy, even on trusted master with +// staging enabled. Ordinary dispatch routing above remains unchanged. +for (const ref of ['refs/heads/master', 'refs/heads/codex/postgres', 'refs/tags/v9.8.7']) { + const ctx = context(ref, 'workflow_dispatch', { postgres_benchmark: true, candidate_sha: 'a'.repeat(40) }); + ctx.vars.ENABLE_STAGING_DEPLOY = 'true'; + const jobs = route(ctx); + for (const name of ['build-and-publish', 'release-artifacts', 'deploy', 'publish']) { + assert.equal(jobs[name].result, 'skipped', 'benchmark dispatch must skip ' + name); + } + assert.equal(evaluate(value(block(deploy, 'postgres-benchmark', 2), 'if', 4), ctx), true); +} +const benchmarkCall = block(deploy, 'postgres-benchmark', 2); +assert.ok(!benchmarkCall.includes('secrets:') && !benchmarkCall.includes('environment:'), 'benchmark must receive no deployment secrets/environment'); +console.log('PASS benchmark-only dispatch cannot publish or deploy'); + +// PR benchmarks are deliberate opt-ins and consume the head repository/SHA, +// while ordinary CI continues to test GitHub's merge ref. +const benchmarkWorkflow = read('postgres-benchmark.yml'); +const prBenchmark = context('refs/pull/1/merge', 'pull_request'); +prBenchmark.github.workflow_sha = 'c'.repeat(40); +prBenchmark.github.event = { pull_request: { + body: '- [x] Run PostgreSQL comparison', + head: { sha: 'b'.repeat(40), repo: { full_name: 'contributor/project' } } +} }; +for (const [body, code, expected] of [ + ['- [x] Run PostgreSQL comparison', 'true', true], + ['- [X] Run PostgreSQL comparison', 'true', true], + ['- [ ] Run PostgreSQL comparison', 'true', false], + ['', 'true', false], + [null, 'true', false], + ['- [x] Run PostgreSQL comparison', 'false', false] +]) { + prBenchmark.github.event.pull_request.body = body; + prBenchmark.needs.changes = { result: 'success', outputs: { code } }; + assert.equal(evaluate(value(benchmarkCall, 'if', 4), prBenchmark), expected, 'PR checkbox/code-scope routing'); +} +assert.equal(value(benchmarkCall, 'needs', 4), '[changes]', 'PR benchmarks must wait for code scope'); +const dispatchBenchmark = context('refs/heads/candidate', 'workflow_dispatch', { postgres_benchmark: true, candidate_sha: 'a'.repeat(40) }); +dispatchBenchmark.needs.changes = { result: 'success', outputs: { code: 'false' } }; +assert.equal(evaluate(value(benchmarkCall, 'if', 4), dispatchBenchmark), true, 'benchmark-only dispatch must survive its normal code=false output'); +assert.equal(evaluate(value(block(benchmarkCall, 'with', 4), 'candidate_sha', 6), prBenchmark), prBenchmark.github.event.pull_request.head.sha, 'benchmark candidate must be the PR head, not merge SHA'); +assert.equal(value(block(benchmarkCall, 'permissions', 4), 'contents', 6), 'read'); +const benchmarkSteps = steps(block(benchmarkWorkflow, 'benchmark', 2)); +const candidateCheckout = benchmarkSteps.find(step => value(step, '- name', 6) === 'Checkout the exact candidate'); +assert.equal(value(candidateCheckout, 'persist-credentials', 10), 'false'); +assert.equal(evaluate(value(candidateCheckout, 'repository', 10), prBenchmark), 'contributor/project'); +assert.equal(evaluate(value(candidateCheckout, 'repository', 10), context()), 'example/corescope'); +assert.equal(value(candidateCheckout, 'ref', 10), '${{ inputs.candidate_sha }}'); +assert.equal(value(block(benchmarkWorkflow, 'permissions', 0), 'contents', 2), 'read'); + +// Execute the real snapshot guard with only Git's external boundary stubbed. +// A PR's merge SHA differs from the benchmark head; mismatched code/workflows +// must still fail before package installation or benchmark execution. +const snapshotStep = benchmarkSteps.find(step => value(step, '- name', 6) === 'Verify candidate and workflow snapshot'); +function snapshotGuard({ candidate = 'b'.repeat(40), checkout = candidate, expected = candidate, merge = 'a'.repeat(40), matches = true } = {}) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'corescope-benchmark-guard-')); + const log = path.join(dir, 'git-calls'); + fs.writeFileSync(log, ''); + try { + const stubs = ` + git() { + printf '%s\\n' "$*" >> "$CHECK_LOG" + case "$1" in + rev-parse) printf '%s\\n' "$CHECKOUT_SHA" ;; + fetch|cat-file) return 0 ;; + diff) [ "$WORKFLOW_MATCHES" = true ] ;; + *) return 99 ;; + esac + } + `; + const result = spawnSync(bash, ['--noprofile', '--norc'], { + input: stubs + value(snapshotStep, 'run', 8), cwd: dir, encoding: 'utf8', + env: { ...process.env, CANDIDATE_SHA: candidate, CHECKOUT_SHA: checkout, + EXPECTED_SHA: expected, GITHUB_SHA: merge, WORKFLOW_SHA: 'c'.repeat(40), + GITHUB_REPOSITORY: 'example/corescope', GITHUB_SERVER_URL: 'https://github.com', + WORKFLOW_MATCHES: String(matches), CHECK_LOG: bashPath(log) } + }); + return { status: result.status, error: result.stderr, calls: fs.readFileSync(log, 'utf8') }; + } finally { fs.rmSync(dir, { recursive: true, force: true }); } +} +const acceptedHead = snapshotGuard(); +assert.equal(acceptedHead.status, 0, 'PR head must be accepted despite a distinct merge SHA: ' + acceptedHead.error); +assert.ok(acceptedHead.calls.includes('fetch --no-tags --depth=1 https://github.com/example/corescope.git ' + 'c'.repeat(40)), 'verify the actual caller workflow commit'); +for (const path of ['.github/workflows/deploy.yml', '.github/workflows/postgres-benchmark.yml', 'scripts/postgres-benchmark', 'scripts/install-postgres-ci.sh']) { + assert.ok(acceptedHead.calls.includes(path), 'snapshot comparison omitted ' + path); +} +assert.notEqual(snapshotGuard({ matches: false }).status, 0, 'different workflow/harness content accepted'); +assert.notEqual(snapshotGuard({ checkout: 'd'.repeat(40) }).status, 0, 'wrong checkout accepted'); +assert.notEqual(snapshotGuard({ expected: 'd'.repeat(40) }).status, 0, 'candidate differs from event head'); +assert.equal(snapshotGuard({ candidate: 'a'.repeat(40), merge: 'a'.repeat(40) }).status, 0, 'ordinary exact-SHA dispatch failed'); +const snapshotEnv = block(snapshotStep, 'env', 8); +assert.equal(evaluate(value(snapshotEnv, 'EXPECTED_SHA', 10), prBenchmark), 'b'.repeat(40)); +assert.equal(evaluate(value(snapshotEnv, 'EXPECTED_SHA', 10), context()), 'a'.repeat(40)); +assert.equal(evaluate(value(snapshotEnv, 'WORKFLOW_SHA', 10), prBenchmark), 'c'.repeat(40)); +console.log('PASS opted-in PR benchmark uses exact head and matching workflow/harness, without persisted credentials'); + +// Execute the real restore-URL transformation. A role and its database can have +// the same name; replacing the first matching substring silently changes login. +const restoreStep = steps(block(deploy, 'e2e-shard', 2)).find(step => value(step, '- name', 6) === 'Restore native HTTP backups and verify browser sessions'); +const restoreScript = value(restoreStep, 'run', 8); +const switchURLs = restoreScript.split('# Switch restored database URLs.\n')[1]?.split('# End restored database URLs.')[0]; +assert.ok(switchURLs, 'missing restore URL transformation'); +const sourceURLs = { + CORESCOPE_USERS_DATABASE_URL: 'postgresql://corescope_accounts:p%40ss%2F%3A%3F%23@db.example.invalid:5433/corescope_accounts?sslmode=verify-full&application_name=corescope_accounts', + CORESCOPE_READER_DATABASE_URL: 'postgresql://corescope_reader:p%2540%3A%2F@[::1]:5432/corescope_telemetry?sslmode=require&application_name=reader%3Bsmoke' +}; +sourceURLs.CORESCOPE_WRITER_DATABASE_URL = sourceURLs.CORESCOPE_READER_DATABASE_URL.replace('corescope_reader:', 'corescope_writer:'); +sourceURLs.CORESCOPE_APPROVED_CHANNELS_DATABASE_URL = sourceURLs.CORESCOPE_USERS_DATABASE_URL.replace('corescope_accounts:', 'corescope_channels:'); +sourceURLs.CORESCOPE_DATABASE_URL = sourceURLs.CORESCOPE_READER_DATABASE_URL.replace('corescope_reader:', 'corescope_owner:'); +sourceURLs.CORESCOPE_USERS_OWNER_DATABASE_URL = sourceURLs.CORESCOPE_USERS_DATABASE_URL.replace('corescope_accounts:', 'corescope_owner:'); +const rewritten = spawnSync(bash, ['--noprofile', '--norc', '-e', '-o', 'pipefail'], { + input: (process.platform === 'win32' ? 'python3(){ python "$@"; }\n' : '') + switchURLs + + '\nprintf "%s\\n" ' + Object.keys(sourceURLs).map(key => '"$' + key + '"').join(' ') + '\n', + encoding: 'utf8', env: { ...process.env, ...sourceURLs } +}); +assert.equal(rewritten.status, 0, rewritten.stderr); +const resultURLs = rewritten.stdout.trim().split('\n'); +for (const [index, key] of Object.keys(sourceURLs).entries()) { + const original = new URL(sourceURLs[key]); + const restored = new URL(resultURLs[index]); + for (const field of ['protocol', 'username', 'password', 'host', 'search', 'hash']) { + assert.equal(restored[field], original[field], 'restore URL changed ' + field); + } + assert.equal(restored.pathname, /USERS|APPROVED_CHANNELS/.test(key) ? '/corescope_restore_accounts' : '/corescope_restore_telemetry'); +} +console.log('PASS native backup restore changes only database URL paths'); +const goMatrix = block(deploy, 'go-backend', 2); +assert(goMatrix.includes('cd internal/dbconfig') && goMatrix.includes('cd ../sqliteutil'), 'new shared storage modules need their own explicit tests'); diff --git a/tests/unit/test-perf-disk-io-1120.js b/tests/unit/test-perf-disk-io-1120.js index 35509fda4..14d025e20 100644 --- a/tests/unit/test-perf-disk-io-1120.js +++ b/tests/unit/test-perf-disk-io-1120.js @@ -1,4 +1,4 @@ -/* Tests for perf.js Disk I/O + Write Sources + SQLite sections (#1120) */ +/* Tests for perf.js Disk I/O + Write Sources + PostgreSQL sections (#1120) */ 'use strict'; const vm = require('vm'); const fs = require('fs'); @@ -45,12 +45,12 @@ function loadPerf() { return sb; } -function stubFetch(sb, perfData, healthData, ioData, sqliteData, sourcesData) { +function stubFetch(sb, perfData, healthData, ioData, postgresData, sourcesData) { sb.ctx.fetch = (url) => { if (url === '/api/perf') return Promise.resolve({ json: () => Promise.resolve(perfData) }); if (url === '/api/health') return Promise.resolve({ json: () => Promise.resolve(healthData) }); if (url === '/api/perf/io') return Promise.resolve({ json: () => Promise.resolve(ioData) }); - if (url === '/api/perf/sqlite') return Promise.resolve({ json: () => Promise.resolve(sqliteData) }); + if (url === '/api/perf/database') return Promise.resolve({ json: () => Promise.resolve(postgresData) }); if (url === '/api/perf/write-sources') return Promise.resolve({ json: () => Promise.resolve(sourcesData) }); return Promise.resolve({ json: () => Promise.resolve({}) }); }; @@ -58,7 +58,7 @@ function stubFetch(sb, perfData, healthData, ioData, sqliteData, sourcesData) { const basePerf = { totalRequests: 100, avgMs: 5, uptime: 3600, - slowQueries: [], endpoints: {}, cache: null, packetStore: null, sqlite: null + slowQueries: [], endpoints: {}, cache: null, packetStore: null, postgres: null }; const goRuntime = { goroutines: 17, numGC: 31, pauseTotalMs: 2.1, lastPauseMs: 0.03, @@ -70,9 +70,9 @@ const ioData = { readBytesPerSec: 1024, writeBytesPerSec: 2048, syscallsRead: 10, syscallsWrite: 20 }; -const sqliteData = { - walSizeMB: 12.3, walSize: 12900000, pageCount: 4096, pageSize: 4096, - cacheSize: 2000, cacheHitRate: 0.987 +const postgresData = { + engine: 'postgresql', databaseBytes: 12900000, openConnections: 4, inUseConnections: 2, + idleConnections: 2, connectionWaitCount: 3, connectionWaitMs: 1.25, cacheHitRate: 0.987 }; const sourcesData = { sources: { tx_inserted: 25, obs_inserted: 1787, backfill_path_json: 0, node_upserts: 329, observer_upserts: 1823, walCommits: 100 }, @@ -82,9 +82,29 @@ const sourcesData = { console.log('\n🧪 perf.js — Disk I/O + Write Sources (#1120)\n'); (async () => { +await test('Selected SQLite renders native diagnostics without PostgreSQL labels', async () => { + const sb = loadPerf(); + const fetched = []; + const sqlite = {engine:'sqlite', dbSizeMB:12, walSizeMB:2, freelistMB:1, + rows:{transmissions:3, observations:4, nodes:2, observers:1}}; + sb.ctx.fetch = url => { + fetched.push(url); + const data = url === '/api/perf' ? {...basePerf, database:sqlite, sqlite} : + url === '/api/perf/database' ? {engine:'sqlite', pageCount:300, pageSize:4096, cacheSize:-2000, walSizeMB:2, journalMode:'wal', plannerStats:true} : null; + return Promise.resolve({json:()=>Promise.resolve(data)}); + }; + sb.pages.perf.init({set innerHTML(v) {}}); + await new Promise(r=>setTimeout(r,30)); + const html=sb.getHtml(); + assert.ok(fetched.includes('/api/perf/database'), 'must fetch selected-backend diagnostics'); + assert.ok(!fetched.includes('/api/perf/postgres'), 'must not assume a PostgreSQL installation'); + assert.ok(html.includes('SQLite data') && html.includes('WAL Size') && html.includes('Planner Statistics'),html); + assert.ok(!html.includes('PostgreSQL'), 'SQLite must not render PostgreSQL labels'); +}); + await test('Renders Disk I/O section', async () => { const sb = loadPerf(); - stubFetch(sb, { ...basePerf, goRuntime }, goHealth, ioData, sqliteData, sourcesData); + stubFetch(sb, { ...basePerf, goRuntime }, goHealth, ioData, postgresData, sourcesData); await sb.pages.perf.init({ set innerHTML(v) {} }); await new Promise(r => setTimeout(r, 100)); const html = sb.getHtml(); @@ -94,7 +114,7 @@ await test('Renders Disk I/O section', async () => { await test('Renders Write Sources section with non-zero rates', async () => { const sb = loadPerf(); - stubFetch(sb, { ...basePerf, goRuntime }, goHealth, ioData, sqliteData, sourcesData); + stubFetch(sb, { ...basePerf, goRuntime }, goHealth, ioData, postgresData, sourcesData); await sb.pages.perf.init({ set innerHTML(v) {} }); await new Promise(r => setTimeout(r, 100)); const html = sb.getHtml(); @@ -103,13 +123,15 @@ await test('Renders Write Sources section with non-zero rates', async () => { assert.ok(html.includes('obs_inserted'), 'should list obs_inserted source'); }); -await test('Renders SQLite section with WAL + cache hit rate', async () => { +await test('Renders PostgreSQL connection and cache statistics', async () => { const sb = loadPerf(); - stubFetch(sb, { ...basePerf, goRuntime }, goHealth, ioData, sqliteData, sourcesData); + stubFetch(sb, { ...basePerf, goRuntime }, goHealth, ioData, postgresData, sourcesData); await sb.pages.perf.init({ set innerHTML(v) {} }); await new Promise(r => setTimeout(r, 100)); const html = sb.getHtml(); - assert.ok(/WAL/i.test(html), 'should show WAL info'); + assert.ok(html.includes('PostgreSQL'), 'should name the active database engine'); + assert.ok(html.includes('Connections in use'), 'should show PostgreSQL pool utilization'); + assert.ok(!html.includes('WAL Size') && !html.includes('Page Count'), 'must not invent SQLite metrics'); assert.ok(/Cache Hit/i.test(html) || /cacheHitRate/i.test(html), 'should show cache hit rate'); }); @@ -118,7 +140,7 @@ await test('Renders SQLite section with WAL + cache hit rate', async () => { await test('Renders cancelledWriteBytesPerSec for server process', async () => { const sb = loadPerf(); const io = { ...ioData, cancelledWriteBytesPerSec: 4096 }; - stubFetch(sb, { ...basePerf, goRuntime }, goHealth, io, sqliteData, sourcesData); + stubFetch(sb, { ...basePerf, goRuntime }, goHealth, io, postgresData, sourcesData); await sb.pages.perf.init({ set innerHTML(v) {} }); await new Promise(r => setTimeout(r, 100)); const html = sb.getHtml(); @@ -139,7 +161,7 @@ await test('Renders ingestor row alongside server row in Disk I/O', async () => syscallsWrite: 0, }, }; - stubFetch(sb, { ...basePerf, goRuntime }, goHealth, io, sqliteData, sourcesData); + stubFetch(sb, { ...basePerf, goRuntime }, goHealth, io, postgresData, sourcesData); await sb.pages.perf.init({ set innerHTML(v) {} }); await new Promise(r => setTimeout(r, 100)); const html = sb.getHtml(); @@ -147,34 +169,25 @@ await test('Renders ingestor row alongside server row in Disk I/O', async () => assert.ok(/1\.0\s*MB/.test(html), 'should render ingestor write 1 MB/s'); }); -await test('WAL >100 MB fires ⚠️ flag', async () => { +await test('Unavailable PostgreSQL cache statistics render as unknown', async () => { const sb = loadPerf(); - const sql = { ...sqliteData, walSizeMB: 150, walSize: 150 * 1048576 }; - stubFetch(sb, { ...basePerf, goRuntime }, goHealth, ioData, sql, sourcesData); + stubFetch(sb, { ...basePerf, goRuntime }, goHealth, ioData, { ...postgresData, cacheHitRate: null }, sourcesData); await sb.pages.perf.init({ set innerHTML(v) {} }); await new Promise(r => setTimeout(r, 100)); - const html = sb.getHtml(); - // The warning appears in the WAL Size card; assert proximity by extracting - // the WAL Size card's text content. - const walSection = html.match(/150\.0MB\s*]*>]*#ph-warning"/); - assert.ok(walSection, 'expected ⚠️ next to 150MB WAL value, html=' + html.slice(html.indexOf('WAL Size') - 200, html.indexOf('WAL Size') + 200)); + assert.ok(sb.getHtml().includes('—
Database Block Cache Hit Rate'), 'missing metrics must not become zero'); }); -await test('WAL <100 MB does NOT fire ⚠️ flag', async () => { +await test('PostgreSQL numeric string metrics render without an exception', async () => { const sb = loadPerf(); - const sql = { ...sqliteData, walSizeMB: 12.3 }; - stubFetch(sb, { ...basePerf, goRuntime }, goHealth, ioData, sql, sourcesData); + stubFetch(sb, { ...basePerf, goRuntime }, goHealth, ioData, { ...postgresData, cacheHitRate: '0.987', connectionWaitMs: '1.25' }, sourcesData); await sb.pages.perf.init({ set innerHTML(v) {} }); await new Promise(r => setTimeout(r, 100)); - const html = sb.getHtml(); - const walIdx = html.indexOf('WAL Size'); - const slice = html.slice(Math.max(0, walIdx - 200), walIdx); - assert.ok(!/12\.3MB\s*]*>]*#ph-warning"/.test(slice), 'expected NO warning icon next to 12.3MB WAL value'); + assert.ok(sb.getHtml().includes('98.7%') && sb.getHtml().includes('1.3ms'), 'string metrics must format numerically'); }); await test('Cache hit <90% fires ⚠️ flag', async () => { const sb = loadPerf(); - const sql = { ...sqliteData, cacheHitRate: 0.85 }; + const sql = { ...postgresData, cacheHitRate: 0.85 }; stubFetch(sb, { ...basePerf, goRuntime }, goHealth, ioData, sql, sourcesData); await sb.pages.perf.init({ set innerHTML(v) {} }); await new Promise(r => setTimeout(r, 100)); @@ -184,7 +197,7 @@ await test('Cache hit <90% fires ⚠️ flag', async () => { await test('Cache hit ≥90% does NOT fire ⚠️ flag', async () => { const sb = loadPerf(); - const sql = { ...sqliteData, cacheHitRate: 0.987 }; + const sql = { ...postgresData, cacheHitRate: 0.987 }; stubFetch(sb, { ...basePerf, goRuntime }, goHealth, ioData, sql, sourcesData); await sb.pages.perf.init({ set innerHTML(v) {} }); await new Promise(r => setTimeout(r, 100)); @@ -194,31 +207,28 @@ await test('Cache hit ≥90% does NOT fire ⚠️ flag', async () => { // === #1167 must-fix #7: threshold boundary cases === -await test('WAL exactly 100 MB does NOT fire ⚠️ (boundary, strict >)', async () => { +await test('Unavailable database diagnostics do not invent a PostgreSQL panel', async () => { const sb = loadPerf(); - const sql = { ...sqliteData, walSizeMB: 100 }; - stubFetch(sb, { ...basePerf, goRuntime }, goHealth, ioData, sql, sourcesData); + stubFetch(sb, { ...basePerf, goRuntime }, goHealth, ioData, null, sourcesData); await sb.pages.perf.init({ set innerHTML(v) {} }); await new Promise(r => setTimeout(r, 100)); - const html = sb.getHtml(); - const walIdx = html.indexOf('WAL Size'); - const slice = html.slice(Math.max(0, walIdx - 200), walIdx); - assert.ok(!/100\.0MB\s*]*>]*#ph-warning"/.test(slice), 'expected NO warning icon at exactly 100 MB WAL (boundary), slice=' + slice); + assert.ok(!sb.getHtml().includes('PostgreSQL connections'), 'unavailable statistics should stay unavailable'); }); -await test('WAL infinitesimally over 100 MB DOES fire ⚠️', async () => { +await test('PostgreSQL data panel reports counts without SQLite storage fields', async () => { const sb = loadPerf(); - const sql = { ...sqliteData, walSizeMB: 100.01 }; - stubFetch(sb, { ...basePerf, goRuntime }, goHealth, ioData, sql, sourcesData); + const postgres = { engine: 'postgresql', dbSizeMB: 42, rows: { transmissions: 30000, observations: 90000, nodes: 2000, observers: 32 } }; + stubFetch(sb, { ...basePerf, goRuntime, postgres }, goHealth, ioData, postgresData, sourcesData); await sb.pages.perf.init({ set innerHTML(v) {} }); await new Promise(r => setTimeout(r, 100)); const html = sb.getHtml(); - assert.ok(/100\.0MB\s*]*>]*#ph-warning"/.test(html), 'expected warning icon next to 100.0MB WAL value (just over threshold)'); + assert.ok(html.includes('PostgreSQL data') && html.includes('42MB') && html.includes((30000).toLocaleString()), 'native storage figures missing'); + assert.ok(!html.includes('Freelist') && !html.includes('WAL Size'), 'SQLite-only metrics must not survive the conversion'); }); await test('Cache hit exactly 90% does NOT fire ⚠️ (boundary, strict <)', async () => { const sb = loadPerf(); - const sql = { ...sqliteData, cacheHitRate: 0.90 }; + const sql = { ...postgresData, cacheHitRate: 0.90 }; stubFetch(sb, { ...basePerf, goRuntime }, goHealth, ioData, sql, sourcesData); await sb.pages.perf.init({ set innerHTML(v) {} }); await new Promise(r => setTimeout(r, 100)); @@ -228,7 +238,7 @@ await test('Cache hit exactly 90% does NOT fire ⚠️ (boundary, strict <)', as await test('Cache hit infinitesimally below 90% DOES fire ⚠️', async () => { const sb = loadPerf(); - const sql = { ...sqliteData, cacheHitRate: 0.8999 }; + const sql = { ...postgresData, cacheHitRate: 0.8999 }; stubFetch(sb, { ...basePerf, goRuntime }, goHealth, ioData, sql, sourcesData); await sb.pages.perf.init({ set innerHTML(v) {} }); await new Promise(r => setTimeout(r, 100)); @@ -236,6 +246,24 @@ await test('Cache hit infinitesimally below 90% DOES fire ⚠️', async () => { assert.ok(/90\.0%\s*]*>]*#ph-warning"/.test(html), 'expected warning icon next to 90.0% cache hit value (just under threshold)'); }); +await test('Stale PostgreSQL sample is labelled and retains verified counts', async () => { + const sb = loadPerf(); + const postgres = { engine: 'postgresql', dbSizeMB: 42, rows: { transmissions: 30000, observations: 90000, nodes: 2000, observers: 32 }, sampledAt: '2026-01-01T12:00:00Z', sampleIntervalSeconds: 30, stale: true, error: 'unavailable' }; + stubFetch(sb, { ...basePerf, goRuntime, postgres }, goHealth, ioData, { ...postgresData, sampledAt: postgres.sampledAt, sampleIntervalSeconds: 30, stale: true, cacheHitRate: null }, sourcesData); + await sb.pages.perf.init({ set innerHTML(v) {} }); + await new Promise(r => setTimeout(r, 100)); + assert.ok(sb.getHtml().includes('42MB') && sb.getHtml().includes('Last verified PostgreSQL sample'), 'stale data must remain labelled as a prior verified sample'); +}); + +await test('Initial PostgreSQL sampling failure does not invent zero storage', async () => { + const sb = loadPerf(); + const postgres = { engine: 'postgresql', dbSizeMB: null, rows: null, stale: true, error: 'unavailable' }; + stubFetch(sb, { ...basePerf, goRuntime, postgres }, goHealth, ioData, { ...postgresData, databaseBytes: null, cacheHitRate: null, stale: true }, sourcesData); + await sb.pages.perf.init({ set innerHTML(v) {} }); + await new Promise(r => setTimeout(r, 100)); + assert.ok(sb.getHtml().includes('PostgreSQL storage unavailable'), 'failed first sample must be explicit'); + assert.ok(!sb.getHtml().includes('0MB
DB Size'), 'unknown storage must not become zero'); +}); await test('Backfill anomaly: rate >10× its stable rolling baseline shows a warning', async () => { // The current detector compares a source with its own rolling baseline. // Prime a 60-second baseline at 1/s, then jump to 1000/s for the next tick. @@ -246,7 +274,7 @@ await test('Backfill anomaly: rate >10× its stable rolling baseline shows a war { sources: { tx_inserted: 405, backfill_path_json: 1060 }, sampleAt: '2026-01-01T00:01:01Z' }, ]; for (const sample of samples) { - stubFetch(sb, { ...basePerf, goRuntime }, goHealth, ioData, sqliteData, sample); + stubFetch(sb, { ...basePerf, goRuntime }, goHealth, ioData, postgresData, sample); await sb.pages.perf.init({ set innerHTML(v) {} }); await new Promise(r => setTimeout(r, 50)); } @@ -266,10 +294,10 @@ await test('Backfill anomaly: insufficient history suppresses the warning', asyn const t1 = '2026-01-01T00:00:01Z'; const phase1 = { sources: { tx_inserted: 5, backfill_path_json: 0 }, sampleAt: t0 }; const phase2 = { sources: { tx_inserted: 6, backfill_path_json: 1000 }, sampleAt: t1 }; - stubFetch(sb, { ...basePerf, goRuntime }, goHealth, ioData, sqliteData, phase1); + stubFetch(sb, { ...basePerf, goRuntime }, goHealth, ioData, postgresData, phase1); await sb.pages.perf.init({ set innerHTML(v) {} }); await new Promise(r => setTimeout(r, 50)); - stubFetch(sb, { ...basePerf, goRuntime }, goHealth, ioData, sqliteData, phase2); + stubFetch(sb, { ...basePerf, goRuntime }, goHealth, ioData, postgresData, phase2); await sb.pages.perf.init({ set innerHTML(v) {} }); await new Promise(r => setTimeout(r, 50)); const html = sb.getHtml(); diff --git a/tests/unit/test-postgres-ops.js b/tests/unit/test-postgres-ops.js new file mode 100644 index 000000000..e60ba5dd2 --- /dev/null +++ b/tests/unit/test-postgres-ops.js @@ -0,0 +1,257 @@ +// Exercise the actual management functions with a fake container boundary. +// No running Docker service or real database is touched. +const assert = require('assert'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const { spawnSync } = require('child_process'); + +const root = path.resolve(__dirname, '../..'); +const workflow=fs.readFileSync(path.join(root,'.github/workflows/deploy.yml'),'utf8').replace(/\r/g,''); +const e2eJob=workflow.split('\n e2e-shard:\n')[1]?.split(/^ [a-z][a-z0-9-]*:\s*$/m)[0]; +const userStep=workflow.split('- name: Start user-management E2E server (fake mailer)')[1]?.split('\n - name:')[0]; +const sqliteSetup=userStep?.split('if [ "$CORESCOPE_TEST_BACKEND" = sqlite ]; then')[1]?.split(/^\s*else\s*$/m)[0]; +const fixtureCommands=sqliteSetup?.replace(/\\\n\s*/g,' ').match(/\.\/corescope-migrate[^\n]+/g)||[]; +assert.deepStrictEqual({actions:fixtureCommands.map(command=>command.match(/-storage-action=(\w+)/)?.[1]),failFastDisabled:/strategy:\n\s+fail-fast: false\n/.test(e2eJob||'')}, + {actions:['adopt','setup'],failFastDisabled:true},'UM fixture must adopt legacy telemetry then initialize accounts; all independent E2E shards must run'); +const fixtureArgs='-backend=sqlite -offline -config-dir /tmp/cs-um -selection-file /tmp/cs-um/storage-selection.json -sqlite-path "$E2E_SQLITE" -users-sqlite-path /tmp/cs-um/users.db'; +for(const command of fixtureCommands){ + assert.strictEqual(command.replace(/^\.\/corescope-migrate -storage-action=\w+\s+/,'').replace(/\s+/g,' '),fixtureArgs,'account setup changed the adopted selection, SQLite paths or config directory'); +} +const source = fs.readFileSync(path.join(root, 'manage.sh'), 'utf8').replace(/\r/g, ''); +const bash = require('../../scripts/bash-path')(); +const functions = ['pg_container_exec', 'pg_exec', 'pg_empty', 'pg_dump_file', 'pg_restore_file', 'sqlite_source_exists', 'sqlite_dump_file', 'backup_state', 'stage_backup_state', 'restore_sqlite_bundle', 'cmd_backup', 'cmd_restore', 'prepare_staging_db', 'prepare_staging_config', 'write_env_managed_values', 'is_true', 'cmd_start'] + .map(name => source.match(new RegExp(`^${name}\\(\\)\\s*\\{[\\s\\S]*?^}`, 'm'))?.[0] || '').join('\n'); +const quote = value => "'" + value.replace(/'/g, "'\\''") + "'"; +const posix = value => value.replace(/\\/g, '/'); +// The example's published HTTP port must reach a listener in the default +// supervisor selection. A healthy private port 3000 alone does not prove this. +const example = fs.readFileSync(path.join(root, 'docker-compose.example.yml'), 'utf8'); +const disableCaddy = example.match(/DISABLE_CADDY=\$\{DISABLE_CADDY:-(true|false)\}/)?.[1]; +assert(disableCaddy, 'example must declare its Caddy default'); +const supervisor = fs.readFileSync(path.join(root, 'docker', disableCaddy === 'true' ? 'supervisord-go-no-caddy.conf' : 'supervisord-go.conf'), 'utf8'); +const listeners = new Set([Number(supervisor.match(/corescope-server[^\n]* -port (\d+)/)?.[1])]); +if (supervisor.includes('[program:caddy]')) { + const caddy = fs.readFileSync(path.join(root, 'docker/Caddyfile'), 'utf8'); + listeners.add(Number(caddy.match(/^:(\d+)\s*\{/m)?.[1])); + assert(listeners.has(Number(caddy.match(/^\s*reverse_proxy localhost:(\d+)/m)?.[1])), 'Caddy upstream does not reach the application'); +} +const publishedPort = Number(example.match(/HTTP_PORT:-\d+\}:(\d+)/)?.[1]); +assert(listeners.has(publishedPort), 'default example publishes HTTP to a port with no running listener'); +const smoke = fs.readFileSync(path.join(root, 'scripts/test-postgres-compose.sh'), 'utf8'); +assert(!/\bDISABLE_CADDY=true\b/.test(smoke), 'packaged smoke bypasses the example default HTTP listener'); +for (const endpoint of ['http://127.0.0.1:3000/api/healthz', 'http://127.0.0.1:80/api/healthz', 'http://127.0.0.1:80/api/stats']) { + assert(smoke.includes(endpoint), `packaged smoke does not exercise ${endpoint}`); +} +assert(smoke.includes('proxy_health.get("ready") is True'), 'packaged smoke does not validate readiness through Caddy'); +const nilPreparation=smoke.slice(smoke.indexOf('phase=nil-account-adoption'),smoke.indexOf('phase=startup')); +assert(nilPreparation.includes('-storage-action=adopt')&&nilPreparation.includes('-users-sqlite-path /app/data/uninitialized-users.db'), 'packaged smoke lacks native adoption of telemetry without accounts'); +assert(nilPreparation.includes('bootstrap switch -backend=postgres'),'packaged nil-account fixture bypasses the verified switch'); +const firstAccount=smoke.slice(smoke.indexOf('phase=first-account-setup'),smoke.indexOf('phase=recreation')); +assert(firstAccount.indexOf('stop corescope')>=0&&firstAccount.indexOf('stop corescope')&2; } +confirm(){ return 0; }; container_running(){ return 1; } +docker(){ return 0; }; require_database_credentials(){ :; } +selected_backend(){ printf '%s\\n' "$BACKEND"; } +require_managed_postgres(){ [ "$BACKEND" = postgres ]; } +storage_field(){ case "$2" in has_accounts) echo "$HAS_ACCOUNTS";;state) if [ "$1" = staging ]; then echo unrecorded; else echo ready; fi;;backend) echo "$BACKEND";;telemetry.sqlite_path) echo '/app/data/custom telemetry.db';;accounts.sqlite_path) echo '/app/data/users.db';;state_dir) echo '/app/data';;esac; } +storage_action(){ :; } +storage_at(){ shift 2; if [ "$1" = field ]; then if [ "$2" = has_accounts ]; then echo "$BACKUP_HAS_ACCOUNTS"; else storage_field prod "$2"; fi; elif [ "$1" = managed-postgres ]; then [ "$BACKEND" = postgres ]; fi; } +dc_staging_base(){ dc_prod_base "$@"; } +dc_prod_base(){ + printf 'base %s\\n' "$*" >> "$CALLS" + case "$*" in + *'--entrypoint sqlite3'*) + case "$*" in *'PRAGMA quick_check'*) printf 'ok\\n'; return;;esac + local destination='' previous='' part + for part in "$@"; do [ "$previous" != -v ] || destination="\${part%:/backup}"; previous="$part"; done + case "$*" in *telemetry.db.partial*) printf 'SQLite format 3\\0native WAL snapshot' >| "$destination/telemetry.db.partial";;*accounts.db.partial*) printf 'SQLite format 3\\0native accounts snapshot' >| "$destination/accounts.db.partial";;esac + ;; + *) :;; + esac +} +preflight_validate_prod_ports(){ return 0; }; migrate_config(){ :; }; ensure_config(){ :; } +database_reply(){ + case "$*" in + *pg_dump*) [ "$PROD_READY" = 1 ] || { err 'Production PostgreSQL is not running'; return 1; }; printf 'PGDMP-native-test-archive';; + *pg_restore*) cat >/dev/null; [ "$FAIL_RESTORE" != 1 ];; + *SELECT*COUNT*) printf '%s\\n' "$OCCUPIED";; + *) :;; + esac +} +dc_prod(){ + printf 'prod %s\\n' "$*" >> "$CALLS" + if [ "$*" = 'up -d prod' ]; then + [ "$PROD_READY" != fail ] || return 1 + PROD_READY=1 + fi + database_reply "$@" +} +dc_postgres(){ local environment="$1"; shift; "dc_$environment" "$@"; } +dc_staging(){ printf 'staging %s\\n' "$*" >> "$CALLS"; database_reply "$@"; } +${functions} +${command} +`; + // Keep the extracted shell source off Windows' bounded command line. + return spawnSync(bash, ['--noprofile', '--norc'], { input: setup, cwd: temp, encoding: 'utf8', timeout: 20000 }); +} +try { + fs.mkdirSync(path.join(temp, 'state')); + fs.mkdirSync(path.join(temp, 'docker')); + fs.writeFileSync(path.join(temp, 'docker/postgres-grants.sql'), 'SELECT 1;'); + fs.writeFileSync(path.join(temp, 'state/meshcore.db'), 'SQLite format 3\0legacy recovery'); + fs.writeFileSync(path.join(temp, 'state/config.json'), '{"siteName":"test"}'); + const sqliteBackup = path.join(temp, 'sqlite-backup'); + const sqlite = run(`cmd_backup ${quote(posix(sqliteBackup))}`, '0', '0', '1', 'sqlite'); + assert.strictEqual(sqlite.status, 0, sqlite.stderr || sqlite.error?.message); + for (const name of ['telemetry.db','accounts.db']) assert(fs.existsSync(path.join(sqliteBackup,name)), 'SQLite native snapshot missing: '+name); + assert(fs.readFileSync(path.join(sqliteBackup,'telemetry.db')).includes(Buffer.from('native WAL snapshot')), 'backup copied the live main file instead of taking a native snapshot'); + const sqliteCalls = fs.readFileSync(path.join(temp,'calls'),'utf8'); + assert(sqliteCalls.includes('--entrypoint sqlite3') && sqliteCalls.includes('-readonly') && sqliteCalls.includes('/app/data/custom telemetry.db'), 'backup ignored the selected path or read-only boundary'); + assert(!sqliteCalls.includes('pg_dump'), 'SQLite backup contacted PostgreSQL'); + assert(fs.existsSync(path.join(sqliteBackup,'config.json')), 'SQLite backup omitted configuration'); + assert(fs.existsSync(path.join(sqliteBackup,'state.tar')), 'SQLite backup omitted persistent selection/queue/state recovery archive'); + const missingRecorded = run(` + storage_field(){ case "$2" in has_accounts) echo true;;state) echo ready;;backend) echo sqlite;;*) printf '%s/missing-recorded.db\\n' "$PROD_DATA";;esac; } + sqlite_dump_file(){ :; } + dc_prod_base(){ while [ "$1" != -c ]; do shift; done; shift; bash -eu -c "$@"; } + cmd_backup ${quote(posix(path.join(temp,'missing-recorded-backup')))} + `, '0','0','1','sqlite'); + assert.notStrictEqual(missingRecorded.status,0,'backup labelled a missing recorded account database as intentionally absent'); + const badAccounts = path.join(temp,'bad-account-backup'); + fs.cpSync(sqliteBackup,badAccounts,{recursive:true}); + fs.unlinkSync(path.join(badAccounts,'accounts.db')); + fs.writeFileSync(path.join(badAccounts,'accounts.absent'),''); + const badRestore = run(`PROD_DATA=${quote(posix(path.join(temp,'bad-restored-state')))}; cmd_restore ${quote(posix(badAccounts))}`,'0','0','1','sqlite'); + assert.notStrictEqual(badRestore.status,0,'restore skipped initialized accounts based on an absence marker'); + const restoredSQLite = path.join(temp,'restored-sqlite'); + const sqliteRestore = run(`PROD_DATA=${quote(posix(restoredSQLite))}; cmd_restore ${quote(posix(sqliteBackup))}`, '0', '0', '1', 'sqlite'); + assert.strictEqual(sqliteRestore.status,0,sqliteRestore.stderr || sqliteRestore.error?.message); + assert(fs.readFileSync(path.join(restoredSQLite,'custom telemetry.db')).includes(Buffer.from('native WAL snapshot')), 'SQLite restore ignored the recorded target'); + assert(fs.existsSync(path.join(restoredSQLite,'users.db')), 'SQLite restore lost accounts'); + assert(fs.existsSync(path.join(restoredSQLite,'config.json')), 'SQLite restore lost persistent state'); + const repeatedSQLite = run(`PROD_DATA=${quote(posix(restoredSQLite))}; cmd_restore ${quote(posix(sqliteBackup))}`, '0', '0', '1', 'sqlite'); + assert.notStrictEqual(repeatedSQLite.status,0,'SQLite restore overwrote existing state'); + fs.writeFileSync(path.join(temp,'calls'),''); + const backup = path.join(temp, 'backup'); + const made = run(`cmd_backup ${quote(posix(backup))}`); + assert.strictEqual(made.status, 0, made.stderr || made.error?.message); + for (const name of ['telemetry.dump', 'accounts.dump']) { + assert(fs.existsSync(path.join(backup, name)), `native archive missing: ${name}`); + assert(fs.readFileSync(path.join(backup, name), 'utf8').startsWith('PGDMP')); + } + assert(fs.existsSync(path.join(backup, 'config.json')), 'configuration omitted'); + assert(!fs.existsSync(path.join(backup, 'meshcore.db')), 'backup copied a legacy live SQLite file'); + const blocked = run(`cmd_restore ${quote(posix(backup))}`, '1'); + assert.notStrictEqual(blocked.status, 0, 'restore overwrote a nonempty destination'); + assert(!fs.readFileSync(path.join(temp, 'calls'), 'utf8').includes('pg_restore'), 'restore ran before emptiness check'); + const failedRestore = run(`cmd_restore ${quote(posix(backup))}`, '0', '1'); + assert.notStrictEqual(failedRestore.status, 0, 'failed native restore was reported as success'); + const restored = run(`cmd_restore ${quote(posix(backup))}`); + assert.strictEqual(restored.status, 0, restored.stderr || restored.error?.message); + const calls = fs.readFileSync(path.join(temp, 'calls'), 'utf8'); + assert(calls.includes('pg_restore'), 'native restore was not invoked'); + assert(!calls.includes('must-not-appear-in-argv'), 'database password appeared in command arguments'); + const nilBackup=path.join(temp,'nil-account-backup'); + fs.writeFileSync(path.join(temp,'calls'),''); + const nilMade=run(`cmd_backup ${quote(posix(nilBackup))}`,'0','0','1','postgres','false'); + assert.strictEqual(nilMade.status,0,nilMade.stderr||nilMade.error?.message); + assert(fs.existsSync(path.join(nilBackup,'accounts.absent')),'nullable account target was not recorded as absent'); + assert(!fs.existsSync(path.join(nilBackup,'accounts.dump')),'backup dumped an unselected account database'); + assert(!fs.readFileSync(path.join(temp,'calls'),'utf8').includes('corescope_accounts'),'nil-account backup contacted the account database'); + fs.writeFileSync(path.join(temp,'calls'),''); + const nilRestore=run(`cmd_restore ${quote(posix(nilBackup))}`,'0','0','1','postgres','false'); + assert.strictEqual(nilRestore.status,0,nilRestore.stderr||nilRestore.error?.message); + const nilCalls=fs.readFileSync(path.join(temp,'calls'),'utf8'); + assert(nilCalls.includes('pg_restore'),'telemetry-only restore did not restore telemetry'); + assert(!nilCalls.includes('corescope_accounts'),'telemetry-only restore read or wrote the unselected account database'); + for(const [bundle,selected,recorded] of [[nilBackup,'true','false'],[nilBackup,'false','true'],[backup,'false','true']]){ + fs.writeFileSync(path.join(temp,'calls'),''); + const mismatch=run(`cmd_restore ${quote(posix(bundle))}`,'0','0','1','postgres',selected,recorded); + assert.notStrictEqual(mismatch.status,0,'restore accepted account snapshot/selection disagreement'); + assert(!fs.readFileSync(path.join(temp,'calls'),'utf8').includes('pg_restore'),'mismatched selection reached restore writes'); + } + const legacyBundle=path.join(temp,'legacy-two-dump-backup'); + fs.cpSync(backup,legacyBundle,{recursive:true});fs.unlinkSync(path.join(legacyBundle,'state.tar')); + const legacyBundleRestore=run(`cmd_restore ${quote(posix(legacyBundle))}`); + assert.strictEqual(legacyBundleRestore.status,0,'existing two-archive PG backup support regressed: '+legacyBundleRestore.stderr); + const absentWithoutSelection=path.join(temp,'absence-without-selection'); + fs.cpSync(nilBackup,absentWithoutSelection,{recursive:true});fs.unlinkSync(path.join(absentWithoutSelection,'state.tar')); + fs.writeFileSync(path.join(temp,'calls'),''); + const unprovedAbsence=run(`cmd_restore ${quote(posix(absentWithoutSelection))}`,'0','0','1','postgres','false'); + assert.notStrictEqual(unprovedAbsence.status,0,'restore trusted a bare absence marker without its retained selection'); + assert(!fs.readFileSync(path.join(temp,'calls'),'utf8').includes('pg_restore'),'unproved account absence reached restore writes'); + fs.writeFileSync(path.join(temp,'calls'),''); + const unselectedSQL=run('pg_exec prod corescope_accounts psql -c SELECT','0','0','1','postgres','false'); + assert.notStrictEqual(unselectedSQL.status,0,'generic managed executor contacted an unselected account target'); + assert(!fs.readFileSync(path.join(temp,'calls'),'utf8').includes('corescope_accounts'),'unselected database reached the container boundary'); + const legacy = run(`cmd_restore ${quote(posix(path.join(temp, 'state/meshcore.db')))}`); + assert.notStrictEqual(legacy.status, 0, 'SQLite restore accepted without offline import'); + fs.writeFileSync(path.join(temp, 'calls'), ''); + const sqliteStage = run('require_managed_postgres(){ :; }; prepare_staging_db', '0', '0', '1', 'sqlite'); + assert.strictEqual(sqliteStage.status,0,sqliteStage.stderr || sqliteStage.error?.message); + assert(fs.existsSync(path.join(temp,'staging/custom telemetry.db')), 'SQLite staging did not clone native telemetry'); + assert(!fs.existsSync(path.join(temp,'staging/users.db')), 'staging copied production accounts'); + assert(!fs.readFileSync(path.join(temp,'calls'),'utf8').includes('pg_dump'), 'SQLite staging required PostgreSQL'); + // The independent PG cases below use another fresh staging directory. + fs.renameSync(path.join(temp,'staging'),path.join(temp,'staging-sqlite-retained')); + fs.writeFileSync(path.join(temp, 'calls'), ''); + const staged = run('prepare_staging_db'); + assert.strictEqual(staged.status, 0, staged.stderr || staged.error?.message); + assert(!fs.readFileSync(path.join(temp, 'calls'), 'utf8').includes('corescope_accounts'), 'staging copied production accounts'); + fs.writeFileSync(path.join(temp, 'calls'), ''); + const productionOnly = run("cmd_start ''", '0', '0', '0'); + assert.strictEqual(productionOnly.status, 0, productionOnly.stderr || productionOnly.error?.message); + assert.deepStrictEqual(fs.readFileSync(path.join(temp, 'calls'), 'utf8').trim().split('\n'), ['prod up -d prod'], 'ordinary cold start changed'); + fs.writeFileSync(path.join(temp, 'calls'), ''); + const coldStaging = run('cmd_start --with-staging', '0', '0', '0'); + assert.strictEqual(coldStaging.status, 0, coldStaging.stderr || coldStaging.error?.message); + const startCalls = fs.readFileSync(path.join(temp, 'calls'), 'utf8').trim().split('\n'); + assert.strictEqual(startCalls.filter(call => call === 'prod up -d prod').length, 1, 'production must start once through normal Compose dependencies'); + assert(startCalls.indexOf('prod up -d prod') < startCalls.findIndex(call => call.includes('pg_dump')), 'staging dump preceded production bootstrap'); + const restoreIndex = startCalls.findIndex(call => call.includes('pg_restore')); + assert(restoreIndex >= 0 && restoreIndex < startCalls.indexOf('staging up -d staging-go'), 'staging application started before restore'); + assert(!startCalls.some(call => call.includes('corescope_accounts')), 'cold staging start copied production accounts'); + fs.writeFileSync(path.join(temp, 'calls'), ''); + const retainedStaging = run('cmd_start --with-staging', '1', '0', '0'); + assert.strictEqual(retainedStaging.status, 0, retainedStaging.stderr || retainedStaging.error?.message); + assert(!/pg_dump|pg_restore/.test(fs.readFileSync(path.join(temp, 'calls'), 'utf8')), 'cold start replaced existing staging data'); + fs.writeFileSync(path.join(temp, 'calls'), ''); + const failedStart = run('cmd_start --with-staging', '0', '0', 'fail'); + assert.notStrictEqual(failedStart.status, 0, 'failed production startup reported success'); + assert.deepStrictEqual(fs.readFileSync(path.join(temp, 'calls'), 'utf8').trim().split('\n'), ['prod up -d prod'], 'staging started after production bootstrap failed'); + for (const [executable, script] of [ + [process.execPath, 'scripts/migrate-dedup.js'], + [process.platform === 'win32' ? 'python' : 'python3', 'scripts/prune-nodes-outside-geo-filter.py'] + ]) { + const retired = spawnSync(executable, [path.join(root, script)], { cwd: temp, encoding: 'utf8', timeout: 10000 }); + assert.notStrictEqual(retired.status, 0, 'legacy tool executed without explicit offline opt-in'); + assert(/offline legacy SQLite/i.test(retired.stderr), retired.stderr); + } + fs.writeFileSync(path.join(temp, '.env'), 'CORESCOPE_OWNER_PASSWORD=private-test-value\nPROD_HTTP_PORT=80\n', { mode: 0o600 }); + const configured = run("write_env_managed_values 8080 8443 1883 ./state false"); + assert.strictEqual(configured.status, 0, configured.stderr); + const config = fs.readFileSync(path.join(temp, '.env'), 'utf8'); + assert(config.includes('CORESCOPE_OWNER_PASSWORD=private-test-value'), 'setup lost database credentials'); + assert(config.includes('PROD_HTTP_PORT=8080'), 'setup did not update the requested port'); + if (process.platform !== 'win32') assert.strictEqual(fs.statSync(path.join(temp, '.env')).mode & 0o777, 0o600, 'setup exposed private credentials'); + console.log('PostgreSQL management backup/restore safeguards passed'); +} finally { + const base = path.resolve(os.tmpdir()) + path.sep; + assert(path.resolve(temp).startsWith(base) && path.basename(temp).startsWith('corescope-pg-ops-')); + fs.rmSync(temp, { recursive: true }); +} diff --git a/tests/unit/test-storage-entrypoint.js b/tests/unit/test-storage-entrypoint.js new file mode 100644 index 000000000..2b59610be --- /dev/null +++ b/tests/unit/test-storage-entrypoint.js @@ -0,0 +1,214 @@ +// Run the packaged startup guard against a fake migration/process boundary. +// No Docker daemon, host state, database, or credentials are used. +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); +const root = path.resolve(__dirname, '../..'); +const bash = require('../../scripts/bash-path')(); +const temp = fs.mkdtempSync(path.join(os.tmpdir(), 'corescope-storage-entry-')); +const app = path.join(temp, 'app'); +const posix = value => value.replace(/\\/g, '/').replace(/^([A-Za-z]):/, (_, drive) => '/' + drive.toLowerCase()); +const shellQuote = value => "'" + value.replace(/'/g, "'\\''") + "'"; +const calls = path.join(temp, 'calls'); +fs.mkdirSync(path.join(app, 'data'), { recursive: true }); +fs.mkdirSync(path.join(temp, 'bin')); +fs.writeFileSync(path.join(app, 'data/config.json'), '{}'); +fs.writeFileSync(path.join(temp, 'mountinfo'), `1 0 8:1 / ${posix(app)}/data rw - ext4 /dev/owned rw\n`); +const tool = path.join(temp, 'tools.js'); +fs.writeFileSync(tool, ` +const fs=require('fs'),path=require('path'); +const [name,...args]=process.argv.slice(2); +const calls=process.env.CALLS; +if(name!=='jq')fs.appendFileSync(calls,JSON.stringify([name,...args])+'\\n'); +const field=flag=>args.find(v=>v.startsWith(flag+'='))?.slice(flag.length+1)||args[args.indexOf(flag)+1]; +if(name==='corescope-migrate'){ + if(args.includes('-check-ready')){ + fs.appendFileSync(calls,JSON.stringify(['readiness-accounts',Boolean(process.env.CORESCOPE_USERS_DATABASE_URL)])+'\\n'); + const scenario=process.env.SCENARIO; + const failure={ + 'unsupported-schema':'unsupported telemetry schema version 99 (binary supports 2)', + 'missing-schema':'telemetry schema is not initialized', + 'corrupt-schema':'invalid telemetry schema layout' + }[scenario]; + if(failure){process.stderr.write(failure+'\\n');process.exit(1)} + if(['old-schema','failed-upgrade','failed-recheck'].includes(scenario)&&!fs.existsSync(calls+'.upgraded')){ + process.stderr.write('unsupported telemetry schema version 1 (binary supports 2)\\n');process.exit(1); + } + if(scenario==='failed-recheck'){process.stderr.write('telemetry schema verification failed after upgrade\\n');process.exit(1)} + console.log('telemetry PostgreSQL schema is ready'); + process.exit(0); + } + const selection=field('-selection-file');const action=field('-storage-action'); + if(action==='status'){ + let report={version:1,state:'unrecorded',backend:'',generation:'',job_id:'',source_backend:'',target_backend:''}; + if(fs.existsSync(selection))report=JSON.parse(fs.readFileSync(selection,'utf8')); + if(process.env.SCENARIO==='pending')report={...report,state:'pending',job_id:'0123456789abcdef0123456789abcdef'}; + if(process.env.SCENARIO==='corrupt'){console.log(JSON.stringify({...report,state:'corrupt'}));process.exit(1)}; + console.log(JSON.stringify(report)); + }else if(action==='setup'){ + fs.appendFileSync(calls,JSON.stringify(['setup-account',process.env.FIXTURE_RUN_AS||'root'])+'\\n'); + if(process.env.SCENARIO==='fail-setup')process.exit(1); + const backend=field('-backend')||'sqlite';fs.mkdirSync(path.dirname(selection),{recursive:true}); + fs.writeFileSync(selection,JSON.stringify({version:1,state:'ready',backend,generation:'one',job_id:''}));console.log(JSON.stringify({verified:true})); + }else if(action==='upgrade'){ + const report=JSON.parse(fs.readFileSync(selection,'utf8')); + if(report.state!=='ready'||report.backend!=='postgres'||['unsupported-schema','missing-schema','corrupt-schema'].includes(process.env.SCENARIO)){ + process.stderr.write('recorded target cannot be upgraded\\n');process.exit(1); + } + process.stderr.write('[storage] PostgreSQL telemetry schema upgrade required: 1 -> 2\\n'); + if(process.env.SCENARIO==='failed-upgrade'){process.stderr.write('owner upgrade failed\\n');process.exit(1)} + fs.appendFileSync(calls,JSON.stringify(['upgraded-stores',report.accounts?['telemetry','accounts']:['telemetry']])+'\\n'); + fs.writeFileSync(calls+'.upgraded','verified'); + process.stderr.write('[storage] upgraded recorded PostgreSQL telemetry schema 1 -> 2\\n'); + }else{process.stderr.write('unexpected migration action');process.exit(2)} +}else if(name==='su'){ + if(process.env.SCENARIO==='fail-user-switch')process.exit(1); + const env={...process.env,FIXTURE_RUN_AS:args.at(-1)}; + // MSYS converts path-shaped environment values for the native Node stub. + env.CORESCOPE_STATE_DIR=env.CORESCOPE_STATE_DIR.replace(/\\\\/g,'/').replace(/^([A-Za-z]):/,(_,drive)=>'/'+drive.toLowerCase()); + const result=require('child_process').spawnSync(process.env.FIXTURE_BASH,['--noprofile','--norc','-c',args[args.indexOf('-c')+1]], + {env,stdio:'inherit'}); + process.exit(result.status??1); +}else if(name==='jq'){ + const value=JSON.parse(fs.readFileSync(0,'utf8'));const query=args.at(-1); + if(query==='.accounts != null'){console.log(value.accounts!=null);process.exit(0)} + if(query.includes('.telemetry.postgres ==')){ + const canonical=(target,database)=>target&&Object.keys(target).length===4&&target.host==='postgres'&&target.port===5432&&target.database===database&&target.schema==='public'; + const valid=value.state==='ready'&&value.backend==='postgres'&&canonical(value.telemetry?.postgres,'corescope_telemetry')&& + ((query.includes('.accounts == null or')&&value.accounts===null)||canonical(value.accounts?.postgres,'corescope_accounts')); + if(!valid)process.exit(1);console.log(true);process.exit(0); + } + const key=query.replace(/^\\./,'').replace(/ \\/\\/.*$/,''); + if(!/^[a-z_.]+$/.test(key))throw Error('unhandled jq boundary query: '+query); + let selected=key.split('.').reduce((obj,key)=>obj?.[key],value); + if(selected==null||selected===''){process.exit(1)}else{console.log(selected)} +}else if(name==='psql'){ + fs.appendFileSync(calls,JSON.stringify(['grant-database',process.env.PGDATABASE])+'\\n'); +} +`); +for (const name of ['corescope-migrate','jq','supervisord','psql','su']) { + const file = path.join(name === 'corescope-migrate' ? app : path.join(temp, 'bin'), name); + fs.writeFileSync(file, `#!/bin/sh\nexec node ${shellQuote(posix(tool))} ${name} "$@"\n`, { mode: 0o755 }); +} +// Account resolution/.env trust has its own shell suite. Here the process +// boundary supplies the resolved identity and records who runs setup. +fs.writeFileSync(path.join(temp, 'select-service-user.sh'), 'prepare_service_user(){ CORESCOPE_SERVICE_USER=${TEST_SERVICE_USER:-root}; export CORESCOPE_SERVICE_USER; }\n'); +fs.writeFileSync(path.join(temp, 'bin/id'), '#!/bin/sh\nprintf "1000\\n"\n', { mode: 0o755 }); +function materialize() { + let source = fs.readFileSync(path.join(root, 'docker/entrypoint-go.sh'), 'utf8'); + source = source.replaceAll('/app', posix(app)).replace('/usr/bin/supervisord', posix(path.join(temp, 'bin/supervisord'))) + .replaceAll('/select-service-user.sh', posix(path.join(temp, 'select-service-user.sh'))); + fs.writeFileSync(path.join(temp, 'entrypoint.sh'), source, { mode: 0o755 }); + const helper = path.join(root, 'docker/storage.sh'); + if (fs.existsSync(helper)) fs.writeFileSync(path.join(app, 'storage.sh'), fs.readFileSync(helper, 'utf8').replaceAll('/app', posix(app)).replace('/proc/self/mountinfo', posix(path.join(temp, 'mountinfo'))), { mode: 0o755 }); + fs.writeFileSync(path.join(app,'postgres-bootstrap.sh'),fs.readFileSync(path.join(root,'docker/postgres-bootstrap.sh'),'utf8').replaceAll('/app',posix(app)),{mode:0o755}); +} +function run(extra = {}, action = [], bootstrap = false) { + fs.writeFileSync(calls, ''); + fs.rmSync(calls + '.upgraded', { force: true }); + const env = { ...process.env, CALLS: calls, FIXTURE_BASH: bash, TEST_SERVICE_USER: 'root', CORESCOPE_STATE_DIR: posix(path.join(app, 'data/custom')), CORESCOPE_DB_BACKEND: 'sqlite', SCENARIO: '', ...extra }; + const inherited = env.PATH || env.Path || ''; + delete env.Path; + env.PATH = path.join(temp, 'bin') + path.delimiter + inherited; + // A run launches several native stub processes through the shell. Allow + // Windows process-start overhead without changing any product deadline. + const result = spawnSync(bash, ['--noprofile','--norc',posix(bootstrap ? path.join(app,'postgres-bootstrap.sh') : action.length ? path.join(app,'storage.sh') : path.join(temp,'entrypoint.sh')), ...action], { cwd: app, env, encoding: 'utf8', timeout: 30000 }); + result.calls = fs.readFileSync(calls,'utf8').trim().split('\n').filter(Boolean).map(JSON.parse); + return result; +} +try { + materialize(); + const fresh = run(); + assert.equal(fresh.status,0,fresh.stderr || fresh.error?.message); + assert(fresh.calls.some(call => call.includes('setup') || call.includes('-storage-action=setup')), 'default Docker startup did not invoke guarded fresh/legacy setup'); + const last = fresh.calls.at(-1); assert.equal(last?.[0], 'supervisord', 'supervisor started before setup completed'); + const setup = fresh.calls.find(call => call.includes('setup') || call.includes('-storage-action=setup')); + assert(setup.includes('-offline'), 'guarded setup must be offline, before child processes'); + assert(setup.some(arg => arg.includes('/data/custom/storage-selection.json')), 'custom persistent selection path was ignored'); + assert.deepEqual(fresh.calls.find(call => call[0] === 'setup-account'), ['setup-account', 'root']); + assert(!fresh.calls.some(call => call[0] === 'su'), 'default root startup switched users'); + const unprivileged = run({ TEST_SERVICE_USER: 'corescope', CORESCOPE_STATE_DIR: posix(path.join(app, 'data/unprivileged')) }); + assert.equal(unprivileged.status, 0, unprivileged.stderr || unprivileged.error?.message); + assert.deepEqual(unprivileged.calls.find(call => call[0] === 'setup-account'), ['setup-account', 'corescope'], + 'fresh SQLite files must be created by the service account after ownership hand-over'); + assert.equal(unprivileged.calls.at(-1)?.[0], 'supervisord'); + const switchFailure = run({ TEST_SERVICE_USER: 'corescope', SCENARIO: 'fail-user-switch' }); + assert.notEqual(switchFailure.status, 0, 'failed service-account switch started the server'); + assert(!switchFailure.calls.some(call => call[0] === 'supervisord' || call[0] === 'corescope-migrate')); + const recreated = run({ CORESCOPE_DB_BACKEND: 'postgres' }); + assert.equal(recreated.status,0,recreated.stderr); + assert(!recreated.calls.some(call => call.includes('setup') || call.includes('-storage-action=setup')), 'recreate interpreted a stale bootstrap backend as new setup'); + assert.equal(recreated.calls.at(-1)?.[0],'supervisord'); + for (const scenario of ['pending','corrupt']) { + const blocked = run({ SCENARIO: scenario }); + assert.notEqual(blocked.status,0,scenario+' selection started'); + assert(!blocked.calls.some(call => call[0]==='supervisord'),scenario+' selection launched application'); + } + const corruptField = run({ SCENARIO: 'corrupt' }, ['field','backend']); + assert.notEqual(corruptField.status,0,'field reader hid the CLI corrupt-status failure'); + const failed = run({ SCENARIO: 'fail-setup', CORESCOPE_STATE_DIR: posix(path.join(app,'data/new')) }); + assert.notEqual(failed.status,0,'failed setup started the server'); + assert(!failed.calls.some(call=>call[0]==='supervisord')); + const ephemeral = run({ CORESCOPE_STATE_DIR: posix(path.join(temp,'ephemeral')) }); + assert.notEqual(ephemeral.status,0,'ephemeral selection directory accepted'); + assert.equal(ephemeral.calls.length,0,'unsupported path reached migration/runtime before rejection'); + for (const file of fs.readdirSync(path.join(root,'docker')).filter(name => /^supervisord-go.*[.]conf$/.test(name))) { + const config = fs.readFileSync(path.join(root,'docker',file),'utf8'); + assert(!/corescope-server[^\n]*-state-dir/.test(config), file+' overrides shared selection directory'); + } + const selectionFile=path.join(app,'data/custom/storage-selection.json'); + const target=database=>({postgres:{host:'postgres',port:5432,database,schema:'public'}}); + const pgReport={version:1,state:'ready',backend:'postgres',generation:'one',telemetry:target('corescope_telemetry'),accounts:null}; + for(const accounts of [null,target('corescope_accounts')]){ + fs.writeFileSync(selectionFile,JSON.stringify({...pgReport,accounts})); + const guard=run({},['managed-postgres']); + assert.equal(guard.status,0,'canonical PostgreSQL with optional accounts was rejected: '+guard.stderr); + const started=run({CORESCOPE_OWNER_PASSWORD:'synthetic-owner',CORESCOPE_USERS_OWNER_DATABASE_URL:accounts?'synthetic-owner-url':'',CORESCOPE_USERS_DATABASE_URL:'runtime-account-url'},[],true); + assert.equal(started.status,0,started.stderr||started.error?.message); + assert.deepEqual(started.calls.filter(call=>call[0]==='grant-database').map(call=>call[1]),accounts?['corescope_telemetry','corescope_accounts']:['corescope_telemetry']); + assert.deepEqual(started.calls.find(call=>call[0]==='readiness-accounts'),['readiness-accounts',accounts!==null]); + assert(!started.calls.some(call=>call.includes('-storage-action=setup')),'ready PostgreSQL was implicitly initialized'); + assert(!started.calls.some(call=>call.includes('-storage-action=upgrade')),'current schema was unnecessarily upgraded'); + assert.match(started.stdout,/telemetry PostgreSQL schema is ready/,'current readiness output was hidden'); + const originalSelection=fs.readFileSync(selectionFile,'utf8'); + const upgraded=run({SCENARIO:'old-schema',CORESCOPE_OWNER_PASSWORD:'synthetic-owner',CORESCOPE_USERS_OWNER_DATABASE_URL:'synthetic-owner-url',CORESCOPE_USERS_DATABASE_URL:'runtime-account-url'},[],true); + assert.equal(upgraded.status,0,upgraded.stderr||upgraded.error?.message); + assert.match(upgraded.stderr,/upgrade required: 1 -> 2/,'supported upgrade did not report stored and current versions'); + assert(!upgraded.stderr.includes('unsupported telemetry'),'successful supported upgrade emitted a provisional failure'); + const upgrade=upgraded.calls.filter(call=>call.includes('-storage-action=upgrade')); + assert.equal(upgrade.length,1,'old recorded schema did not invoke exactly one owner upgrade'); + assert(upgrade[0].includes('-offline'),'schema upgrade was not explicitly offline'); + assert(upgrade[0].includes('-selection-file')&&upgrade[0].some(arg=>arg.endsWith('/data/custom/storage-selection.json')),'upgrade lost its recorded selection'); + assert.deepEqual(upgraded.calls.find(call=>call[0]==='upgraded-stores'),['upgraded-stores',accounts?['telemetry','accounts']:['telemetry']]); + assert.deepEqual(upgraded.calls.filter(call=>call[0]==='readiness-accounts'),[['readiness-accounts',accounts!==null],['readiness-accounts',accounts!==null]],'upgrade must recheck precisely the selected stores'); + const finalReady=upgraded.calls.map(call=>call[0]).lastIndexOf('readiness-accounts'); + assert(upgraded.calls.findIndex(call=>call[0]==='psql')>finalReady,'runtime grants preceded upgrade verification'); + assert(!upgraded.calls.some(call=>call.includes('-storage-action=setup')),'recorded upgrade could initialize a replacement or absent account target'); + assert.equal(fs.readFileSync(selectionFile,'utf8'),originalSelection,'upgrade changed the selection'); + for(const [scenario,diagnostic] of [ + ['unsupported-schema','unsupported telemetry schema version 99'], + ['missing-schema','telemetry schema is not initialized'], + ['corrupt-schema','invalid telemetry schema layout'], + ['failed-upgrade','owner upgrade failed'], + ['failed-recheck','telemetry schema verification failed after upgrade'] + ]){ + const unsupported=run({SCENARIO:scenario,CORESCOPE_OWNER_PASSWORD:'synthetic-owner',CORESCOPE_USERS_OWNER_DATABASE_URL:'synthetic-owner-url'},[],true); + assert.notEqual(unsupported.status,0,scenario+' passed bootstrap'); + assert(!unsupported.calls.some(call=>call[0]==='psql'),scenario+' reached grants'); + assert.match(unsupported.stderr,new RegExp(diagnostic),scenario+' diagnostic was hidden'); + assert.equal(fs.readFileSync(selectionFile,'utf8'),originalSelection,'failed upgrade changed the selection'); + } + } + for(const report of [{...pgReport,telemetry:target('unselected_telemetry')},{...pgReport,accounts:target('unselected_accounts')},{...pgReport,accounts:{sqlite_path:'/app/data/users.db'}}]){ + fs.writeFileSync(selectionFile,JSON.stringify(report)); + const rejected=run({CORESCOPE_OWNER_PASSWORD:'synthetic-owner'},[],true); + assert.notEqual(rejected.status,0,'custom/non-PostgreSQL target passed the managed guard'); + assert(!rejected.calls.some(call=>call[0]==='psql'||call.includes('-check-ready')),'rejected target reached database operations'); + } + console.log('PASS packaged SQLite setup, recorded choice on recreate, blocked journals/errors and persistent custom state'); +} finally { + assert(path.resolve(temp).startsWith(path.resolve(os.tmpdir())+path.sep)&&path.basename(temp).startsWith('corescope-storage-entry-')); + fs.rmSync(temp,{recursive:true}); +} diff --git a/tests/unit/test-storage-manage.js b/tests/unit/test-storage-manage.js new file mode 100644 index 000000000..8396a2f04 --- /dev/null +++ b/tests/unit/test-storage-manage.js @@ -0,0 +1,103 @@ +// Exercise the actual management shell against a fake Compose/process boundary. +const assert=require('node:assert/strict'); +const fs=require('node:fs'),os=require('node:os'),path=require('node:path'); +const {spawnSync}=require('node:child_process'); +const root=path.resolve(__dirname,'../..'); +const source=fs.readFileSync(path.join(root,'manage.sh'),'utf8').replace(/\r/g,''); +const bash=require('../../scripts/bash-path')(); +const temp=fs.mkdtempSync(path.join(os.tmpdir(),'corescope-storage-manage-')); +const quote=value=>"'"+value.replace(/'/g,"'\\''")+"'"; +const posix=value=>value.replace(/\\/g,'/'); +const names=['is_true','dc_prod_base','dc_staging_base','dc_postgres','storage_at','storage_action','storage_field','selected_backend','require_managed_postgres','choose_storage','dc_prod','dc_staging','cmd_start','cmd_update','cmd_stop','cmd_storage']; +const functions=names.map(name=>source.match(new RegExp(`^${name}\\(\\)\\s*\\{[\\s\\S]*?^}`,'m'))?.[0]||'').join('\n'); +function run(command,backend='sqlite',state='ready',input=''){ + const log=path.join(temp,'calls');fs.writeFileSync(log,''); + const script=` +set -e +DC=fake_compose +PROD_DATA=${quote(posix(path.join(temp,'prod')))} +STAGING_DATA=${quote(posix(path.join(temp,'staging')))} +STAGING_COMPOSE_FILE=docker-compose.staging.yml +STAGING_CONTAINER=corescope-staging-go +RECORD_BACKEND=${quote(backend)} +RECORD_STATE=${quote(state)} +CORESCOPE_DB_BACKEND=sqlite +CALLS=${quote(posix(log))} +log(){ :; }; info(){ :; }; warn(){ :; }; err(){ printf '%s\\n' "$*" >&2; } +confirm(){ return 0; } +migrate_config(){ :; }; ensure_config(){ :; }; preflight_validate_prod_ports(){ :; } +write_private_env(){ export "$1=$2"; }; +prepare_database_credentials(){ printf 'credentials prepared\\n' >> "$CALLS"; }; +require_database_credentials(){ [ "$RECORD_BACKEND" = postgres ] || [ "\${STORAGE_OPERATION:-}" = switch ] || { err 'PostgreSQL credentials are not configured'; return 1; }; } +container_running(){ return 1; } +prepare_staging_db(){ :; };prepare_staging_config(){ :; } +docker(){ if [ "$1" = ps ];then return 0;fi;printf 'docker %s\\n' "$*" >> "$CALLS"; } +git(){ case "$1" in describe) echo v1.0.0;;rev-parse) echo fixture;;*) printf 'git %s\\n' "$*" >> "$CALLS";;esac; } +fake_compose(){ + case "$*" in + *'/app/storage.sh prod field state'|*'/app/storage.sh staging-go field state') printf '%s\\n' "$RECORD_STATE";; + *'/app/storage.sh prod field backend'|*'/app/storage.sh staging-go field backend') printf '%s\\n' "$RECORD_BACKEND";; + *'/app/storage.sh prod field has_accounts') printf 'false\\n';; + *'/app/storage.sh prod field state_dir') printf '/app/data\\n';; + *'/app/storage.sh prod field source_backend'|*'/app/storage.sh prod field target_backend') printf '%s\\n' "$RECORD_BACKEND";; + *'/app/storage.sh prod field job_id'|*'/app/storage.sh staging-go field job_id') printf '0123456789abcdef0123456789abcdef\\n';; + *) printf 'compose %s\\n' "$*" >> "$CALLS";; + esac +} +${functions} +${input ? `printf %s ${quote(input)} | ${command}` : command} +`; + const result=spawnSync(bash,['--noprofile','--norc'],{input:script,cwd:temp,encoding:'utf8',timeout:20000}); + result.calls=fs.readFileSync(log,'utf8');return result; +} +try{ + const sqlite=run("cmd_start ''"); + assert.equal(sqlite.status,0,sqlite.stderr||sqlite.error?.message); + assert(sqlite.calls.includes('up -d prod'),'SQLite app never started'); + assert(!sqlite.calls.includes('.postgres.yml'),'SQLite default selected a PostgreSQL overlay'); + const pg=run("cmd_start ''",'postgres'); + assert.equal(pg.status,0,pg.stderr); + assert(pg.calls.includes('-f docker-compose.postgres.yml'),'recorded PostgreSQL was replaced by stale SQLite bootstrap hint'); + const pending=run("cmd_start ''",'postgres','pending'); + assert.notEqual(pending.status,0,'pending conversion started'); + assert(!pending.calls.includes('up -d'),'pending conversion reached application startup'); + const update=run('cmd_update latest','postgres'); + assert.equal(update.status,0,update.stderr); + assert(update.calls.includes('build prod'),'update did not build its source'); + assert(update.calls.includes('docker-compose.postgres.yml'),'update discarded recorded backend'); + assert(!/storage.sh (switch|init)/.test(update.calls),'ordinary update changed backend'); + const defaultChoice=run('choose_storage','sqlite','unrecorded','\n'); + assert.equal(defaultChoice.status,0,defaultChoice.stderr); + assert(!defaultChoice.calls.includes('credentials prepared'),'default setup prepared PostgreSQL credentials'); + const postgresChoice=run('choose_storage','postgres','unrecorded','2\n'); + assert.equal(postgresChoice.status,0,postgresChoice.stderr); + assert(postgresChoice.calls.includes('credentials prepared'),'explicit PostgreSQL setup did not prepare its credentials'); + const keep=run('choose_storage','postgres','ready','1\n'); + assert.equal(keep.status,0,keep.stderr); + assert(!keep.calls.includes('credentials prepared'),'setup reconfigured credentials for an existing recorded backend'); + const switched=run('cmd_storage switch postgres','sqlite'); + assert.equal(switched.status,0,switched.stderr); + assert(switched.calls.includes('stop prod'),'conversion did not stop owned app processes'); + assert(switched.calls.includes('up -d --wait postgres'),'conversion did not prepare only the database service'); + assert(switched.calls.includes('bootstrap switch -backend=postgres'),'conversion bypassed verified CLI action'); + assert(!switched.calls.includes('up -d bootstrap'),'conversion pre-initialized its target schema'); + assert(!switched.calls.includes('up -d prod'),'conversion automatically restarted writers before operator validation'); + const reversed=run('cmd_storage switch sqlite','postgres'); + assert.equal(reversed.status,0,reversed.stderr); + assert(reversed.calls.includes('prod managed-postgres'),'reverse conversion skipped the recorded target guard'); + assert(reversed.calls.includes('bootstrap switch -backend=sqlite'),'reverse conversion bypassed the verified CLI'); + assert(!reversed.calls.includes('corescope_accounts'),'reverse routing touched the unselected account database'); + assert(reversed.calls.includes('stop postgres'),'completed reverse conversion left the managed PG service running'); + const sqliteResume=run('cmd_storage resume 0123456789abcdef0123456789abcdef','sqlite','pending'); + assert.equal(sqliteResume.status,0,sqliteResume.stderr); + assert(sqliteResume.calls.includes('prod resume -job-id='),'same-backend SQLite recovery bypassed the CLI'); + assert(!sqliteResume.calls.includes('.postgres.yml')&&!sqliteResume.calls.includes('up -d --wait postgres'),'SQLite-only recovery required PostgreSQL'); + const pgAbort=run('require_database_credentials(){ return 1; }; cmd_storage abort 0123456789abcdef0123456789abcdef','postgres','pending'); + assert.equal(pgAbort.status,0,pgAbort.stderr); + assert(pgAbort.calls.includes('prod abort -job-id='),'abort bypassed metadata-only CLI'); + assert(!pgAbort.calls.includes('.postgres.yml')&&!pgAbort.calls.includes('up -d --wait postgres'),'metadata abort started PostgreSQL'); + const stop=run('cmd_stop prod','postgres','pending'); + assert.equal(stop.status,0,stop.stderr); + assert(stop.calls.includes('stop prod'),'operator cannot stop app while recovery journal exists'); + console.log('PASS SQLite default start, authoritative PG routing/update, pending start refusal and stop during recovery'); +}finally{assert(path.resolve(temp).startsWith(path.resolve(os.tmpdir())+path.sep)&&path.basename(temp).startsWith('corescope-storage-manage-'));fs.rmSync(temp,{recursive:true});}