diff --git a/.eslintrc.json b/.eslintrc.json index a469ec574..deb317e8d 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -38,6 +38,7 @@ "MeshAudio": "readonly", "MeshConfigReady": "readonly", "NodeHopAnalytics": "readonly", + "ObserverAirtime": "readonly", "PAYLOAD_COLORS": "readonly", "PAYLOAD_TYPES": "readonly", "PERF_SLOW_MS": "readonly", diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index c85340853..301abfd7d 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -242,6 +242,9 @@ jobs: - name: Staging disk-monitor unit tests (issue #1684) run: bash scripts/staging/test-disk-monitor.sh + - name: Container service-user selection unit tests + run: bash docker/test-select-service-user.sh + - name: QA SQL parameter-binding unit tests (issue #1977) run: bash qa/scripts/test-blacklist-sql.sh @@ -543,6 +546,10 @@ jobs: done - name: Run Playwright E2E tests (fail-fast) + env: + # On master pushes every suite records the frontend coverage of the + # pages it opens into .nyc_output/ (#2203). Off on PRs to keep them fast. + NODE_OPTIONS: ${{ github.event_name == 'push' && '--require ./scripts/e2e-coverage-hook.js' || '' }} run: | # suite [VAR=value ...]: runs tests/e2e/ with # exactly those variables, but only on its own shard. The shard numbers @@ -579,6 +586,7 @@ jobs: suite 2 test-map-nodes-pagination-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B suite 3 test-observer-iata-1188-e2e.js BASE_URL=$B suite 2 test-issue-1639-observers-sort-e2e.js BASE_URL=$B + suite 2 test-issue-661-airtime-e2e.js BASE_URL=$B suite 2 test-issue-1758-ng-filter-rerenders-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B suite 2 test-nav-fluid-1055-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B suite 3 test-nav-priority-1102-e2e.js CHROMIUM_REQUIRE=1 BASE_URL=$B @@ -720,26 +728,26 @@ jobs: done echo "3 passed" - # Shard 2 runs test-e2e-playwright.js, the one suite that writes - # .nyc_output, so the coverage collection and its badge live there too. + # An extra coverage source on top of the suites: a tour of the main + # page groups. Its files land in .nyc_output/ beside the suites' files. - name: Collect frontend coverage (parallel) if: success() && github.event_name == 'push' && matrix.shard == 2 run: | BASE_URL=http://localhost:13581 node scripts/collect-frontend-coverage.js 2>&1 | tee fe-coverage-output.txt || true - - name: Generate frontend coverage badge - if: success() + # One file per suite is about 6 MB, mostly the same maps, so the shard + # merges them before upload. e2e-test merges the three shards and makes + # the frontend coverage badge. + # Coverage never decides the outcome: a failure here must not skip the + # shard upload below (if: success()) and so fail the E2E gate. + - name: Merge this shard's frontend coverage + if: success() && github.event_name == 'push' + continue-on-error: true run: | - mkdir -p .badges - if [ -f .nyc_output/frontend-coverage.json ] || [ -f .nyc_output/e2e-coverage.json ]; then - npx nyc report --reporter=text-summary --reporter=text 2>&1 | tee fe-report.txt - FE_COVERAGE=$(grep 'Statements' fe-report.txt | head -1 | grep -oP '[\d.]+(?=%)' || echo "0") - FE_COVERAGE=${FE_COVERAGE:-0} - FE_COLOR="red" - [ "$(echo "$FE_COVERAGE > 50" | bc -l 2>/dev/null)" = "1" ] && FE_COLOR="yellow" - [ "$(echo "$FE_COVERAGE > 80" | bc -l 2>/dev/null)" = "1" ] && FE_COLOR="brightgreen" - echo "{\"schemaVersion\":1,\"label\":\"frontend coverage\",\"message\":\"${FE_COVERAGE}%\",\"color\":\"${FE_COLOR}\"}" > .badges/frontend-coverage.json - echo "## Frontend: ${FE_COVERAGE}% coverage" >> $GITHUB_STEP_SUMMARY + if ls .nyc_output/*.json >/dev/null 2>&1; then + npx nyc merge .nyc_output coverage-shard.json + else + echo "::warning::no frontend coverage was recorded on shard ${{ matrix.shard }}" fi - name: Stop test server @@ -761,7 +769,7 @@ jobs: name: e2e-shard-${{ matrix.shard }} path: | e2e-output.txt - .badges/ + coverage-shard.json retention-days: 1 if-no-files-found: ignore include-hidden-files: true @@ -800,8 +808,9 @@ jobs: ls shards [ "$(ls -d shards/e2e-shard-* | wc -l)" -eq 3 ] || { echo "::error::expected results from 3 shards"; exit 1; } cat shards/e2e-shard-*/e2e-output.txt > e2e-output.txt - mkdir -p .badges - cp shards/e2e-shard-*/.badges/*.json .badges/ 2>/dev/null || true + # Start empty: the checkout holds master's old .badges/ files, + # which would otherwise go out in e2e-badges (#2136). + rm -rf .badges && mkdir .badges # Aggregate per-suite PASS/FAIL across every test-*-e2e.js summary. # The previous regex (grep -oP '[0-9]+(?=/)' | tail -1) caught a # stray digits-before-slash like the '2' in '2/3 tests passed' from @@ -819,6 +828,40 @@ jobs: echo "{\"schemaVersion\":1,\"label\":\"e2e tests\",\"message\":\"${E2E_MSG}\",\"color\":\"${E2E_COLOR}\"}" > .badges/e2e-tests.json echo "E2E: ${E2E_MSG} ($(grep -c '^=== E2E SUITE' e2e-output.txt) suite runs)" | tee -a "$GITHUB_STEP_SUMMARY" + - name: Set up Node.js 22 + if: github.event_name == 'push' + continue-on-error: true + uses: actions/setup-node@v5 + with: + node-version: '22' + + # Coverage from every suite on every shard (#2203), merged into one + # number. Only master pushes record it, and it never fails the gate. + - name: Generate frontend coverage badge + if: github.event_name == 'push' + continue-on-error: true + run: | + mkdir -p .nyc_output + for f in shards/e2e-shard-*/coverage-shard.json; do + [ -f "$f" ] && cp "$f" ".nyc_output/$(basename "$(dirname "$f")").json" + done + if ! ls .nyc_output/*.json >/dev/null 2>&1; then + echo "::warning::no shard recorded frontend coverage; the badge keeps its previous value" + exit 0 + fi + echo "Frontend coverage from: $(ls .nyc_output | paste -sd ' ' -)" + npx --yes nyc@18.0.0 report --reporter=text-summary --reporter=text 2>&1 | tee fe-report.txt + FE_COVERAGE=$(grep 'Statements' fe-report.txt | head -1 | grep -oP '[\d.]+(?=%)' || true) + if [ -z "$FE_COVERAGE" ]; then + echo "::warning::nyc report gave no statement coverage; the badge keeps its previous value" + exit 0 + fi + FE_COLOR="red" + [ "$(echo "$FE_COVERAGE > 50" | bc -l 2>/dev/null)" = "1" ] && FE_COLOR="yellow" + [ "$(echo "$FE_COVERAGE > 80" | bc -l 2>/dev/null)" = "1" ] && FE_COLOR="brightgreen" + echo "{\"schemaVersion\":1,\"label\":\"frontend coverage\",\"message\":\"${FE_COVERAGE}%\",\"color\":\"${FE_COLOR}\"}" > .badges/frontend-coverage.json + echo "## Frontend: ${FE_COVERAGE}% coverage" >> "$GITHUB_STEP_SUMMARY" + - name: Upload E2E badges uses: actions/upload-artifact@v6 with: @@ -1225,54 +1268,78 @@ jobs: # jobs, so ordering is unchanged. needs: [build-and-publish] runs-on: ubuntu-latest + # The badges go to their own `badges` branch (#2136). master is protected, + # so writing there took a PAT (BADGE_PUSH_TOKEN), which stopped working. + # The badges branch is unprotected, so the job's own GITHUB_TOKEN can + # write it. + permissions: + contents: write steps: - name: Checkout code uses: actions/checkout@v5 + with: + # The push below passes its own token; later steps need none. + persist-credentials: false - - name: Download Go coverage badges + # go-badges second, so its files win if e2e-badges ever carries go + # badges again. + - name: Download E2E badges continue-on-error: true uses: actions/download-artifact@v6 with: - name: go-badges - path: .badges/ + name: e2e-badges + path: badges-new/ - - name: Download E2E badges + - name: Download Go coverage badges continue-on-error: true uses: actions/download-artifact@v6 with: - name: e2e-badges - path: .badges/ + name: go-badges + path: badges-new/ - - name: Publish coverage badges to repo + - name: Publish coverage badges to the badges branch continue-on-error: true env: - GH_TOKEN: ${{ secrets.BADGE_PUSH_TOKEN }} + GH_TOKEN: ${{ github.token }} run: | - # GITHUB_TOKEN cannot push to protected branches (required status checks). - # Use admin PAT (BADGE_PUSH_TOKEN) via GitHub Contents API instead. - for badge in .badges/*.json; do - FILENAME=$(basename "$badge") - FILEPATH=".badges/$FILENAME" - CONTENT=$(base64 -w0 "$badge") - CURRENT_SHA=$(gh api "repos/${{ github.repository }}/contents/$FILEPATH" --jq '.sha' 2>/dev/null || echo "") - if [ -n "$CURRENT_SHA" ]; then - gh api "repos/${{ github.repository }}/contents/$FILEPATH" \ - -X PUT \ - -f message="ci: update $FILENAME [skip ci]" \ - -f content="$CONTENT" \ - -f sha="$CURRENT_SHA" \ - -f branch="master" \ - --silent 2>&1 || echo "Failed to update $FILENAME" - else - gh api "repos/${{ github.repository }}/contents/$FILEPATH" \ - -X PUT \ - -f message="ci: update $FILENAME [skip ci]" \ - -f content="$CONTENT" \ - -f branch="master" \ - --silent 2>&1 || echo "Failed to create $FILENAME" - fi - done - echo "Badge publish complete" + # Start from what the badges branch holds now (master's .badges/ the + # first time) and overwrite only the badges this run produced, so a + # missing artifact never removes a badge. The branch is one orphan + # commit, force-pushed, so it never grows. + # Every step checks its own result: bash ignores set -e inside a + # function whose status is tested, as the `if` below does. + publish() { + ls badges-new/*.json >/dev/null 2>&1 || { echo "no badge artifacts were downloaded"; return 1; } + out=$(mktemp -d) || return 1 + # Exit 2 means the branch does not exist yet; any other failure + # must not be taken for that, or the badges this run did not + # produce would be reset. + git ls-remote --exit-code --heads origin badges >/dev/null + case $? in + 0) + git fetch -q --depth=1 origin badges || return 1 + git archive FETCH_HEAD | tar -x -C "$out" || return 1 + ;; + 2) + if ls .badges/*.json >/dev/null 2>&1; then + cp .badges/*.json "$out"/ || return 1 + fi + ;; + *) return 1 ;; + esac + cp badges-new/*.json "$out"/ || return 1 + cd "$out" || return 1 + git init -q -b badges || return 1 + git add -- *.json || return 1 + git -c user.name="github-actions[bot]" \ + -c user.email="41898282+github-actions[bot]@users.noreply.github.com" \ + commit -q -m "ci: badges for ${GITHUB_SHA::8}" || return 1 + git push -q --force "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" HEAD:badges || return 1 + ls -- *.json + } + if ! (publish); then + echo "::warning title=Badges not published::Publishing to the badges branch failed (see the step log); the README badges keep their previous values." + fi - name: Post deployment summary run: | diff --git a/Dockerfile b/Dockerfile index 76514d2a9..35499abe1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -128,6 +128,23 @@ FROM alpine:3.20 RUN apk add --no-cache mosquitto mosquitto-clients supervisor caddy wget +# Account for the Go services when a deployment opts into running them +# unprivileged with RUN_AS_UID=1000, the usual choice. Creating it costs +# nothing when it goes unused, and saves the entrypoint creating one at start. +# +# Nothing in the image is given to it here. -H matters for that: without it +# busybox adduser creates the home directory itself, and /app would be +# corescope-owned mode 2755 — the setgid bit then giving gid 1000 to /app/data +# and to every file written into it, which Docker copies into each new named +# volume. The home field still points at /app; only the directory is left to +# the WORKDIR below, root-owned as in earlier releases. +RUN addgroup -S -g 1000 corescope && \ + adduser -S -u 1000 -G corescope -H -h /app -s /sbin/nologin corescope + +# Default: the services run as root, exactly as in earlier releases. The +# entrypoint overrides this when RUN_AS_UID asks for an unprivileged account. +ENV CORESCOPE_SERVICE_USER=root + WORKDIR /app # Go binaries (statically linked; they do not use this image's libc) @@ -155,6 +172,8 @@ RUN mkdir -p /app/data /var/lib/mosquitto /data/caddy && \ # Entrypoint COPY docker/entrypoint-go.sh /entrypoint.sh +# Sourced by the entrypoint, so deliberately not executable. +COPY docker/select-service-user.sh /select-service-user.sh RUN chmod +x /entrypoint.sh EXPOSE 80 443 1883 diff --git a/Makefile b/Makefile index 65c25df4d..34f71ccd5 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ # corescope build entry point. # -# This repo is 14 Go modules wired with `replace ../../internal/*` and no +# This repo is 16 Go modules wired with `replace ../../internal/*` and no # go.work, so every recipe changes into its module. The build flags live here # because the SQLite driver is cgo now (github.com/mattn/go-sqlite3) and the # flags stopped being something you can safely retype at each call site. diff --git a/README.md b/README.md index 7345d3d02..5b3822600 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ # CoreScope -[![Go Server Coverage](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/Kpa-clawbot/CoreScope/master/.badges/go-server-coverage.json)](https://github.com/Kpa-clawbot/CoreScope/actions/workflows/deploy.yml) -[![Go Ingestor Coverage](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/Kpa-clawbot/CoreScope/master/.badges/go-ingestor-coverage.json)](https://github.com/Kpa-clawbot/CoreScope/actions/workflows/deploy.yml) -[![E2E Tests](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/Kpa-clawbot/CoreScope/master/.badges/e2e-tests.json)](https://github.com/Kpa-clawbot/CoreScope/actions/workflows/deploy.yml) -[![Frontend Coverage](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/Kpa-clawbot/CoreScope/master/.badges/frontend-coverage.json)](https://github.com/Kpa-clawbot/CoreScope/actions/workflows/deploy.yml) +[![Go Server Coverage](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/Kpa-clawbot/CoreScope/badges/go-server-coverage.json)](https://github.com/Kpa-clawbot/CoreScope/actions/workflows/deploy.yml) +[![Go Ingestor Coverage](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/Kpa-clawbot/CoreScope/badges/go-ingestor-coverage.json)](https://github.com/Kpa-clawbot/CoreScope/actions/workflows/deploy.yml) +[![E2E Tests](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/Kpa-clawbot/CoreScope/badges/e2e-tests.json)](https://github.com/Kpa-clawbot/CoreScope/actions/workflows/deploy.yml) +[![Frontend Coverage](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/Kpa-clawbot/CoreScope/badges/frontend-coverage.json)](https://github.com/Kpa-clawbot/CoreScope/actions/workflows/deploy.yml) [![Deploy](https://github.com/Kpa-clawbot/CoreScope/actions/workflows/deploy.yml/badge.svg)](https://github.com/Kpa-clawbot/CoreScope/actions/workflows/deploy.yml) > High-performance mesh network analyzer powered by Go. Sub-millisecond packet queries, ~300 MB memory for 56K+ packets, real-time WebSocket broadcast, full channel decryption. @@ -12,7 +12,7 @@ Self-hosted, open-source MeshCore packet analyzer. Collects MeshCore packets via ## ⚡ Performance -The Go backend serves all 40+ API endpoints from an in-memory packet store with 5 indexes (hash, txID, obsID, observer, node). SQLite is for persistence only — reads never touch disk. +The Go backend serves its packet and analytics API from an in-memory packet store with 7 indexes (hash, txID, obsID, observer, node, path hop, payload type). SQLite is for persistence only — reads never touch disk. | Metric | Value | |--------|-------| @@ -28,7 +28,7 @@ See [PERFORMANCE.md](PERFORMANCE.md) for full benchmarks. ## ✨ Features ### 📡 Live Trace Map -Real-time animated map with packet route visualization, VCR-style playback controls, and a retro LCD clock. Replay the last 24 hours of mesh activity, scrub through the timeline, or watch packets flow live at up to 4× speed. +Real-time animated map with packet route visualization, VCR-style playback controls, and a retro LCD clock. Replay the last 24 hours of mesh activity, scrub through the timeline, or watch packets flow live at up to 8× speed. ![Live VCR playback — watch packets flow across the Bay Area mesh](docs/screenshots/MeshVCR.gif) @@ -59,11 +59,12 @@ Full experience on your phone — proper touch controls, iOS safe area support, ### And More -- **11 Analytics Tabs** — RF, topology, channels, hash stats, distance, route patterns, and more +- **17 Analytics Tabs** — RF, topology, channels, hash stats, distance, route patterns, scopes, and more - **Node Directory** — searchable list with role tabs, detail panel, QR codes, advert timeline - **Packet Tracing** — follow individual packets across observers with SNR/RSSI timeline - **Observer Status** — health monitoring, packet counts, uptime, per-observer analytics - **Hash Collision Matrix** — detect address collisions across the mesh +- **Scope Audit**: per repeater, which declared regions show no forwarding and which scopes it forwards without declaring them - **Channel Key Auto-Derivation** — hashtag channels (`#channel`) keys derived via SHA256 - **Multi-Broker MQTT** — connect to multiple brokers with per-source IATA filtering - **Dark / Light Mode** — auto-detects system preference, map tiles swap too @@ -73,6 +74,13 @@ Full experience on your phone — proper touch controls, iOS safe area support, - **Protobuf API Contract** — typed API definitions in `proto/` - **Accessible** — ARIA patterns, keyboard navigation, screen reader support +### Optional Features + +Off by default, enabled in `config.json`: + +- **User Accounts**: registration with email activation, settings sync, an admin area, hashtag channel proposals, mail notifications for watched nodes, and a personal data export. The dashboard stays public. See [docs/user-guide/accounts.md](docs/user-guide/accounts.md) +- **Mobile RX Coverage**: crowdsourced coverage from phones connected to a companion radio, drawn as hex bins on the RX Coverage page and on each node's Reach page. See [docs/client-rx-coverage.md](docs/client-rx-coverage.md) + ## Quick Start ### Pre-built Image (Recommended) @@ -162,8 +170,10 @@ Copy `config.example.json` to `config.json` and edit: | Variable | Description | |----------|-------------| -| `PORT` | Override config port | | `DB_PATH` | Override SQLite database path | +| `GOMEMLIMIT` | Go soft memory limit, wins over `runtime.maxMemoryMB` | + +The HTTP port is set with `port` in `config.json` or the server's `-port` flag. ## Architecture @@ -190,7 +200,7 @@ Observer → USB → │ Mosquitto ──→ Go Ingestor ──→ SQ 3. **Configure meshcoretomqtt** with your IATA region code and MQTT broker address 4. **Packets appear** on topic `meshcore/{IATA}/{PUBKEY}/packets` -Or POST raw hex packets to `POST /api/packets` for manual injection. +To decode a single raw hex packet without storing it, use `POST /api/decode`. ## Project Structure @@ -204,12 +214,12 @@ corescope/ │ │ ├── db.go # SQLite persistence layer │ │ ├── decoder.go # MeshCore packet decoder │ │ ├── websocket.go # WebSocket broadcast -│ │ └── *_test.go # 327 test functions +│ │ └── *_test.go # Server tests │ └── ingestor/ # Go MQTT ingestor │ ├── main.go # MQTT subscription + packet processing │ ├── decoder.go # Packet decoder (shared logic) │ ├── db.go # SQLite write path -│ └── *_test.go # 53 test functions +│ └── *_test.go # Ingestor tests ├── proto/ # Protobuf API definitions ├── public/ # Vanilla JS frontend (no build step) │ ├── index.html # SPA shell @@ -219,7 +229,7 @@ corescope/ │ ├── live.js # Live trace + VCR playback │ ├── channels.js # Channel chat │ ├── nodes.js # Node directory + detail views -│ ├── analytics.js # 11-tab analytics dashboard +│ ├── analytics.js # Analytics dashboard │ └── style.css # CSS variable theming (light/dark) ├── docker/ │ ├── supervisord-go.conf # Process manager (server + ingestor) @@ -253,14 +263,14 @@ that runs on Alpine or scratch. The container build does the same thing — see ### Test Suite -**380 Go tests** covering the backend, plus **150+ Node.js tests** for the frontend and legacy logic, plus **49 Playwright E2E tests** for browser validation. +Go tests cover the backend, Node.js unit tests in `tests/unit/` cover the frontend logic, and Playwright tests in `tests/e2e/` validate the UI in a browser. ```bash # Go backend tests cd cmd/server && go test ./... -v cd cmd/ingestor && go test ./... -v -# Or across all 14 modules at once +# Or across all 16 modules at once make test # Node.js frontend + integration tests @@ -284,9 +294,9 @@ If you're running an existing Node.js deployment, see [docs/go-migration.md](doc Contributions welcome. Please read [AGENTS.md](AGENTS.md) for coding conventions, testing requirements, and engineering principles before submitting a PR. -**Live instance:** [analyzer.00id.net](https://analyzer.00id.net) — all API endpoints are public, no auth required. +**Live instance:** [analyzer.on8ar.eu](https://analyzer.on8ar.eu): all read endpoints are public, no auth required. -**API Documentation:** CoreScope auto-generates an OpenAPI 3.0 spec. Browse the interactive Swagger UI at [`/api/docs`](https://analyzer.00id.net/api/docs) or fetch the machine-readable spec at [`/api/spec`](https://analyzer.00id.net/api/spec). +**API Documentation:** CoreScope auto-generates an OpenAPI 3.0 spec. Browse the interactive Swagger UI at [`/api/docs`](https://analyzer.on8ar.eu/api/docs) or fetch the machine-readable spec at [`/api/spec`](https://analyzer.on8ar.eu/api/spec). ## License diff --git a/cmd/ingestor/approved_channels.go b/cmd/ingestor/approved_channels.go index 3cd42e969..c95a43da9 100644 --- a/cmd/ingestor/approved_channels.go +++ b/cmd/ingestor/approved_channels.go @@ -51,6 +51,11 @@ func newChannelKeySet(configured map[string]string, usersDBPath string, max int) // Snapshot returns the key map to decode one message with. func (s *channelKeySet) Snapshot() map[string]string { return *s.cur.Load() } +// current returns the snapshot pointer itself. refresh stores a new one on +// every successful read, changed or not, so callers that compare pointers +// must treat a new one as "maybe changed" (see keyBackfiller.keySetMoved). +func (s *channelKeySet) current() *map[string]string { return s.cur.Load() } + // refresh re-reads the approved names and swaps in configured + approved. // A failed read keeps the current snapshot: only a successful read changes // the set, so a locked or missing users.db never drops a key. diff --git a/cmd/ingestor/key_backfill.go b/cmd/ingestor/key_backfill.go new file mode 100644 index 000000000..3aa9867f5 --- /dev/null +++ b/cmd/ingestor/key_backfill.go @@ -0,0 +1,622 @@ +package main + +import ( + "context" + "crypto/sha256" + "database/sql" + "encoding/hex" + "fmt" + "log" + "sort" + "strings" + "time" +) + +// Key-set backfill (#2107). +// +// A GRP_TXT stored before its channel key was known keeps channel_hash +// "enc_XX" forever, and a transport-scoped packet stored before its region key +// was known keeps scope_name "" forever: ingest decides once, at insert time. +// When the key set grows - a hashChannels entry is added and the ingestor +// restarts, a hashtag channel is approved through user management, or the +// autoRegionKeys tier picks up a newly declared region - this worker +// re-decides exactly those unresolved rows with the keys now in force. +// +// Rules, each pinned by a test in key_backfill_test.go: +// +// - Only unresolved rows are touched. A decrypted channel or a named scope is +// never rewritten, even if a newer key would now decide it differently. +// Every UPDATE carries the unresolved value in its WHERE clause, so a row +// that live ingest or another writer resolved meanwhile is left alone. +// - The decision is the one ingest makes today: DecodePacket with the full +// channel key map, and regionKeySnapshot.match with the full region +// snapshot - including abstaining on an ambiguous collision (#1609). The +// ingest-path counters are not touched; they measure live traffic. +// - Bounded and resumable: fixed windows of transmissions.id, one +// transaction per window holding the rewrites, their feed rows and the +// advanced cursor together. A cancelled or crashed run resumes from the +// last commit; nothing is applied twice and nothing is skipped. +// - A run is tied to the fingerprint of the key set it started with. If the +// key set changes mid-run the run restarts from the beginning with the new +// set, so a key added halfway through still sees all of history. Both key +// sets swap their snapshot on every refresh, changed or not, so a new +// snapshot only counts as a change when its fingerprint differs. +// - Each key gets one complete pass. Its fingerprint is recorded when a run +// finishes; a restart with an unchanged key set does no work. A key that +// leaves the set loses its record, so if it returns it is replayed over +// whatever arrived while it was absent. +// - Every rewritten transmission gets a tx_rewrite_feed row in the same +// transaction. The server tails that feed (cmd/server/tx_rewrite_feed.go) +// and refreshes the in-memory copy, so no restart is needed. The feed is +// pruned on its own ticker, not only after a pass, so it stays bounded +// even when no key ever changes. + +const ( + keyBackfillKindChannel = "channel" + keyBackfillKindRegion = "region" + + // keyBackfillWindow is how many transmission ids one batch covers. It + // bounds the READ as well as the write: the ingestor has a single SQLite + // connection, and unresolved rows are sparse (hundreds in a million), so a + // "next N matching rows" query would scan most of the table while live + // ingest waits. A fixed id window keeps every query a short primary-key + // range scan however few rows match. ~230 batches cover 1.1M ids. + keyBackfillWindow = 5000 + // keyBackfillYield is the pause between batches that hands the single + // SQLite writer back to live ingest. + keyBackfillYield = 10 * time.Millisecond + // txRewriteFeedRetention bounds the feed. The server only needs rows it + // has not tailed yet; a server that was down longer than this reloads the + // rewritten rows from the DB on startup anyway. + txRewriteFeedRetention = 24 * time.Hour + // txRewriteFeedPruneEvery is how often the feed is pruned between passes. + // A pass may never come (autoRegionKeys off, no approved channels), so + // pruning cannot depend on one. + txRewriteFeedPruneEvery = time.Hour +) + +// keyBackfiller re-decides unresolved rows when the key set grows. One +// goroutine runs every pass, so two passes never race each other; Kick only +// asks that goroutine to look again. +type keyBackfiller struct { + store *Store + channels *channelKeySet // configured + approved; the set ingest decodes with + regionSet *regionKeySet + window int64 // ids per batch; see keyBackfillWindow + yield time.Duration + pruneEvery time.Duration // see txRewriteFeedPruneEvery + wake chan struct{} +} + +func newKeyBackfiller(store *Store, channels *channelKeySet, regionSet *regionKeySet) *keyBackfiller { + return &keyBackfiller{ + store: store, + channels: channels, + regionSet: regionSet, + window: keyBackfillWindow, + yield: keyBackfillYield, + pruneEvery: txRewriteFeedPruneEvery, + wake: make(chan struct{}, 1), + } +} + +// Kick schedules a pass. It never blocks: a pass already pending absorbs it. +func (b *keyBackfiller) Kick() { + select { + case b.wake <- struct{}{}: + default: + } +} + +// Start runs a first pass and then one more per Kick, and prunes the feed +// every pruneEvery, until ctx is cancelled. Passes and prunes share one +// goroutine, so they never run concurrently. The returned channel closes when +// the goroutine has exited. +func (b *keyBackfiller) Start(ctx context.Context) <-chan struct{} { + done := make(chan struct{}) + every := b.pruneEvery + if every <= 0 { + every = txRewriteFeedPruneEvery + } + b.Kick() + go func() { + defer close(done) + prune := time.NewTicker(every) + defer prune.Stop() + for { + select { + case <-ctx.Done(): + return + case <-prune.C: + b.pruneFeed(ctx) + case <-b.wake: + b.pass(ctx) + } + } + }() + return done +} + +// pass runs both kinds once and prunes the feed. Errors are logged, not +// fatal: the state row keeps the cursor, so the next Kick or restart resumes. +func (b *keyBackfiller) pass(ctx context.Context) { + defer func() { + if r := recover(); r != nil { + log.Printf("[key-backfill] panic recovered: %v", r) + } + }() + for _, kind := range []string{keyBackfillKindChannel, keyBackfillKindRegion} { + if err := b.runKind(ctx, kind); err != nil { + if ctx.Err() != nil { + log.Printf("[key-backfill] %s pass interrupted; will resume from the last committed batch", kind) + return + } + log.Printf("[key-backfill] %s pass failed, will retry on the next key change or restart: %v", kind, err) + } + } + b.pruneFeed(ctx) +} + +// pruneFeed drops feed rows older than txRewriteFeedRetention. +func (b *keyBackfiller) pruneFeed(ctx context.Context) { + if _, err := b.store.db.ExecContext(ctx, `DELETE FROM tx_rewrite_feed WHERE created_at < ?`, + time.Now().Add(-txRewriteFeedRetention).Unix()); err != nil && ctx.Err() == nil { + log.Printf("[key-backfill] pruning tx_rewrite_feed: %v", err) + } +} + +// keyFingerprint identifies one key by name AND key material, so a channel +// whose secret changes under the same name counts as a new key. +func keyFingerprint(kind, name string, key []byte) string { + h := sha256.New() + h.Write([]byte(kind)) + h.Write([]byte{0}) + h.Write([]byte(name)) + h.Write([]byte{0}) + h.Write(key) + return hex.EncodeToString(h.Sum(nil)[:16]) +} + +// keySetFingerprint identifies a whole key set independent of map order. +func keySetFingerprint(fps []string) string { + sorted := append([]string(nil), fps...) + sort.Strings(sorted) + h := sha256.Sum256([]byte(strings.Join(sorted, ","))) + return hex.EncodeToString(h[:16]) +} + +// channelKeyHashByte is the channel-hash byte a GRP_TXT encrypted under key +// carries in clear: the first byte of SHA-256 over the 16-byte secret +// (MeshCore firmware, GroupChannel::hash). ok is false for a malformed key. +func channelKeyHashByte(keyHex string) (byte, bool) { + key, err := hex.DecodeString(keyHex) + if err != nil || len(key) != 16 { + return 0, false + } + h := sha256.Sum256(key) + return h[0], true +} + +// keyBackfillPlan is what one kind needs to run: every key fingerprint in +// force, the subset never given a full pass, and for channels the hash bytes +// of the pending keys (only rows with those bytes can newly decrypt). +type keyBackfillPlan struct { + all []string + pending []string + channelHash []string // "enc_XX" values, channel kind only + channels map[string]string // channel kind: the key map this run decodes with + channelSnap *map[string]string // channel kind: the snapshot channels came from + snapshot *regionKeySnapshot // region kind +} + +// channelKeyFingerprints returns one fingerprint per channel key. +func channelKeyFingerprints(keys map[string]string) []string { + fps := make([]string, 0, len(keys)) + for name, keyHex := range keys { + raw, _ := hex.DecodeString(keyHex) + fps = append(fps, keyFingerprint(keyBackfillKindChannel, name, raw)) + } + return fps +} + +// regionKeyFingerprints returns one fingerprint per region key. +func regionKeyFingerprints(snap *regionKeySnapshot) []string { + fps := make([]string, 0, len(snap.all)) + for name, key := range snap.all { + fps = append(fps, keyFingerprint(keyBackfillKindRegion, name, key)) + } + return fps +} + +func (b *keyBackfiller) plan(kind string, done map[string]bool) keyBackfillPlan { + var p keyBackfillPlan + switch kind { + case keyBackfillKindChannel: + p.channelSnap = b.channels.current() + p.channels = *p.channelSnap + seenHash := map[string]bool{} + for name, keyHex := range p.channels { + raw, _ := hex.DecodeString(keyHex) + fp := keyFingerprint(kind, name, raw) + p.all = append(p.all, fp) + if done[fp] { + continue + } + p.pending = append(p.pending, fp) + if hb, ok := channelKeyHashByte(keyHex); ok { + enc := fmt.Sprintf("enc_%02X", hb) + if !seenHash[enc] { + seenHash[enc] = true + p.channelHash = append(p.channelHash, enc) + } + } + } + sort.Strings(p.channelHash) + case keyBackfillKindRegion: + p.snapshot = b.regionSet.snapshot() + for name, key := range p.snapshot.all { + fp := keyFingerprint(kind, name, key) + p.all = append(p.all, fp) + if !done[fp] { + p.pending = append(p.pending, fp) + } + } + } + return p +} + +// runKind brings one kind up to date with the key set currently in force. +func (b *keyBackfiller) runKind(ctx context.Context, kind string) error { + db := b.store.db + done, err := loadDoneKeys(ctx, db, kind) + if err != nil { + return err + } + p := b.plan(kind, done) + + // A key that left the set forgets its pass, so it is replayed if it + // returns: rows that arrived while it was absent were decided without it. + if err := forgetRemovedKeys(ctx, db, kind, p.all, done); err != nil { + return err + } + + if len(p.pending) == 0 { + // Nothing new. Drop a state row a since-superseded run left behind. + _, err := db.ExecContext(ctx, `DELETE FROM key_backfill_state WHERE kind=?`, kind) + return err + } + + runFP := keySetFingerprint(p.all) + cursor, upper, rewritten, err := b.resumeOrStart(ctx, kind, runFP) + if err != nil { + return err + } + if cursor == 0 { + log.Printf("[key-backfill] %s: %d of %d key(s) not yet applied to history; scanning transmissions up to id %d", + kind, len(p.pending), len(p.all), upper) + } else { + log.Printf("[key-backfill] %s: resuming at id %d of %d (%d row(s) rewritten so far)", kind, cursor, upper, rewritten) + } + + for cursor < upper { + if err := ctx.Err(); err != nil { + return err + } + // The key set moved under us: abandon this run, the next pass starts + // a fresh one over the new set. Both kinds refresh at runtime + // (approved channels, autoRegionKeys). + if b.keySetMoved(kind, &p, runFP) { + log.Printf("[key-backfill] %s key set changed mid-run; restarting from the beginning", kind) + b.Kick() + return nil + } + next, n, err := b.batchOnce(ctx, kind, p, runFP, cursor, upper) + if err != nil { + return err + } + rewritten += int64(n) + cursor = next + timer := time.NewTimer(b.yield) + select { + case <-ctx.Done(): + timer.Stop() + return ctx.Err() + case <-timer.C: + } + } + + if err := finishRun(ctx, db, kind, runFP, p.all); err != nil { + return err + } + b.store.Stats.IncBackfill("key_" + kind) + log.Printf("[key-backfill] %s: pass complete, %d row(s) rewritten", kind, rewritten) + return nil +} + +// keySetMoved reports whether the keys in force now differ from the ones the +// run for runFP works with. Both key sets store a fresh snapshot on every +// refresh even when nothing changed, so a new snapshot is only a hint: when +// its fingerprint still equals runFP the run adopts it and carries on at its +// cursor instead of being abandoned. +func (b *keyBackfiller) keySetMoved(kind string, p *keyBackfillPlan, runFP string) bool { + switch kind { + case keyBackfillKindChannel: + cur := b.channels.current() + if cur == p.channelSnap { + return false + } + if keySetFingerprint(channelKeyFingerprints(*cur)) != runFP { + return true + } + p.channelSnap, p.channels = cur, *cur + case keyBackfillKindRegion: + cur := b.regionSet.snapshot() + if cur == p.snapshot { + return false + } + if keySetFingerprint(regionKeyFingerprints(cur)) != runFP { + return true + } + p.snapshot = cur + } + return false +} + +func loadDoneKeys(ctx context.Context, db *sql.DB, kind string) (map[string]bool, error) { + rows, err := db.QueryContext(ctx, `SELECT key_fp FROM key_backfill_done WHERE kind=?`, kind) + if err != nil { + return nil, fmt.Errorf("load done keys: %w", err) + } + defer rows.Close() + done := map[string]bool{} + for rows.Next() { + var fp string + if err := rows.Scan(&fp); err != nil { + return nil, err + } + done[fp] = true + } + return done, rows.Err() +} + +func forgetRemovedKeys(ctx context.Context, db *sql.DB, kind string, all []string, done map[string]bool) error { + inForce := make(map[string]bool, len(all)) + for _, fp := range all { + inForce[fp] = true + } + for fp := range done { + if inForce[fp] { + continue + } + if _, err := db.ExecContext(ctx, `DELETE FROM key_backfill_done WHERE kind=? AND key_fp=?`, kind, fp); err != nil { + return fmt.Errorf("forget removed key: %w", err) + } + } + return nil +} + +// resumeOrStart returns the cursor of the run for runFP, starting a new run +// when there is none or the stored one was for a different key set. A new run +// is bounded by the newest transmission now: anything inserted later is +// decided by live ingest with the same keys. +func (b *keyBackfiller) resumeOrStart(ctx context.Context, kind, runFP string) (cursor, upper, rewritten int64, err error) { + db := b.store.db + var storedFP string + err = db.QueryRowContext(ctx, `SELECT run_fp, cursor, upper, rewritten FROM key_backfill_state WHERE kind=?`, kind). + Scan(&storedFP, &cursor, &upper, &rewritten) + if err == nil && storedFP == runFP { + return cursor, upper, rewritten, nil + } + if err != nil && err != sql.ErrNoRows { + return 0, 0, 0, fmt.Errorf("load backfill state: %w", err) + } + if err := db.QueryRowContext(ctx, `SELECT COALESCE(MAX(id),0) FROM transmissions`).Scan(&upper); err != nil { + return 0, 0, 0, fmt.Errorf("backfill upper bound: %w", err) + } + if _, err := db.ExecContext(ctx, `INSERT OR REPLACE INTO key_backfill_state(kind, run_fp, cursor, upper, rewritten) VALUES(?,?,0,?,0)`, + kind, runFP, upper); err != nil { + return 0, 0, 0, fmt.Errorf("start backfill run: %w", err) + } + return 0, upper, 0, nil +} + +func finishRun(ctx context.Context, db *sql.DB, kind, runFP string, all []string) error { + writerMu.Lock() + defer writerMu.Unlock() + tx, err := db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + for _, fp := range all { + if _, err := tx.ExecContext(ctx, `INSERT OR IGNORE INTO key_backfill_done(kind, key_fp) VALUES(?,?)`, kind, fp); err != nil { + return fmt.Errorf("record done key: %w", err) + } + } + if _, err := tx.ExecContext(ctx, `DELETE FROM key_backfill_state WHERE kind=? AND run_fp=?`, kind, runFP); err != nil { + return err + } + return tx.Commit() +} + +// keyRewrite is one row's new state, decided before the writer is taken. +type keyRewrite struct { + id int64 + channelHash string // channel kind: the decrypted channel name + decodedJSON string // channel kind + oldChannel string // channel kind: the enc_XX value the row must still hold + scopeName string // region kind + advertFrom string // region kind: ADVERT origin, for nodes.default_scope +} + +// batchOnce decides the unresolved rows with id in (cursor, cursor+window] +// and commits the rewrites, their feed rows and the advanced cursor together. +// It returns the new cursor (always > cursor, capped at upper) and the number +// of rows actually rewritten. +func (b *keyBackfiller) batchOnce(ctx context.Context, kind string, p keyBackfillPlan, runFP string, cursor, upper int64) (int64, int, error) { + db := b.store.db + window := b.window + if window <= 0 { + window = keyBackfillWindow + } + hi := cursor + window + if hi > upper { + hi = upper + } + var ( + rows *sql.Rows + err error + ) + switch kind { + case keyBackfillKindChannel: + if len(p.channelHash) == 0 { + // No pending key has a usable hash byte: nothing in the rest of + // the range can match, so jump the cursor straight to the end. + return b.commitEmpty(ctx, kind, runFP, upper) + } + args := []interface{}{cursor, hi} + for _, h := range p.channelHash { + args = append(args, h) + } + rows, err = db.QueryContext(ctx, ` + SELECT id, COALESCE(raw_hex,''), channel_hash FROM transmissions + WHERE id > ? AND id <= ? AND payload_type = 5 + AND channel_hash IN (`+strings.TrimSuffix(strings.Repeat("?,", len(p.channelHash)), ",")+`) + ORDER BY id`, args...) + case keyBackfillKindRegion: + rows, err = db.QueryContext(ctx, ` + SELECT id, COALESCE(raw_hex,''), '' FROM transmissions + WHERE id > ? AND id <= ? AND scope_name = '' + ORDER BY id`, cursor, hi) + default: + return cursor, 0, fmt.Errorf("unknown backfill kind %q", kind) + } + if err != nil { + return cursor, 0, fmt.Errorf("select unresolved rows: %w", err) + } + + var rewrites []keyRewrite + for rows.Next() { + var ( + id int64 + raw, old string + ) + if err := rows.Scan(&id, &raw, &old); err != nil { + rows.Close() + return cursor, 0, err + } + // Undecodable rows are simply not rewritten; the window moves past + // them, so they are never re-selected within this run. + if r, ok := b.decide(kind, p, id, raw, old); ok { + rewrites = append(rewrites, r) + } + } + err = rows.Err() + rows.Close() // the single connection must be free before BeginTx + if err != nil { + return cursor, 0, err + } + + n, err := b.commit(ctx, kind, runFP, hi, rewrites) + if err != nil { + return cursor, 0, err + } + return hi, n, nil +} + +// commitEmpty advances the cursor over a window that cannot contain a match. +func (b *keyBackfiller) commitEmpty(ctx context.Context, kind, runFP string, hi int64) (int64, int, error) { + if _, err := b.commit(ctx, kind, runFP, hi, nil); err != nil { + return 0, 0, err + } + return hi, 0, nil +} + +// decide is the ingest-time decision, re-run with the keys in force now. +func (b *keyBackfiller) decide(kind string, p keyBackfillPlan, id int64, raw, oldChannel string) (keyRewrite, bool) { + if raw == "" { + return keyRewrite{}, false + } + switch kind { + case keyBackfillKindChannel: + decoded, err := DecodePacket(raw, p.channels, false) + if err != nil || decoded.Payload.Type != "CHAN" || decoded.Payload.Channel == "" { + return keyRewrite{}, false + } + return keyRewrite{ + id: id, + channelHash: decoded.Payload.Channel, + decodedJSON: PayloadJSON(&decoded.Payload), + oldChannel: oldChannel, + }, true + case keyBackfillKindRegion: + decoded, err := DecodePacket(raw, nil, false) + if err != nil || decoded.TransportCodes == nil || decoded.TransportCodes.Code1 == "0000" { + return keyRewrite{}, false + } + m := p.snapshot.match(byte(decoded.Header.PayloadType), decoded.payloadRaw, decoded.TransportCodes.Code1) + if m.Name == "" { // none, or an ambiguous collision: stays unmatched + return keyRewrite{}, false + } + r := keyRewrite{id: id, scopeName: m.Name} + if decoded.Header.PayloadType == PayloadADVERT && decoded.Payload.PubKey != "" { + r.advertFrom = strings.ToLower(decoded.Payload.PubKey) + } + return r, true + } + return keyRewrite{}, false +} + +func (b *keyBackfiller) commit(ctx context.Context, kind, runFP string, next int64, rewrites []keyRewrite) (int, error) { + writerMu.Lock() + defer writerMu.Unlock() + tx, err := b.store.db.BeginTx(ctx, nil) + if err != nil { + return 0, err + } + defer tx.Rollback() + + n := 0 + for _, r := range rewrites { + var res sql.Result + switch kind { + case keyBackfillKindChannel: + res, err = tx.ExecContext(ctx, `UPDATE transmissions SET channel_hash=?, decoded_json=? WHERE id=? AND channel_hash=?`, + r.channelHash, r.decodedJSON, r.id, r.oldChannel) + case keyBackfillKindRegion: + res, err = tx.ExecContext(ctx, `UPDATE transmissions SET scope_name=? WHERE id=? AND scope_name=''`, r.scopeName, r.id) + } + if err != nil { + return 0, fmt.Errorf("rewrite tx %d: %w", r.id, err) + } + if changed, _ := res.RowsAffected(); changed == 0 { + continue // resolved meanwhile by someone else; theirs stands + } + n++ + if _, err := tx.ExecContext(ctx, `INSERT INTO tx_rewrite_feed(tx_id) VALUES(?)`, r.id); err != nil { + return 0, fmt.Errorf("feed tx %d: %w", r.id, err) + } + // A node whose default_scope is still unknown gets the region its own + // advert now names. A known default_scope is newer evidence than a + // backfilled advert can be, so it is never overwritten. + if r.advertFrom != "" { + for _, table := range []string{"nodes", "inactive_nodes"} { + if _, err := tx.ExecContext(ctx, `UPDATE `+table+` SET default_scope=? + WHERE public_key=? AND (default_scope IS NULL OR default_scope='')`, r.scopeName, r.advertFrom); err != nil { + return 0, fmt.Errorf("default_scope for %s: %w", r.advertFrom, err) + } + } + } + } + res, err := tx.ExecContext(ctx, `UPDATE key_backfill_state SET cursor=?, rewritten=rewritten+? WHERE kind=? AND run_fp=?`, + next, n, kind, runFP) + if err != nil { + return 0, fmt.Errorf("advance cursor: %w", err) + } + if moved, _ := res.RowsAffected(); moved != 1 { + // The state row was replaced by a newer run; this batch's rewrites + // are still correct, but its cursor belongs to nobody. + return 0, fmt.Errorf("backfill state for %s run %s vanished", kind, runFP) + } + return n, tx.Commit() +} diff --git a/cmd/ingestor/key_backfill_test.go b/cmd/ingestor/key_backfill_test.go new file mode 100644 index 000000000..d6699321a --- /dev/null +++ b/cmd/ingestor/key_backfill_test.go @@ -0,0 +1,631 @@ +package main + +import ( + "context" + "crypto/aes" + "crypto/hmac" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "fmt" + "strings" + "sync/atomic" + "testing" + "time" +) + +// --- fixture builders: real wire frames, encoded the way firmware does --- + +// grpTxtFrame builds a FLOOD GRP_TXT encrypted under keyHex, exactly as +// decryptChannelMessage expects it: AES-128-ECB over ts(4 LE)+flags(1)+text, +// zero-padded, MAC = first two bytes of HMAC-SHA256(key||16 zero bytes). +func grpTxtFrame(t *testing.T, keyHex, text string) string { + t.Helper() + key, err := hex.DecodeString(keyHex) + if err != nil || len(key) != 16 { + t.Fatalf("bad key %q", keyHex) + } + plain := make([]byte, 5, 64) + binary.LittleEndian.PutUint32(plain[0:4], 1767225600) + plain = append(plain, []byte(text)...) + for len(plain)%aes.BlockSize != 0 { + plain = append(plain, 0) + } + block, _ := aes.NewCipher(key) + ct := make([]byte, len(plain)) + for i := 0; i < len(plain); i += aes.BlockSize { + block.Encrypt(ct[i:i+aes.BlockSize], plain[i:i+aes.BlockSize]) + } + secret := make([]byte, 32) + copy(secret, key) + m := hmac.New(sha256.New, secret) + m.Write(ct) + mac := m.Sum(nil)[:2] + hb, _ := channelKeyHashByte(keyHex) + payload := append([]byte{hb}, mac...) + payload = append(payload, ct...) + return "15" + "00" + hex.EncodeToString(payload) // FLOOD|GRP_TXT, no hops +} + +// regionCodeBytes is the transport code1 a sender in region key would put on +// payload - the inverse of matchingRegions, kept independent of it on purpose. +func regionCodeBytes(key []byte, payloadType byte, payload []byte) []byte { + m := hmac.New(sha256.New, key) + m.Write([]byte{payloadType}) + m.Write(payload) + sum := m.Sum(nil) + code := uint16(sum[0]) | uint16(sum[1])<<8 + if code == 0 { + code = 1 + } else if code == 0xFFFF { + code = 0xFFFE + } + return []byte{byte(code & 0xFF), byte(code >> 8)} +} + +func regionKey(name string) []byte { + h := sha256.Sum256([]byte(name)) + return h[:16] +} + +// transportFrame builds a TRANSPORT_FLOOD frame of payloadType whose code1 is +// what region key would produce for payload. +func transportFrame(key []byte, payloadType byte, payload []byte) string { + header := []byte{payloadType << 2} // route type 0 = TRANSPORT_FLOOD + code1 := regionCodeBytes(key, payloadType, payload) + return hex.EncodeToString(header) + hex.EncodeToString(code1) + "0000" + "00" + hex.EncodeToString(payload) +} + +// advertPayload is the smallest ADVERT the decoder accepts: pubkey(32) + +// timestamp(4) + signature(64) + flags(1, repeater) + name. +func advertPayload(pubkey byte, name string) []byte { + p := make([]byte, 0, 110) + for i := 0; i < 32; i++ { + p = append(p, pubkey) + } + ts := make([]byte, 4) + binary.LittleEndian.PutUint32(ts, 1767225600) + p = append(p, ts...) + p = append(p, make([]byte, 64)...) + p = append(p, 0x02) + return append(p, []byte(name)...) +} + +// insertTx writes a row the way InsertTransmission leaves one: scope is nil +// for a non-transport row, "" for unmatched, "#name" for matched. +func insertTx(t *testing.T, s *Store, raw string, payloadType int, channel string, decoded string, scope *string) int64 { + t.Helper() + var ch interface{} + if channel != "" { + ch = channel + } + res, err := s.db.Exec(`INSERT INTO transmissions(raw_hex, hash, first_seen, payload_type, decoded_json, channel_hash, scope_name) + VALUES(?,?,?,?,?,?,?)`, raw, fmt.Sprintf("kb-%d", txSeq.Add(1)), "2026-10-01T00:00:00Z", + payloadType, decoded, ch, scope) + if err != nil { + t.Fatal(err) + } + id, _ := res.LastInsertId() + return id +} + +// txSeq keeps fixture hashes unique; transmissions.hash is UNIQUE. +var txSeq atomic.Int64 + +func strp(s string) *string { return &s } + +func regionSetOf(names ...string) *regionKeySet { + all := map[string][]byte{} + for _, n := range names { + all[n] = regionKey(n) + } + s := ®ionKeySet{} + s.cur.Store(®ionKeySnapshot{all: all, explicit: map[string]bool{}}) + return s +} + +func newTestBackfiller(s *Store, channelKeys map[string]string, rs *regionKeySet) *keyBackfiller { + b := newKeyBackfiller(s, newChannelKeySet(channelKeys, "", 0), rs) + b.yield = 0 + return b +} + +func feedRows(t *testing.T, s *Store) []int64 { + t.Helper() + rows, err := s.db.Query(`SELECT tx_id FROM tx_rewrite_feed ORDER BY id`) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + var out []int64 + for rows.Next() { + var id int64 + rows.Scan(&id) + out = append(out, id) + } + return out +} + +func colOf(t *testing.T, s *Store, col string, id int64) string { + t.Helper() + var v *string + if err := s.db.QueryRow(`SELECT `+col+` FROM transmissions WHERE id=?`, id).Scan(&v); err != nil { + t.Fatal(err) + } + if v == nil { + return "" + } + return *v +} + +// --- channels --- + +func TestKeyBackfillChannelDecryptsOnlyUnresolvedRows(t *testing.T) { + s := newTestStore(t) + saar := deriveHashtagChannelKey("#saar") + other := deriveHashtagChannelKey("#nobody-has-this") + + encSaar := grpTxtFrame(t, saar, "Alice: moin") + hb, _ := channelKeyHashByte(saar) + encTag := fmt.Sprintf("enc_%02X", hb) + resolvable := insertTx(t, s, encSaar, 5, encTag, `{"type":"GRP_TXT"}`, nil) + + // Already decrypted: never touched, even though the key would decrypt it. + already := insertTx(t, s, encSaar+"00", 5, "#saar", `{"type":"CHAN","keep":1}`, nil) + + // A different, still-unknown channel stays encrypted. + encOther := grpTxtFrame(t, other, "Bob: hi") + hbo, _ := channelKeyHashByte(other) + unknown := insertTx(t, s, encOther, 5, fmt.Sprintf("enc_%02X", hbo), `{"type":"GRP_TXT"}`, nil) + + b := newTestBackfiller(s, map[string]string{"#saar": saar}, nil) + if err := b.runKind(context.Background(), keyBackfillKindChannel); err != nil { + t.Fatal(err) + } + + if got := colOf(t, s, "channel_hash", resolvable); got != "#saar" { + t.Fatalf("resolvable row channel_hash = %q, want #saar", got) + } + if dj := colOf(t, s, "decoded_json", resolvable); !strings.Contains(dj, `"type":"CHAN"`) || !strings.Contains(dj, "moin") { + t.Fatalf("decoded_json not what ingest would store: %s", dj) + } + if got := colOf(t, s, "decoded_json", already); got != `{"type":"CHAN","keep":1}` { + t.Fatalf("already-decrypted row was rewritten: %s", got) + } + if got := colOf(t, s, "channel_hash", unknown); !strings.HasPrefix(got, "enc_") { + t.Fatalf("row with no key was touched: %s", got) + } + if f := feedRows(t, s); len(f) != 1 || f[0] != resolvable { + t.Fatalf("feed = %v, want exactly [%d]", f, resolvable) + } +} + +// Same decision as ingest: a backfilled row's decoded_json is byte-for-byte +// what BuildPacketData would have stored had the key been known at insert. +func TestKeyBackfillChannelMatchesIngestDecision(t *testing.T) { + s := newTestStore(t) + key := deriveHashtagChannelKey("#parity") + raw := grpTxtFrame(t, key, "Carol: same bytes") + hb, _ := channelKeyHashByte(key) + id := insertTx(t, s, raw, 5, fmt.Sprintf("enc_%02X", hb), `{"type":"GRP_TXT"}`, nil) + + keys := map[string]string{"#parity": key} + b := newTestBackfiller(s, keys, nil) + if err := b.runKind(context.Background(), keyBackfillKindChannel); err != nil { + t.Fatal(err) + } + decoded, err := DecodePacket(raw, keys, false) + if err != nil { + t.Fatal(err) + } + pd := BuildPacketData(&MQTTPacketMessage{Raw: raw}, decoded, "obs", "", nil) + if got := colOf(t, s, "decoded_json", id); got != pd.DecodedJSON { + t.Fatalf("backfill decoded_json differs from ingest:\n got %s\nwant %s", got, pd.DecodedJSON) + } + if got := colOf(t, s, "channel_hash", id); got != pd.ChannelHash { + t.Fatalf("backfill channel_hash %q, ingest %q", got, pd.ChannelHash) + } +} + +// --- regions --- + +func TestKeyBackfillRegionNamesUniqueAndAbstainsOnAmbiguity(t *testing.T) { + s := newTestStore(t) + payload := []byte("transport-scoped payload") + + // Find two names whose codes collide on this exact payload: the + // ambiguous case #1609 says must stay unnamed. + seen := map[string]string{} + var a, c string + for i := 0; a == ""; i++ { + n := fmt.Sprintf("#r%d", i) + code := hex.EncodeToString(regionCodeBytes(regionKey(n), 5, payload)) + if prev, ok := seen[code]; ok { + a, c = prev, n + } + seen[code] = n + } + + uniqueRow := insertTx(t, s, transportFrame(regionKey("#saar"), 5, []byte("only saar")), 5, "", "{}", strp("")) + ambiguous := insertTx(t, s, transportFrame(regionKey(a), 5, payload), 5, "", "{}", strp("")) + named := insertTx(t, s, transportFrame(regionKey("#saar"), 5, []byte("named already")), 5, "", "{}", strp("#elsewhere")) + notTransport := insertTx(t, s, "1500"+hex.EncodeToString([]byte("plain flood")), 5, "", "{}", nil) + + b := newTestBackfiller(s, nil, regionSetOf("#saar", a, c)) + if err := b.runKind(context.Background(), keyBackfillKindRegion); err != nil { + t.Fatal(err) + } + if got := colOf(t, s, "scope_name", uniqueRow); got != "#saar" { + t.Fatalf("unique match = %q, want #saar", got) + } + if got := colOf(t, s, "scope_name", ambiguous); got != "" { + t.Fatalf("ambiguous collision between %s and %s was named %q; must stay unmatched", a, c, got) + } + if got := colOf(t, s, "scope_name", named); got != "#elsewhere" { + t.Fatalf("already-named row rewritten to %q", got) + } + if got := colOf(t, s, "scope_name", notTransport); got != "" { + t.Fatalf("non-transport row got scope %q", got) + } + if f := feedRows(t, s); len(f) != 1 || f[0] != uniqueRow { + t.Fatalf("feed = %v, want [%d]", f, uniqueRow) + } +} + +// The backfill must not pollute the ingest-path tally that the tier-3 spec is +// gated on: those counters describe live traffic. +func TestKeyBackfillDoesNotTouchScopeMatchCounters(t *testing.T) { + s := newTestStore(t) + insertTx(t, s, transportFrame(regionKey("#saar"), 5, []byte("x")), 5, "", "{}", strp("")) + before := scopeMatchCounters.unique.Load() + scopeMatchCounters.none.Load() + b := newTestBackfiller(s, nil, regionSetOf("#saar")) + if err := b.runKind(context.Background(), keyBackfillKindRegion); err != nil { + t.Fatal(err) + } + if after := scopeMatchCounters.unique.Load() + scopeMatchCounters.none.Load(); after != before { + t.Fatalf("scope match counters moved by %d", after-before) + } +} + +func TestKeyBackfillAdvertFillsOnlyEmptyDefaultScope(t *testing.T) { + s := newTestStore(t) + pkEmpty := strings.Repeat("aa", 32) + pkKnown := strings.Repeat("bb", 32) + for _, q := range []string{ + `INSERT INTO nodes(public_key, name) VALUES('` + pkEmpty + `','empty')`, + `INSERT INTO nodes(public_key, name, default_scope) VALUES('` + pkKnown + `','known','#newer')`, + } { + if _, err := s.db.Exec(q); err != nil { + t.Fatal(err) + } + } + insertTx(t, s, transportFrame(regionKey("#saar"), 4, advertPayload(0xaa, "E")), 4, "", "{}", strp("")) + insertTx(t, s, transportFrame(regionKey("#saar"), 4, advertPayload(0xbb, "K")), 4, "", "{}", strp("")) + + b := newTestBackfiller(s, nil, regionSetOf("#saar")) + if err := b.runKind(context.Background(), keyBackfillKindRegion); err != nil { + t.Fatal(err) + } + var e, k string + s.db.QueryRow(`SELECT COALESCE(default_scope,'') FROM nodes WHERE public_key=?`, pkEmpty).Scan(&e) + s.db.QueryRow(`SELECT COALESCE(default_scope,'') FROM nodes WHERE public_key=?`, pkKnown).Scan(&k) + if e != "#saar" { + t.Fatalf("empty default_scope = %q, want #saar", e) + } + if k != "#newer" { + t.Fatalf("known default_scope overwritten with %q", k) + } +} + +// --- lifecycle: idempotence, resume, key-set growth --- + +func TestKeyBackfillIsIdempotentAndRecordsDoneKeys(t *testing.T) { + s := newTestStore(t) + insertTx(t, s, transportFrame(regionKey("#saar"), 5, []byte("one")), 5, "", "{}", strp("")) + b := newTestBackfiller(s, nil, regionSetOf("#saar")) + for i := 0; i < 2; i++ { + if err := b.runKind(context.Background(), keyBackfillKindRegion); err != nil { + t.Fatal(err) + } + } + if f := feedRows(t, s); len(f) != 1 { + t.Fatalf("second pass rewrote again: feed=%v", f) + } + var done, state int + s.db.QueryRow(`SELECT COUNT(*) FROM key_backfill_done WHERE kind='region'`).Scan(&done) + s.db.QueryRow(`SELECT COUNT(*) FROM key_backfill_state`).Scan(&state) + if done != 1 || state != 0 { + t.Fatalf("done=%d state=%d, want 1 and 0", done, state) + } +} + +func TestKeyBackfillResumesAfterCancelWithoutDuplicates(t *testing.T) { + s := newTestStore(t) + var ids []int64 + for i := 0; i < 5; i++ { + ids = append(ids, insertTx(t, s, transportFrame(regionKey("#saar"), 5, []byte(fmt.Sprintf("p%d", i))), 5, "", "{}", strp(""))) + } + rs := regionSetOf("#saar") + + // One id per window and a long yield: cancel lands between commits. + b := newTestBackfiller(s, nil, rs) + b.window = 1 + b.yield = time.Hour + ctx, cancel := context.WithCancel(context.Background()) + errc := make(chan error, 1) + go func() { errc <- b.runKind(ctx, keyBackfillKindRegion) }() + deadline := time.Now().Add(10 * time.Second) + for { + var cur int64 + s.db.QueryRow(`SELECT COALESCE((SELECT cursor FROM key_backfill_state WHERE kind='region'),0)`).Scan(&cur) + if cur > 0 { + break + } + if time.Now().After(deadline) { + t.Fatal("first batch never committed") + } + time.Sleep(5 * time.Millisecond) + } + cancel() + if err := <-errc; err == nil { + t.Fatal("cancelled run reported success") + } + var cur, rewritten int64 + s.db.QueryRow(`SELECT cursor, rewritten FROM key_backfill_state WHERE kind='region'`).Scan(&cur, &rewritten) + if cur != ids[0] || rewritten != 1 { + t.Fatalf("after cancel cursor=%d rewritten=%d, want %d and 1", cur, rewritten, ids[0]) + } + + b2 := newTestBackfiller(s, nil, rs) + if err := b2.runKind(context.Background(), keyBackfillKindRegion); err != nil { + t.Fatal(err) + } + f := feedRows(t, s) + if len(f) != 5 { + t.Fatalf("feed=%v, want each of the 5 rows exactly once", f) + } + for i, id := range ids { + if f[i] != id { + t.Fatalf("feed order %v, want %v", f, ids) + } + } +} + +// A key added after a run started must still see the rows below the old +// cursor: a stale run fingerprint restarts the scan from the beginning. +func TestKeyBackfillKeySetGrowthRestartsFromBeginning(t *testing.T) { + s := newTestStore(t) + low := insertTx(t, s, transportFrame(regionKey("#new"), 5, []byte("early")), 5, "", "{}", strp("")) + high := insertTx(t, s, transportFrame(regionKey("#new"), 5, []byte("late")), 5, "", "{}", strp("")) + // A half-finished run for an older key set, already past `low`. + if _, err := s.db.Exec(`INSERT INTO key_backfill_state(kind, run_fp, cursor, upper) VALUES('region','stale-set',?,?)`, low, high); err != nil { + t.Fatal(err) + } + b := newTestBackfiller(s, nil, regionSetOf("#old", "#new")) + if err := b.runKind(context.Background(), keyBackfillKindRegion); err != nil { + t.Fatal(err) + } + if colOf(t, s, "scope_name", low) != "#new" || colOf(t, s, "scope_name", high) != "#new" { + t.Fatal("a key added mid-run did not reach every row") + } +} + +func TestKeyBackfillNewKeyLaterOnlyScansForThatKey(t *testing.T) { + s := newTestStore(t) + first := insertTx(t, s, transportFrame(regionKey("#a"), 5, []byte("a")), 5, "", "{}", strp("")) + second := insertTx(t, s, transportFrame(regionKey("#b"), 5, []byte("b")), 5, "", "{}", strp("")) + rs := regionSetOf("#a") + b := newTestBackfiller(s, nil, rs) + if err := b.runKind(context.Background(), keyBackfillKindRegion); err != nil { + t.Fatal(err) + } + if colOf(t, s, "scope_name", second) != "" { + t.Fatal("#b named before its key existed") + } + rs.cur.Store(®ionKeySnapshot{all: map[string][]byte{"#a": regionKey("#a"), "#b": regionKey("#b")}, explicit: map[string]bool{}}) + if err := b.runKind(context.Background(), keyBackfillKindRegion); err != nil { + t.Fatal(err) + } + if colOf(t, s, "scope_name", first) != "#a" || colOf(t, s, "scope_name", second) != "#b" { + t.Fatal("growth pass did not name the new key's row") + } + if f := feedRows(t, s); len(f) != 2 { + t.Fatalf("feed=%v, want one entry per row", f) + } +} + +func TestKeyBackfillRemovedKeyIsReplayedWhenItReturns(t *testing.T) { + s := newTestStore(t) + rs := regionSetOf("#a") + b := newTestBackfiller(s, nil, rs) + if err := b.runKind(context.Background(), keyBackfillKindRegion); err != nil { + t.Fatal(err) + } + // #a leaves the derived tier; a packet for it arrives meanwhile. + rs.cur.Store(®ionKeySnapshot{all: map[string][]byte{}, explicit: map[string]bool{}}) + if err := b.runKind(context.Background(), keyBackfillKindRegion); err != nil { + t.Fatal(err) + } + gap := insertTx(t, s, transportFrame(regionKey("#a"), 5, []byte("while absent")), 5, "", "{}", strp("")) + rs.cur.Store(®ionKeySnapshot{all: map[string][]byte{"#a": regionKey("#a")}, explicit: map[string]bool{}}) + if err := b.runKind(context.Background(), keyBackfillKindRegion); err != nil { + t.Fatal(err) + } + if got := colOf(t, s, "scope_name", gap); got != "#a" { + t.Fatalf("returning key did not replay the gap: %q", got) + } +} + +// A row resolved between decide and commit (live ingest, another writer) +// keeps its value and gets no feed entry. +func TestKeyBackfillCommitLeavesConcurrentlyResolvedRow(t *testing.T) { + s := newTestStore(t) + id := insertTx(t, s, transportFrame(regionKey("#saar"), 5, []byte("race")), 5, "", "{}", strp("")) + b := newTestBackfiller(s, nil, regionSetOf("#saar")) + if _, err := s.db.Exec(`INSERT INTO key_backfill_state(kind, run_fp, cursor, upper) VALUES('region','fp',0,?)`, id); err != nil { + t.Fatal(err) + } + if _, err := s.db.Exec(`UPDATE transmissions SET scope_name='#live' WHERE id=?`, id); err != nil { + t.Fatal(err) + } + n, err := b.commit(context.Background(), keyBackfillKindRegion, "fp", id, []keyRewrite{{id: id, scopeName: "#saar"}}) + if err != nil { + t.Fatal(err) + } + if n != 0 || colOf(t, s, "scope_name", id) != "#live" || len(feedRows(t, s)) != 0 { + t.Fatalf("n=%d scope=%q feed=%v; concurrent resolution must win", n, colOf(t, s, "scope_name", id), feedRows(t, s)) + } +} + +func TestChannelKeyHashByteMatchesBuiltinPublic(t *testing.T) { + // builtinChannelKeys documents Public's channel-hash byte as 0x11. + if hb, ok := channelKeyHashByte(builtinChannelKeys()["Public"]); !ok || hb != 0x11 { + t.Fatalf("Public hash byte = %#x ok=%v, want 0x11", hb, ok) + } +} + +func TestKeyBackfillPrunesOldFeedRows(t *testing.T) { + s := newTestStore(t) + id := insertTx(t, s, "1500aa", 5, "", "{}", nil) + old := time.Now().Add(-48 * time.Hour).Unix() + if _, err := s.db.Exec(`INSERT INTO tx_rewrite_feed(tx_id, created_at) VALUES(?,?),(?,?)`, id, old, id, time.Now().Unix()); err != nil { + t.Fatal(err) + } + newTestBackfiller(s, nil, nil).pass(context.Background()) + if f := feedRows(t, s); len(f) != 1 { + t.Fatalf("feed after prune = %v, want only the recent row", f) + } +} + +// --- runtime key-set changes (review on #2133) --- + +// A hashtag channel approved through user management at runtime is in the +// set ingest decodes with; history encrypted under it must be backfilled too, +// not only the hashChannels map loaded at startup. +func TestKeyBackfillApprovedChannelIsBackfilled(t *testing.T) { + s := newTestStore(t) + key := deriveHashtagChannelKey("#foo") + hb, _ := channelKeyHashByte(key) + id := insertTx(t, s, grpTxtFrame(t, key, "Dave: before approval"), 5, fmt.Sprintf("enc_%02X", hb), `{"type":"GRP_TXT"}`, nil) + + ks := newChannelKeySet(map[string]string{}, "", 0) + b := newKeyBackfiller(s, ks, regionSetOf()) + b.yield = 0 + if err := b.runKind(context.Background(), keyBackfillKindChannel); err != nil { + t.Fatal(err) + } + if got := colOf(t, s, "channel_hash", id); !strings.HasPrefix(got, "enc_") { + t.Fatalf("decrypted before #foo was approved: %q", got) + } + + // What channelKeySet.refresh stores once an admin approves #foo. + next, added := mergeApprovedKeys(map[string]string{}, []string{"#foo"}) + if added != 1 { + t.Fatalf("mergeApprovedKeys added %d, want 1", added) + } + ks.cur.Store(&next) + if err := b.runKind(context.Background(), keyBackfillKindChannel); err != nil { + t.Fatal(err) + } + if got := colOf(t, s, "channel_hash", id); got != "#foo" { + t.Fatalf("approved channel not backfilled: channel_hash = %q", got) + } + if f := feedRows(t, s); len(f) != 1 || f[0] != id { + t.Fatalf("feed = %v, want [%d]", f, id) + } +} + +// Both key sets store a new snapshot on every refresh. A new snapshot with +// the same keys must not abandon a run; one with a new key must. +func TestKeyBackfillKeySetMovedComparesContentNotPointer(t *testing.T) { + s := newTestStore(t) + + ks := newChannelKeySet(map[string]string{"#a": deriveHashtagChannelKey("#a")}, "", 0) + rs := regionSetOf("#r") + b := newKeyBackfiller(s, ks, rs) + + for _, kind := range []string{keyBackfillKindChannel, keyBackfillKindRegion} { + p := b.plan(kind, map[string]bool{}) + runFP := keySetFingerprint(p.all) + + // Same content, new pointer: what an unchanged refresh does. + switch kind { + case keyBackfillKindChannel: + same := map[string]string{"#a": deriveHashtagChannelKey("#a")} + ks.cur.Store(&same) + case keyBackfillKindRegion: + rs.cur.Store(®ionKeySnapshot{all: map[string][]byte{"#r": regionKey("#r")}, explicit: map[string]bool{}}) + } + if b.keySetMoved(kind, &p, runFP) { + t.Fatalf("%s: unchanged refresh counted as a key-set change", kind) + } + if kind == keyBackfillKindChannel && p.channelSnap != ks.current() { + t.Fatal("channel: run did not adopt the equal snapshot") + } + if kind == keyBackfillKindRegion && p.snapshot != rs.snapshot() { + t.Fatal("region: run did not adopt the equal snapshot") + } + + // A key added: the run must restart. + switch kind { + case keyBackfillKindChannel: + grown, _ := mergeApprovedKeys(map[string]string{"#a": deriveHashtagChannelKey("#a")}, []string{"#b"}) + ks.cur.Store(&grown) + case keyBackfillKindRegion: + rs.cur.Store(®ionKeySnapshot{all: map[string][]byte{"#r": regionKey("#r"), "#q": regionKey("#q")}, explicit: map[string]bool{}}) + } + if !b.keySetMoved(kind, &p, runFP) { + t.Fatalf("%s: added key not detected mid-run", kind) + } + } +} + +// The feed is pruned on its own ticker; with autoRegionKeys off a pass only +// runs at startup, so pruning must not wait for one. +func TestKeyBackfillPrunesFeedWithoutPass(t *testing.T) { + s := newTestStore(t) + b := newKeyBackfiller(s, newChannelKeySet(nil, "", 0), regionSetOf()) + b.yield = 0 + b.pruneEvery = 20 * time.Millisecond + ctx, cancel := context.WithCancel(context.Background()) + done := b.Start(ctx) + defer func() { cancel(); <-done }() + + // Let the startup pass finish, then add a stale row no pass will see. + time.Sleep(50 * time.Millisecond) + id := insertTx(t, s, "1500aa", 5, "", "{}", nil) + if _, err := s.db.Exec(`INSERT INTO tx_rewrite_feed(tx_id, created_at) VALUES(?,?)`, + id, time.Now().Add(-48*time.Hour).Unix()); err != nil { + t.Fatal(err) + } + deadline := time.Now().Add(2 * time.Second) + for len(feedRows(t, s)) != 0 { + if time.Now().After(deadline) { + t.Fatal("stale feed row not pruned without a pass") + } + time.Sleep(10 * time.Millisecond) + } +} + +// tx_rewrite_feed.tx_id carries ON DELETE CASCADE; without an index every +// transmission retention deletes would scan the whole feed. +func TestTxRewriteFeedTxIDIsIndexed(t *testing.T) { + s := newTestStore(t) + rows, err := s.db.Query(`EXPLAIN QUERY PLAN SELECT id FROM tx_rewrite_feed WHERE tx_id = ?`, 1) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + var plan []string + for rows.Next() { + var id, parent, notused int + var detail string + if err := rows.Scan(&id, &parent, ¬used, &detail); err != nil { + t.Fatal(err) + } + plan = append(plan, detail) + } + if !strings.Contains(strings.Join(plan, " | "), "idx_tx_rewrite_feed_tx") { + t.Fatalf("lookup by tx_id does not use idx_tx_rewrite_feed_tx: %v", plan) + } +} diff --git a/cmd/ingestor/main.go b/cmd/ingestor/main.go index 33c9c1d0b..90edcd9d8 100644 --- a/cmd/ingestor/main.go +++ b/cmd/ingestor/main.go @@ -107,13 +107,6 @@ func main() { if cfg.ApprovedChannelsEnabled() { logApprovedChannelsSource(cfg.UsersDBPath()) keySet.refresh() - go func() { - t := time.NewTicker(approvedChannelsRefresh) - defer t.Stop() - for range t.C { - keySet.refresh() - } - }() } // Linked-only ingest (clientRxCoverage.requireLinkedCompanion): nil when @@ -134,6 +127,21 @@ func main() { store.BackfillDefaultScopeAsync(regionSet) store.BackfillTransportCodesAsync() + // #2107: re-decides unresolved GRP_TXT and scope rows whenever either key + // set grows. Built here so the approved-channels refresh can kick it; + // started below, once the write path is ready. + keyBackfill := newKeyBackfiller(store, keySet, regionSet) + if cfg.ApprovedChannelsEnabled() { + go func() { + t := time.NewTicker(approvedChannelsRefresh) + defer t.Stop() + for range t.C { + keySet.refresh() + keyBackfill.Kick() // two small SELECTs unless an approval brought a new key + } + }() + } + // Subscribe-early + buffer (#1608): the MQTT subscription is brought up // before startup maintenance so no packets are missed while the single // SQLite writer is blocked (e.g. a large CREATE INDEX migration). Received @@ -370,6 +378,12 @@ func main() { if err := store.RunAsyncMigration(evidenceCtx, "advert_route_evidence_v1", store.backfillAdvertEvidence); err != nil { log.Printf("[migration] scheduling advert evidence backfill: %v", err) } + // #2107: re-decide unresolved GRP_TXT and scope rows whenever the key set + // grows. Starts after the write path is ready, like the evidence backfill + // above; it yields the writer between bounded batches. + keyBackfillCtx, stopKeyBackfill := context.WithCancel(context.Background()) + defer stopKeyBackfill() + keyBackfillDone := keyBackfill.Start(keyBackfillCtx) if d := ingestBuffer.Dropped(); d > 0 { log.Printf("[ingest-buffer] write path ready; draining backlog (dropped %d during startup — consider raising ingestBufferSize)", d) } else { @@ -489,6 +503,7 @@ func main() { for range regionRefreshTicker.C { regionSet.refreshFromStore(store) logScopeMatchCounters() + keyBackfill.Kick() // no-op unless the refresh brought a new key } }() log.Printf("[regions] auto-derived region keys enabled: refreshing every %v, cap %d", interval, cfg.AutoRegionKeysMaxDerived()) @@ -653,6 +668,12 @@ func main() { log.Println("Shutting down...") stopEvidenceBackfill() + stopKeyBackfill() + select { // let an in-flight batch commit or roll back cleanly + case <-keyBackfillDone: + case <-time.After(5 * time.Second): + log.Printf("[key-backfill] still running at shutdown; its last committed batch is where it resumes") + } retentionTicker.Stop() metricsRetentionTicker.Stop() if packetRetentionTicker != nil { diff --git a/cmd/server/chunked_load.go b/cmd/server/chunked_load.go index 772e4fcca..864c995fb 100644 --- a/cmd/server/chunked_load.go +++ b/cmd/server/chunked_load.go @@ -534,31 +534,34 @@ func (s *PacketStore) scanAndMergeChunk(rows *sql.Rows, relayPM *prefixMap, cold seenTxIDs := make(map[int]bool) var maxID int64 + // Scan overwrites every target on each row, so one set (and one + // argument slice) serves the whole loop instead of escaping per row. + var txID int + var rawHex, hash, firstSeen, decodedJSON sql.NullString + var routeType, payloadType, payloadVersion sql.NullInt64 + var obsID sql.NullInt64 + var observerID, observerName, observerIATA, direction, pathJSON, obsTimestamp sql.NullString + var snr, rssi sql.NullFloat64 + var score sql.NullInt64 + var obsRawHex sql.NullString + var resolvedPathStr sql.NullString + var scopeName sql.NullString + + scanArgs := []interface{}{&txID, &rawHex, &hash, &firstSeen, &routeType, &payloadType, + &payloadVersion, &decodedJSON, + &obsID, &observerID, &observerName, &observerIATA, &direction, + &snr, &rssi, &score, &pathJSON, &obsTimestamp} + if s.db.hasObsRawHex { + scanArgs = append(scanArgs, &obsRawHex) + } + if s.db.hasResolvedPath { + scanArgs = append(scanArgs, &resolvedPathStr) + } + if s.db.hasScopeName { + scanArgs = append(scanArgs, &scopeName) + } + for rows.Next() { - var txID int - var rawHex, hash, firstSeen, decodedJSON sql.NullString - var routeType, payloadType, payloadVersion sql.NullInt64 - var obsID sql.NullInt64 - var observerID, observerName, observerIATA, direction, pathJSON, obsTimestamp sql.NullString - var snr, rssi sql.NullFloat64 - var score sql.NullInt64 - var obsRawHex sql.NullString - var resolvedPathStr sql.NullString - var scopeName sql.NullString - - scanArgs := []interface{}{&txID, &rawHex, &hash, &firstSeen, &routeType, &payloadType, - &payloadVersion, &decodedJSON, - &obsID, &observerID, &observerName, &observerIATA, &direction, - &snr, &rssi, &score, &pathJSON, &obsTimestamp} - if s.db.hasObsRawHex { - scanArgs = append(scanArgs, &obsRawHex) - } - if s.db.hasResolvedPath { - scanArgs = append(scanArgs, &resolvedPathStr) - } - if s.db.hasScopeName { - scanArgs = append(scanArgs, &scopeName) - } if err := rows.Scan(scanArgs...); err != nil { log.Printf("[store] LoadChunked scan error: %v", err) continue @@ -610,10 +613,10 @@ func (s *PacketStore) scanAndMergeChunk(rows *sql.Rows, relayPM *prefixMap, cold obs := &StoreObs{ ID: oid, TransmissionID: txID, - ObserverID: obsIDStr, - ObserverName: nullStrVal(observerName), - ObserverIATA: nullStrVal(observerIATA), - Direction: nullStrVal(direction), + ObserverID: internStr(obsIDStr), + ObserverName: internStr(nullStrVal(observerName)), + ObserverIATA: internStr(nullStrVal(observerIATA)), + Direction: internStr(nullStrVal(direction)), SNR: nullFloatPtr(snr), RSSI: nullFloatPtr(rssi), Score: nullIntPtr(score), @@ -626,8 +629,7 @@ func (s *PacketStore) scanAndMergeChunk(rows *sql.Rows, relayPM *prefixMap, cold rpStr := nullStrVal(resolvedPathStr) if rpStr != "" { - rp := unmarshalResolvedPath(rpStr) - pks := extractResolvedPubkeys(rp) + pks := resolvedPubkeysFromJSON(rpStr) s.indexResolvedPathHops(tx, pks, hopsSeen) } else if relayPM != nil && obsPJ != "" && obsPJ != "[]" { // resolved_path is NULL on live (since #1287 relay data is diff --git a/cmd/server/config.go b/cmd/server/config.go index 5c4de1163..acc3315bc 100644 --- a/cmd/server/config.go +++ b/cmd/server/config.go @@ -209,6 +209,10 @@ type Config struct { // BatteryThresholds: voltage cutoffs for low/critical alerts (#663). BatteryThresholds *BatteryThresholdsConfig `json:"batteryThresholds,omitempty"` + // DutyCycleWarnPct: rolling 1 h TX airtime share (percent) at which the + // RF Health view and observer list warn. 0/missing => 10 (EU g3, #661). + DutyCycleWarnPct float64 `json:"dutyCycleWarnPct,omitempty"` + // Customizer controls operator-side knobs for the in-app customizer modal // (theme/branding/etc.). See CustomizerConfig and issue #1508. Customizer *CustomizerConfig `json:"customizer,omitempty"` @@ -1052,6 +1056,26 @@ func SaveGeoFilter(configDir string, gf *GeoFilterConfig) error { return fmt.Errorf("config.json not found in %s", configDir) } + // Follow a symlink at configPath to the file it points at before the + // tmp+rename below. In the Docker image /app/config.json is a symlink to + // the bind-mounted /app/data/config.json (docker/entrypoint-go.sh), and + // os.Rename onto the link path would replace the link with a regular + // file inside the container: the mounted file would keep the old + // geo_filter, the saved one would be invisible on the host and would go + // away with the container. Resolving first keeps the write atomic and + // the result where the operator (and the next container) will read it. + // + // Unlike the stats file (writeStatsAtomic, refs #1170), replacing a + // symlink at the destination is not wanted here: this path is not + // world-writable, and the link is one the image itself creates. Only a + // regular file is followed, so a link to a device or a fifo is left + // alone and handled as before. + if resolved, err := filepath.EvalSymlinks(configPath); err == nil && resolved != configPath { + if fi, err := os.Lstat(resolved); err == nil && fi.Mode().IsRegular() { + configPath = resolved + } + } + data, err := os.ReadFile(configPath) if err != nil { return fmt.Errorf("read config: %w", err) @@ -1083,13 +1107,22 @@ func SaveGeoFilter(configDir string, gf *GeoFilterConfig) error { // config.json holds the API key and broker passwords, and an operator // who made it 0600 should not find it 0644 after a geo-filter save. mode := os.FileMode(0644) - if fi, err := os.Stat(configPath); err == nil { - mode = fi.Mode().Perm() + orig, origErr := os.Stat(configPath) + if origErr == nil { + mode = orig.Mode().Perm() } tmp := configPath + ".tmp" if err := os.WriteFile(tmp, out, mode); err != nil { return fmt.Errorf("write config: %w", err) } + // Keep the owner as well. The image runs the server as root by default, + // and without this the rename would hand the operator's bind-mounted + // config.json to root:root, which at 0600 they could no longer edit + // without sudo. Best effort: a server that may not chown leaves the file + // as it was created. + if origErr == nil { + copyOwner(tmp, orig) + } if err := os.Rename(tmp, configPath); err != nil { os.Remove(tmp) return fmt.Errorf("rename config: %w", err) diff --git a/cmd/server/config_owner_unix.go b/cmd/server/config_owner_unix.go new file mode 100644 index 000000000..46eba93f8 --- /dev/null +++ b/cmd/server/config_owner_unix.go @@ -0,0 +1,17 @@ +//go:build !windows + +package main + +import ( + "os" + "syscall" +) + +// copyOwner gives path the owner and group recorded in fi, best effort: a +// process that may not chown (not root, or not in the group) leaves path as +// it was created. On Windows there is no uid/gid; see config_owner_windows.go. +func copyOwner(path string, fi os.FileInfo) { + if st, ok := fi.Sys().(*syscall.Stat_t); ok { + _ = os.Chown(path, int(st.Uid), int(st.Gid)) + } +} diff --git a/cmd/server/config_owner_unix_test.go b/cmd/server/config_owner_unix_test.go new file mode 100644 index 000000000..4d610a2e5 --- /dev/null +++ b/cmd/server/config_owner_unix_test.go @@ -0,0 +1,99 @@ +//go:build !windows + +package main + +import ( + "os" + "path/filepath" + "syscall" + "testing" +) + +// TestSaveGeoFilterKeepsOwner checks that a save keeps the owner and group of +// config.json. The image runs the server as root, and a bind-mounted config +// that came back as root:root 0600 would lock the operator out of their own +// file. +func TestSaveGeoFilterKeepsOwner(t *testing.T) { + // Pick an owner the save would not produce by itself. Root can use any + // uid and gid; anyone else can only switch to a supplementary group. + uid, gid := -1, -1 + if os.Geteuid() == 0 { + uid, gid = 4242, 4242 + } else if groups, err := os.Getgroups(); err == nil { + for _, g := range groups { + if g != os.Getegid() { + gid = g + break + } + } + } + if gid == -1 { + t.Skip("not root and no supplementary group to chown to") + } + + dir := t.TempDir() + cfg := filepath.Join(dir, "config.json") + if err := os.WriteFile(cfg, []byte(`{"port":3000}`), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Chown(cfg, uid, gid); err != nil { + t.Fatalf("chown %s: %v", cfg, err) + } + + gf := &GeoFilterConfig{Polygon: [][2]float64{{51.0, 4.0}, {51.0, 5.0}, {50.5, 4.0}}} + if err := SaveGeoFilter(dir, gf); err != nil { + t.Fatalf("SaveGeoFilter: %v", err) + } + + fi, err := os.Stat(cfg) + if err != nil { + t.Fatal(err) + } + st := fi.Sys().(*syscall.Stat_t) + if uid != -1 && int(st.Uid) != uid { + t.Errorf("owner of %s = %d, want %d", cfg, st.Uid, uid) + } + if int(st.Gid) != gid { + t.Errorf("group of %s = %d, want %d", cfg, st.Gid, gid) + } + if perm := fi.Mode().Perm(); perm != 0o600 { + t.Errorf("mode of %s = %#o, want 0600", cfg, perm) + } +} + +// TestSaveGeoFilterFifoSymlinkTarget pins the regular-file check on the +// symlink target: a config.json that links to a fifo is not followed, so the +// save never replaces the fifo with a regular file. A directory target would +// not test this, since reading the config already fails there. +func TestSaveGeoFilterFifoSymlinkTarget(t *testing.T) { + dir := t.TempDir() + fifo := filepath.Join(dir, "feed") + if err := syscall.Mkfifo(fifo, 0o600); err != nil { + t.Skipf("mkfifo unavailable: %v", err) + } + link := filepath.Join(dir, "config.json") + if err := os.Symlink(fifo, link); err != nil { + t.Skipf("symlinks unavailable: %v", err) + } + + // SaveGeoFilter is the fifo's reader; feed it one config. + go func() { + f, err := os.OpenFile(fifo, os.O_WRONLY, 0) + if err != nil { + return + } + f.WriteString(`{"port":3000}`) + f.Close() + }() + if err := SaveGeoFilter(dir, nil); err != nil { + t.Fatalf("SaveGeoFilter: %v", err) + } + + fi, err := os.Lstat(fifo) + if err != nil { + t.Fatalf("lstat %s: %v", fifo, err) + } + if fi.Mode()&os.ModeNamedPipe == 0 { + t.Errorf("%s was replaced (mode %s) — the link to a fifo was followed", fifo, fi.Mode()) + } +} diff --git a/cmd/server/config_owner_windows.go b/cmd/server/config_owner_windows.go new file mode 100644 index 000000000..ffcec7317 --- /dev/null +++ b/cmd/server/config_owner_windows.go @@ -0,0 +1,9 @@ +//go:build windows + +package main + +import "os" + +// copyOwner is a no-op on Windows: there is no uid/gid to copy, and the new +// file takes the ACL of its directory. +func copyOwner(path string, fi os.FileInfo) {} diff --git a/cmd/server/config_symlink_test.go b/cmd/server/config_symlink_test.go new file mode 100644 index 000000000..240c34293 --- /dev/null +++ b/cmd/server/config_symlink_test.go @@ -0,0 +1,81 @@ +package main + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + "testing" +) + +// TestSaveGeoFilterSymlinkedConfig pins the behavior the Docker image depends +// on: /app/config.json is a symlink to the bind-mounted /app/data/config.json, +// so a geo-filter save has to land in the mounted file and leave the symlink a +// symlink. A tmp+rename on the link path instead replaces the link with a +// regular file in the container layer, and the operator's config silently +// stops being the one in use. +func TestSaveGeoFilterSymlinkedConfig(t *testing.T) { + dir := t.TempDir() + dataDir := filepath.Join(dir, "data") + if err := os.Mkdir(dataDir, 0o755); err != nil { + t.Fatal(err) + } + target := filepath.Join(dataDir, "config.json") + if err := os.WriteFile(target, []byte(`{"port":3000,"_comment":"keep me"}`), 0o600); err != nil { + t.Fatal(err) + } + link := filepath.Join(dir, "config.json") + if err := os.Symlink(target, link); err != nil { + t.Skipf("symlinks unavailable: %v", err) + } + + gf := &GeoFilterConfig{ + Polygon: [][2]float64{{51.0, 4.0}, {51.0, 5.0}, {50.5, 4.0}}, + BufferKm: 20, + } + if err := SaveGeoFilter(dir, gf); err != nil { + t.Fatalf("SaveGeoFilter: %v", err) + } + + // The symlink must survive, or the next container start reads a different + // file than the one that was written. + fi, err := os.Lstat(link) + if err != nil { + t.Fatalf("lstat %s: %v", link, err) + } + if fi.Mode()&os.ModeSymlink == 0 { + t.Errorf("%s is no longer a symlink (mode %s) — the rename replaced it", link, fi.Mode()) + } + + // The geo filter must be in the mounted file, next to what was there. + data, err := os.ReadFile(target) + if err != nil { + t.Fatalf("read %s: %v", target, err) + } + var raw map[string]interface{} + if err := json.Unmarshal(data, &raw); err != nil { + t.Fatalf("parse saved config: %v", err) + } + if _, ok := raw["geo_filter"]; !ok { + t.Errorf("geo_filter missing from %s: %s", target, data) + } + if raw["_comment"] != "keep me" { + t.Errorf("unrelated keys not preserved: %s", data) + } + if !bytes.Contains(data, []byte(`"bufferKm"`)) { + t.Errorf("bufferKm missing from saved config: %s", data) + } + + // The 0600 the operator chose is kept (#2126), and no tmp file is left in + // either directory. + if fi, err := os.Stat(target); err != nil { + t.Fatal(err) + } else if perm := fi.Mode().Perm(); perm != 0o600 { + t.Errorf("mode of %s = %#o, want 0600", target, perm) + } + for _, leftover := range []string{link + ".tmp", target + ".tmp"} { + if _, err := os.Lstat(leftover); err == nil { + t.Errorf("leftover temp file %s", leftover) + } + } +} diff --git a/cmd/server/db.go b/cmd/server/db.go index 79d1e7d2b..68ff61550 100644 --- a/cmd/server/db.go +++ b/cmd/server/db.go @@ -39,7 +39,8 @@ const routeTypeNonTransportSQL = "route_type IN (1, 2)" // DB wraps a read-only connection to the MeshCore SQLite database. type DB struct { advertEvidenceTable atomic.Bool - advertEvidenceReadHook func() // test-only: immediately before a bulk mask query + txRewriteFeedTable atomic.Bool // tx_rewrite_feed present (#2107); re-probed while absent + advertEvidenceReadHook func() // test-only: immediately before a bulk mask query conn *sql.DB path string // filesystem path to the database file isV3 bool // v3 schema: observer_idx in observations (vs observer_id in v2) @@ -3037,12 +3038,16 @@ type MetricsSummaryRow struct { CurrentBattMv *int `json:"battery_mv"` SampleCount int `json:"sample_count"` Sparkline []*float64 `json:"sparkline"` + // Airtime share over the hour before now, whatever the window (#661). + TxAirtimePct1h *float64 `json:"tx_airtime_pct_1h"` + RxAirtimePct1h *float64 `json:"rx_airtime_pct_1h"` } // GetMetricsSummary returns a fleet summary of observer metrics within a time window. // Uses a CTE with ROW_NUMBER to get latest values in a single pass (no correlated subqueries). -// Also returns sparkline data (noise_floor time series) per observer. -func (db *DB) GetMetricsSummary(since string) ([]MetricsSummaryRow, error) { +// Also returns sparkline data (noise_floor time series) per observer, and the +// TX/RX airtime share over the hour before now. +func (db *DB) GetMetricsSummary(since string, now time.Time) ([]MetricsSummaryRow, error) { query := ` WITH ranked AS ( SELECT observer_id, noise_floor, battery_mv, @@ -3109,11 +3114,51 @@ func (db *DB) GetMetricsSummary(since string) ([]MetricsSummaryRow, error) { result[i].Sparkline = s } } + + if err := db.fillAirtime1h(result, now); err != nil { + return nil, err + } } return result, nil } +// fillAirtime1h sets the rolling 1 h TX/RX airtime share on each row from +// the cumulative counters sampled in the hour before now, in one query. +func (db *DB) fillAirtime1h(rows []MetricsSummaryRow, now time.Time) error { + since := now.Add(-airtimeWindow).Format(time.RFC3339) + q, err := db.conn.Query(`SELECT observer_id, timestamp, tx_air_secs, rx_air_secs + FROM observer_metrics WHERE timestamp >= ? ORDER BY observer_id, timestamp ASC`, since) + if err != nil { + return err + } + defer q.Close() + + tx := make(map[string][]airtimePoint) + rx := make(map[string][]airtimePoint) + for q.Next() { + var oid, ts string + var txSecs, rxSecs *int + if err := q.Scan(&oid, &ts, &txSecs, &rxSecs); err != nil { + return err + } + at, err := time.Parse(time.RFC3339, ts) + if err != nil { + continue + } + tx[oid] = append(tx[oid], airtimePoint{At: at, Secs: txSecs}) + rx[oid] = append(rx[oid], airtimePoint{At: at, Secs: rxSecs}) + } + if err := q.Err(); err != nil { + return err + } + for i := range rows { + rows[i].TxAirtimePct1h = rollingAirtimePct(tx[rows[i].ObserverID], now) + rows[i].RxAirtimePct1h = rollingAirtimePct(rx[rows[i].ObserverID], now) + } + return nil +} + // (PruneOldMetrics / RemoveStaleObservers removed in #1283 — see note // above the MetricsSample type. Ingestor owns these writes now.) diff --git a/cmd/server/db_test.go b/cmd/server/db_test.go index 977827640..1e788e740 100644 --- a/cmd/server/db_test.go +++ b/cmd/server/db_test.go @@ -1918,7 +1918,7 @@ func TestGetMetricsSummary(t *testing.T) { "obs2", t1, -115.0) since := now.Add(-24 * time.Hour).Format(time.RFC3339) - summary, err := db.GetMetricsSummary(since) + summary, err := db.GetMetricsSummary(since, now) if err != nil { t.Fatal(err) } diff --git a/cmd/server/decoded_cache_keys_test.go b/cmd/server/decoded_cache_keys_test.go new file mode 100644 index 000000000..3e988eea9 --- /dev/null +++ b/cmd/server/decoded_cache_keys_test.go @@ -0,0 +1,89 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" +) + +const decodedCacheFixture = `{"type":"ADVERT","pubKey":"ab12","name":"Rpt","lat":55.9,"lon":-3.2,` + + `"flags":{"hasLocation":true,"repeater":true},"timestamp":1760000000,"timestampISO":"2025-10-09T08:53:20Z",` + + `"signature":"` + "00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff" + `",` + + `"signatureValid":true,"encryptedData":"deadbeef","mac":"cafe","pathData":"0102","ctrlPubKey":"ff","ctrlSubtype":1,` + + `"ctrlTag":2,"ctrlFlags":"x","ctrlZeroHop":true,"ctrlLength":3}` + +// The decoded-payload cache is filled for every stored packet by the +// analytics passes and kept for the packet's lifetime: about 40MB on a busy +// week. A third of the JSON it holds is in fields no server code reads +// (ciphertext, MACs, signatures, ISO copies of timestamps, control-packet +// details), so the cache drops them. +func TestParsedDecodedDropsUnreadKeys(t *testing.T) { + tx := &StoreTx{DecodedJSON: decodedCacheFixture} + d := tx.ParsedDecoded() + for _, k := range decodedCacheDroppedKeys { + if _, ok := d[k]; ok { + t.Errorf("the cache keeps %q, which no server code reads", k) + } + } + for _, k := range []string{"type", "pubKey", "name", "lat", "lon", "flags", "timestamp"} { + if _, ok := d[k]; !ok { + t.Errorf("the cache lost %q", k) + } + } +} + +// The live feed still sends the whole payload: the stored JSON goes out as +// it is, without decoding it again under the store lock or copying the +// cache, so the frontend sees every field. +func TestBroadcastPayloadKeepsEveryKey(t *testing.T) { + tx := &StoreTx{DecodedJSON: decodedCacheFixture} + tx.ParsedDecoded() + raw := broadcastPayload(tx) + var p map[string]interface{} + if err := json.Unmarshal(raw, &p); err != nil { + t.Fatalf("broadcast payload is not JSON: %v", err) + } + for _, k := range append([]string{"type", "pubKey", "flags"}, decodedCacheDroppedKeys...) { + if _, ok := p[k]; !ok { + t.Errorf("broadcast payload is missing %q", k) + } + } + if broadcastPayload(&StoreTx{DecodedJSON: "{not json"}) != nil { + t.Error("an unreadable payload must be left out of the broadcast") + } +} + +// Guard: if server code starts reading one of the dropped keys, it must come +// off the list. Any string literal naming one outside a struct tag fails. +func TestDecodedCacheDroppedKeysAreUnread(t *testing.T) { + files, _ := filepath.Glob("*.go") + for _, f := range files { + if strings.HasSuffix(f, "_test.go") { + continue + } + b, err := os.ReadFile(f) + if err != nil { + t.Fatal(err) + } + inList := false + for i, line := range strings.Split(string(b), "\n") { + if strings.HasPrefix(line, "var decodedCacheDroppedKeys") { + inList = true + } + if inList { + inList = !strings.HasPrefix(line, "}") + continue + } + if strings.Contains(line, "`json:\"") { + continue + } + for _, k := range decodedCacheDroppedKeys { + if strings.Contains(line, `"`+k+`"`) { + t.Errorf("%s:%d names %q, which ParsedDecoded drops: take it off decodedCacheDroppedKeys", f, i+1, k) + } + } + } + } +} diff --git a/cmd/server/eviction_resolved_pubkeys_test.go b/cmd/server/eviction_resolved_pubkeys_test.go new file mode 100644 index 000000000..078ca351f --- /dev/null +++ b/cmd/server/eviction_resolved_pubkeys_test.go @@ -0,0 +1,54 @@ +package main + +import ( + "fmt" + "runtime" + "testing" +) + +func evictionResolvedStore(t testing.TB, n int) (*PacketStore, []int) { + t.Helper() + db := setupTestDB(t) + t.Cleanup(func() { db.Close() }) + pk := func(i int) string { return fmt.Sprintf("%064x", i) } + ids := make([]int, 0, n) + for i := 1; i <= n; i++ { + if _, err := db.conn.Exec(`INSERT INTO transmissions (id, raw_hex, hash, first_seen) VALUES (?, '00', ?, '2026-01-01T00:00:00Z')`, i, fmt.Sprintf("h%015d", i)); err != nil { + t.Fatal(err) + } + for o := 0; o < 3; o++ { + rp := `["` + pk(i%9) + `",null,"` + pk(o+20) + `","` + pk(i%9) + `"]` + if _, err := db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp, resolved_path) VALUES (?, 1, '["aa","bb","cc","dd"]', 0, ?)`, i, rp); err != nil { + t.Fatal(err) + } + } + ids = append(ids, i) + } + return &PacketStore{db: db}, ids +} + +// Eviction looks up the resolved pubkeys of every packet it drops, once a +// minute in steady state. It decoded each resolved_path into a pointer per +// hop and built a de-dup map per packet: about 130MB of allocations per +// five minutes on a busy week. +func TestResolvedPubkeysForEvictionBatch(t *testing.T) { + s, ids := evictionResolvedStore(t, 300) + got := s.resolvedPubkeysForEvictionBatch(ids) + want := []string{fmt.Sprintf("%064x", 1), fmt.Sprintf("%064x", 20), fmt.Sprintf("%064x", 21), fmt.Sprintf("%064x", 22)} + if fmt.Sprint(got[1]) != fmt.Sprint(want) { + t.Fatalf("tx 1: %v, want %v (first-seen order, de-duplicated)", got[1], want) + } + if len(got) != 300 { + t.Fatalf("%d packets resolved, want 300", len(got)) + } + var a, b runtime.MemStats + runtime.GC() + runtime.ReadMemStats(&a) + _ = s.resolvedPubkeysForEvictionBatch(ids) + runtime.ReadMemStats(&b) + per := float64(b.Mallocs-a.Mallocs) / float64(len(ids)) + t.Logf("%.1f allocations per evicted packet (3 observations, 4 hops each)", per) + if per > 20 { + t.Fatalf("%.1f allocations per evicted packet, limit 20", per) + } +} diff --git a/cmd/server/intern.go b/cmd/server/intern.go new file mode 100644 index 000000000..1e97c492d --- /dev/null +++ b/cmd/server/intern.go @@ -0,0 +1,50 @@ +package main + +import ( + "strings" + "sync" +) + +// internMax bounds the intern table. Observer IDs, names, IATA codes and +// directions number in the hundreds on a busy instance, so the cap is only a +// guard against an unexpected flood of distinct values. +const internMax = 1 << 16 + +var ( + internMu sync.RWMutex + internTable = make(map[string]string) +) + +// internStr returns the canonical copy of s, so a store of hundreds of +// thousands of observations keeps one copy of each string that repeats +// across them (observer IDs, names, IATA codes, directions) rather than the +// one each row scan allocates. +// +// The table holds its copies for the life of the process: they are few, and +// a strongly held copy is what makes the sharing survive garbage collections +// (unique.Make holds its copies weakly, so a value interned again after a GC +// came back as a new copy). Past internMax entries a string is returned as +// it came: nothing is shared, nothing breaks. A hit takes only the read lock +// and does not allocate. +func internStr(s string) string { + if s == "" { + return "" + } + internMu.RLock() + c, ok := internTable[s] + internMu.RUnlock() + if ok { + return c + } + internMu.Lock() + defer internMu.Unlock() + if c, ok := internTable[s]; ok { + return c + } + if len(internTable) >= internMax { + return s + } + c = strings.Clone(s) // do not pin a larger buffer s may be a slice of + internTable[c] = c + return c +} diff --git a/cmd/server/neighbor_persist.go b/cmd/server/neighbor_persist.go index d73b972cf..21eca9aad 100644 --- a/cmd/server/neighbor_persist.go +++ b/cmd/server/neighbor_persist.go @@ -221,3 +221,102 @@ func unmarshalResolvedPath(s string) []*string { } return result } + +// resolvedPubkeysFromJSON returns the non-empty pubkeys of a resolved_path +// JSON array, exactly extractResolvedPubkeys(unmarshalResolvedPath(s)). The +// bulk loaders call it for most observations; the usual array of plain +// strings and nulls is read in place, one string per pubkey, instead of +// decoding a string and a pointer per hop. Anything else is decoded. +// +// Pubkeys go through internStr: relays repeat across nearly every +// observation, so a pubkey seen before costs no allocation. Interned keys +// live for the process, evicted nodes' keys included; the table is bounded +// (internMax), and past it a pubkey is returned uninterned. +func resolvedPubkeysFromJSON(s string) []string { + if s == "" { + return nil + } + if out, ok := scanResolvedPath(s); ok { + return out + } + return extractResolvedPubkeys(unmarshalResolvedPath(s)) +} + +// scanResolvedPath reads s as a JSON array of escape-free strings and nulls +// with JSON whitespace between tokens. ok is false for anything else. +func scanResolvedPath(s string) (out []string, ok bool) { + i := 0 + ws := func() { + for i < len(s) && (s[i] == ' ' || s[i] == '\t' || s[i] == '\n' || s[i] == '\r') { + i++ + } + } + ws() + if i >= len(s) { + return nil, false + } + if strings.HasPrefix(s[i:], "null") { + // unmarshalResolvedPath gives nil for a bare null. + i += 4 + ws() + return nil, i == len(s) + } + if s[i] != '[' { + return nil, false + } + i++ + ws() + if i < len(s) && s[i] == ']' { + i++ + ws() + return nil, i == len(s) + } + for { + switch { + case i < len(s) && s[i] == '"': + i++ + start := i + for i < len(s) && s[i] != '"' { + if s[i] == '\\' || s[i] < 0x20 || s[i] >= 0x80 { + return nil, false + } + i++ + } + if i >= len(s) { + return nil, false + } + if i > start { + if out == nil { + out = make([]string, 0, 8) + } + out = append(out, internStr(s[start:i])) + } + i++ + case strings.HasPrefix(s[i:], "null"): + i += 4 + default: + return nil, false + } + ws() + if i >= len(s) { + return nil, false + } + switch s[i] { + case ',': + i++ + ws() + case ']': + i++ + ws() + if i != len(s) { + return nil, false + } + if out == nil { + out = []string{} + } + return out, true + default: + return nil, false + } + } +} diff --git a/cmd/server/obs_parsed_time_test.go b/cmd/server/obs_parsed_time_test.go new file mode 100644 index 000000000..65aabb823 --- /dev/null +++ b/cmd/server/obs_parsed_time_test.go @@ -0,0 +1,72 @@ +package main + +import ( + "sync" + "testing" + "time" + "unsafe" +) + +// StoreObs is the store's most numerous struct (400k on a week of a busy +// network). Its lazy timestamp cache used to be a time.Time, a sync.Once and +// a bool, 40 bytes that put each observation in the 192-byte size class. It +// is 8 bytes now, and each observation fits in 160. +func TestStoreObsFitsIn160Bytes(t *testing.T) { + if n := unsafe.Sizeof(StoreObs{}); n > 160 { + t.Fatalf("StoreObs is %d bytes, want at most 160 (one size class down from 192)", n) + } +} + +func parseObsTimestampReference(s string) (time.Time, bool) { + for _, layout := range []string{time.RFC3339Nano, time.RFC3339, "2006-01-02 15:04:05"} { + if t, err := time.Parse(layout, s); err == nil { + return t, true + } + } + return time.Time{}, false +} + +// ParsedTime gives the same time, location included, as parsing the string. +func TestStoreObsParsedTimeMatchesParse(t *testing.T) { + for _, ts := range []string{ + "2026-10-10T12:34:56.789Z", + "2026-10-10T12:34:56Z", + "2026-10-10T13:34:56+01:00", + "2026-10-10 12:34:56", + "", + "not a time", + "1970-01-01T00:00:00Z", + } { + o := &StoreObs{Timestamp: ts} + want, wantOK := parseObsTimestampReference(ts) + for i := 0; i < 2; i++ { // first call parses, second reads the cache + got, ok := o.ParsedTime() + if ok != wantOK || !got.Equal(want) || got.Location().String() != want.Location().String() { + t.Fatalf("%q call %d: got %v (%v) ok=%v, want %v (%v) ok=%v", ts, i, got, got.Location(), ok, want, want.Location(), wantOK) + } + if wantOK && got.Hour() != want.Hour() { + t.Fatalf("%q: hour %d, want %d", ts, got.Hour(), want.Hour()) + } + } + } +} + +// Concurrent readers under the store's read lock may race to fill the cache. +func TestStoreObsParsedTimeConcurrentFill(t *testing.T) { + o := &StoreObs{Timestamp: "2026-10-10T12:34:56.789Z"} + want, _ := parseObsTimestampReference(o.Timestamp) + var wg sync.WaitGroup + for i := 0; i < 16; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < 100; j++ { + if got, ok := o.ParsedTime(); !ok || !got.Equal(want) { + t.Errorf("got %v ok=%v", got, ok) + return + } + } + }() + } + wg.Wait() +} diff --git a/cmd/server/observer_airtime.go b/cmd/server/observer_airtime.go new file mode 100644 index 000000000..d92b1f423 --- /dev/null +++ b/cmd/server/observer_airtime.go @@ -0,0 +1,81 @@ +package main + +import ( + "math" + "time" +) + +// Rolling 1 h airtime per observer (#661). The firmware enforces its own +// duty-cycle budget over a 1 h window (Dispatcher.cpp, duty_cycle_window_ms), +// but repeaters default to airtime_factor 1.0, i.e. a 50% budget, so the EU +// 10% limit is not enforced on the device. The summary therefore reports the +// TX share of the last hour and the threshold the UI warns at. + +// airtimeWindow is the span the rolling figure covers, matching the +// firmware's duty-cycle window. +const airtimeWindow = time.Hour + +// airtimeMinSpan is the shortest stretch of samples that yields a figure; +// shorter (one sample, or a reboot late in the hour) reads as unknown. The +// ingestor rounds sample times to its sample interval (5 min by default), so +// the span can be off by up to one interval: about 17% at 30 min, 8% at 1 h. +const airtimeMinSpan = 30 * time.Minute + +// airtimeMaxAge is how old the newest sample may be; an observer that has +// gone quiet longer than this reads as unknown rather than "last hour". +const airtimeMaxAge = 15 * time.Minute + +// defaultDutyCycleWarnPct is the EU 868 MHz g3 sub-band duty-cycle limit. +const defaultDutyCycleWarnPct = 10.0 + +// airtimePoint is one sample of a cumulative airtime counter (seconds since +// boot); Secs is nil when the observer did not report it. +type airtimePoint struct { + At time.Time + Secs *int +} + +// rollingAirtimePct returns the airtime share, in percent, over the points +// given in time order. A counter that goes down marks a reboot: only the +// stretch after the last reset counts. Returns nil when that stretch is +// shorter than airtimeMinSpan, or the newest sample is older than +// airtimeMaxAge at now. +func rollingAirtimePct(pts []airtimePoint, now time.Time) *float64 { + var start, last *airtimePoint + for i := range pts { + p := &pts[i] + if p.Secs == nil { + continue + } + if start == nil || *p.Secs < *last.Secs { + start = p + } + last = p + } + if start == nil || now.Sub(last.At) > airtimeMaxAge { + return nil + } + span := last.At.Sub(start.At) + if span < airtimeMinSpan { + return nil + } + pct := float64(*last.Secs-*start.Secs) / span.Seconds() * 100 + pct = math.Min(math.Max(pct, 0), 100) + pct = math.Round(pct*100) / 100 + return &pct +} + +// DutyCycleWarnPctOrDefault returns the configured duty-cycle warning +// threshold in percent, or the EU 10% default. +func (c *Config) DutyCycleWarnPctOrDefault() float64 { + if c.DutyCycleWarnPct > 0 { + return c.DutyCycleWarnPct + } + return defaultDutyCycleWarnPct +} + +// MetricsSummaryResponse is the body of GET /api/observers/metrics/summary. +type MetricsSummaryResponse struct { + Observers []MetricsSummaryRow `json:"observers"` + DutyCycleWarnPct float64 `json:"duty_cycle_warn_pct"` +} diff --git a/cmd/server/observer_airtime_test.go b/cmd/server/observer_airtime_test.go new file mode 100644 index 000000000..a0c151546 --- /dev/null +++ b/cmd/server/observer_airtime_test.go @@ -0,0 +1,124 @@ +package main + +import ( + "encoding/json" + "net/http/httptest" + "testing" + "time" +) + +func airPt(base time.Time, minAgo int, secs int) airtimePoint { + v := secs + return airtimePoint{At: base.Add(-time.Duration(minAgo) * time.Minute), Secs: &v} +} + +func pctOrNil(p *float64) interface{} { + if p == nil { + return nil + } + return *p +} + +func TestRollingAirtimePct(t *testing.T) { + now := time.Date(2026, 10, 14, 12, 0, 0, 0, time.UTC) + cases := []struct { + name string + pts []airtimePoint + want interface{} + }{ + {"steady 10%", []airtimePoint{airPt(now, 60, 1000), airPt(now, 30, 1180), airPt(now, 0, 1360)}, 10.0}, + {"reboot mid-hour uses the segment after the reset", + []airtimePoint{airPt(now, 60, 0), airPt(now, 40, 100), airPt(now, 30, 5), airPt(now, 0, 185)}, 10.0}, + {"single sample", []airtimePoint{airPt(now, 0, 50)}, nil}, + {"no samples", nil, nil}, + {"span below 30 minutes", []airtimePoint{airPt(now, 25, 0), airPt(now, 0, 150)}, nil}, + {"span of exactly 30 minutes", []airtimePoint{airPt(now, 30, 0), airPt(now, 0, 180)}, 10.0}, + {"reboot leaves less than 30 minutes", []airtimePoint{airPt(now, 60, 0), airPt(now, 30, 400), airPt(now, 25, 2), airPt(now, 0, 20)}, nil}, + {"newest sample older than 15 minutes", []airtimePoint{airPt(now, 60, 0), airPt(now, 20, 240)}, nil}, + {"newest sample exactly 15 minutes old", []airtimePoint{airPt(now, 60, 0), airPt(now, 15, 270)}, 10.0}, + {"nil counters are skipped", + []airtimePoint{airPt(now, 60, 0), {At: now.Add(-30 * time.Minute)}, airPt(now, 0, 36)}, 1.0}, + {"clamped to 100", []airtimePoint{airPt(now, 30, 0), airPt(now, 0, 5000)}, 100.0}, + {"rounded to 2 decimals", []airtimePoint{airPt(now, 60, 0), airPt(now, 0, 1)}, 0.03}, + } + for _, c := range cases { + got := pctOrNil(rollingAirtimePct(c.pts, now)) + if got != c.want { + t.Errorf("%s: got %v, want %v", c.name, got, c.want) + } + } +} + +func TestGetMetricsSummaryAirtime1h(t *testing.T) { + db := setupTestDB(t) + seedTestData(t, db) + now := time.Now().UTC().Truncate(time.Second) + ins := func(obs string, minAgo, tx, rx int) { + ts := now.Add(-time.Duration(minAgo) * time.Minute).Format(time.RFC3339) + if _, err := db.conn.Exec("INSERT INTO observer_metrics (observer_id, timestamp, noise_floor, tx_air_secs, rx_air_secs) VALUES (?, ?, ?, ?, ?)", + obs, ts, -110.0, tx, rx); err != nil { + t.Fatal(err) + } + } + // obs1: a sample 3 h ago with a far lower counter must not count; inside + // the hour it transmits 180 s in 30 min (10%) and receives 360 s (20%). + ins("obs1", 180, 0, 0) + ins("obs1", 30, 5000, 9000) + ins("obs1", 0, 5180, 9360) + // obs2: only one sample inside the hour, so no rolling figure. + ins("obs2", 120, 0, 0) + ins("obs2", 5, 900, 900) + // obs3: a full hour at 10%, but silent for the last 40 minutes. + ins("obs3", 60, 0, 0) + ins("obs3", 40, 120, 120) + + summary, err := db.GetMetricsSummary(now.Add(-24*time.Hour).Format(time.RFC3339), now) + if err != nil { + t.Fatal(err) + } + byID := map[string]MetricsSummaryRow{} + for _, r := range summary { + byID[r.ObserverID] = r + } + if got := pctOrNil(byID["obs1"].TxAirtimePct1h); got != 10.0 { + t.Errorf("obs1 tx 1h = %v, want 10", got) + } + if got := pctOrNil(byID["obs1"].RxAirtimePct1h); got != 20.0 { + t.Errorf("obs1 rx 1h = %v, want 20", got) + } + if got := pctOrNil(byID["obs2"].TxAirtimePct1h); got != nil { + t.Errorf("obs2 tx 1h = %v, want nil", got) + } + if _, ok := byID["obs3"]; !ok { + t.Fatal("obs3 missing from the summary") + } + if got := pctOrNil(byID["obs3"].TxAirtimePct1h); got != nil { + t.Errorf("obs3 (stale) tx 1h = %v, want nil", got) + } +} + +func TestMetricsSummaryDutyCycleWarnPct(t *testing.T) { + srv, router := setupTestServer(t) + get := func() float64 { + w := httptest.NewRecorder() + router.ServeHTTP(w, httptest.NewRequest("GET", "/api/observers/metrics/summary?window=1h", nil)) + if w.Code != 200 { + t.Fatalf("status %d: %s", w.Code, w.Body.String()) + } + var resp MetricsSummaryResponse + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } + if resp.Observers == nil { + t.Fatal("observers must be [] not null") + } + return resp.DutyCycleWarnPct + } + if got := get(); got != 10 { + t.Errorf("default duty-cycle warn = %v, want 10", got) + } + srv.cfg.DutyCycleWarnPct = 1 + if got := get(); got != 1 { + t.Errorf("configured duty-cycle warn = %v, want 1", got) + } +} diff --git a/cmd/server/observer_strings_intern_test.go b/cmd/server/observer_strings_intern_test.go new file mode 100644 index 000000000..e1bc1c24e --- /dev/null +++ b/cmd/server/observer_strings_intern_test.go @@ -0,0 +1,181 @@ +package main + +import ( + "database/sql" + "fmt" + "os" + "path/filepath" + "runtime" + "strings" + "sync" + "testing" + "time" + "unsafe" +) + +// Every observation carries its observer's ID, name, IATA code and the +// direction. A store holds hundreds of thousands of observations from a few +// hundred observers, so each loader shares one copy of each of those strings +// instead of keeping the copy every row scan makes. + +func assertObserverStringsShared(t *testing.T, store *PacketStore, wantObs int) { + t.Helper() + store.mu.RLock() + defer store.mu.RUnlock() + var all []*StoreObs + for _, tx := range store.packets { + all = append(all, tx.Observations...) + } + if len(all) < wantObs { + t.Fatalf("loaded %d observations, want at least %d", len(all), wantObs) + } + first := all[0] + for _, f := range []struct { + name string + get func(*StoreObs) string + }{ + {"ObserverID", func(o *StoreObs) string { return o.ObserverID }}, + {"ObserverName", func(o *StoreObs) string { return o.ObserverName }}, + {"ObserverIATA", func(o *StoreObs) string { return o.ObserverIATA }}, + {"Direction", func(o *StoreObs) string { return o.Direction }}, + } { + want := f.get(first) + if want == "" { + t.Fatalf("fixture: %s is empty", f.name) + } + copies := map[*byte]bool{} + for _, o := range all { + if f.get(o) == want { + copies[unsafe.StringData(f.get(o))] = true + } + } + if len(copies) != 1 { + t.Errorf("%s %q is held in %d copies across %d observations, want 1", f.name, want, len(copies), len(all)) + } + } +} + +func openInternTestStore(t *testing.T, numTx int) *PacketStore { + t.Helper() + dir := t.TempDir() + dbPath := filepath.Join(dir, "intern.db") + createTestDBAt(t, dbPath, numTx) + t.Cleanup(func() { os.RemoveAll(dir) }) + // The shared fixture leaves observers empty; give each an IATA code so + // the loaders read one for every observation. + conn, err := sql.Open("sqlite3", dbPath) + if err != nil { + t.Fatal(err) + } + if _, err := conn.Exec(`INSERT INTO observers (id, name, iata) SELECT DISTINCT observer_id, observer_name, 'EDI' FROM observations`); err != nil { + t.Fatalf("fixture observers: %v", err) + } + conn.Close() + db, err := OpenDB(dbPath) + if err != nil { + t.Fatalf("OpenDB: %v", err) + } + t.Cleanup(func() { db.Close() }) + return NewPacketStore(db, &PacketStoreConfig{}) +} + +func TestObserverStringsShared_Load(t *testing.T) { + store := openInternTestStore(t, 50) + if err := store.Load(); err != nil { + t.Fatal(err) + } + assertObserverStringsShared(t, store, 50) +} + +func TestObserverStringsShared_LoadChunked(t *testing.T) { + store := openInternTestStore(t, 50) + if err := store.LoadChunked(10); err != nil { + t.Fatal(err) + } + assertObserverStringsShared(t, store, 50) +} + +func TestObserverStringsShared_LoadChunk(t *testing.T) { + store := openInternTestStore(t, 50) + from := time.Date(2025, 12, 31, 0, 0, 0, 0, time.UTC) + if err := store.loadChunk(from, from.Add(72*time.Hour)); err != nil { + t.Fatal(err) + } + assertObserverStringsShared(t, store, 50) +} + +// Many goroutines interning the same values all get the one shared copy. +func TestInternStrConcurrent(t *testing.T) { + keys := make([]string, 50) + for i := range keys { + keys[i] = fmt.Sprintf("intern-concurrent-%062d", i) + } + var wg sync.WaitGroup + got := make([][]string, 16) + for g := range got { + wg.Add(1) + go func(g int) { + defer wg.Done() + for _, k := range keys { + got[g] = append(got[g], internStr(strings.Clone(k))) + } + }(g) + } + wg.Wait() + for i := range keys { + want := unsafe.StringData(got[0][i]) + for g := range got { + if got[g][i] != keys[i] || unsafe.StringData(got[g][i]) != want { + t.Fatalf("goroutine %d got a different copy of %q", g, keys[i]) + } + } + } +} + +// BenchmarkInternStr measures a hit (the steady state: a few hundred +// observers) alone and from every core. Loaders and ingest intern in +// parallel, so hits must not contend. +// +// go test -run '^$' -bench BenchmarkInternStr -cpu 1,12 ./cmd/server +func BenchmarkInternStr(b *testing.B) { + keys := make([]string, 300) + for i := range keys { + keys[i] = internStr(fmt.Sprintf("bench-observer-%049d", i)) + } + b.Run("serial", func(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = internStr(keys[i%len(keys)]) + } + }) + b.Run("parallel", func(b *testing.B) { + b.RunParallel(func(pb *testing.PB) { + i := 0 + for pb.Next() { + _ = internStr(keys[i%len(keys)]) + i++ + } + }) + }) +} + +// A hit, the steady state, allocates nothing. +func TestInternStrHitDoesNotAllocate(t *testing.T) { + k := internStr("hit-probe-0123456789abcdef0123456789abcdef0123456789abcdef01234567") + if n := testing.AllocsPerRun(1000, func() { _ = internStr(k) }); n != 0 { + t.Fatalf("internStr allocates %.0f times on a hit", n) + } +} + +// A canonical copy outlives a garbage collection: interning the same value +// after a GC returns the copy observations already hold, not a new one, so a +// long load or a long-running server keeps one copy per value in total. +func TestInternStrSurvivesGC(t *testing.T) { + a := internStr(strings.Repeat("x", 3) + "gc-obs") + runtime.GC() + runtime.GC() + b := internStr(strings.Repeat("x", 3) + "gc-obs") + if unsafe.StringData(a) != unsafe.StringData(b) { + t.Fatal("interning after a GC made a second copy of a value still in use") + } + runtime.KeepAlive(a) +} diff --git a/cmd/server/openapi.go b/cmd/server/openapi.go index 3a6b1e7b0..401a9d61e 100644 --- a/cmd/server/openapi.go +++ b/cmd/server/openapi.go @@ -205,7 +205,7 @@ func routeDescriptions() map[string]routeMeta { "GET /api/observers/{id}": {Summary: "Get observer detail", Tag: "observers"}, "GET /api/observers/{id}/metrics": {Summary: "Get observer metrics", Description: "Packet rates, uptime, and performance metrics.", Tag: "observers"}, "GET /api/observers/{id}/analytics": {Summary: "Get observer analytics", Tag: "observers"}, - "GET /api/observers/metrics/summary": {Summary: "Observer metrics summary", Description: "Aggregate metrics across all observers.", Tag: "observers"}, + "GET /api/observers/metrics/summary": {Summary: "Observer metrics summary", Description: "Aggregate metrics across all observers within window (noise floor, battery, sparkline). Each row also carries tx_airtime_pct_1h / rx_airtime_pct_1h: the airtime share over the last hour regardless of window, null with under 30 minutes of samples since the last reboot or no sample in the last 15 minutes. duty_cycle_warn_pct is the configured warning threshold (dutyCycleWarnPct, default 10).", Tag: "observers", QueryParams: []paramMeta{{Name: "window", Description: "Lookback such as 1h, 24h, 7d (default 24h)", Type: "string"}, {Name: "region", Description: "IATA code to filter observers", Type: "string"}}}, // Misc "GET /api/resolve-hops": {Summary: "Resolve hop path", Description: "Resolves hash prefixes in a hop path to node names. Returns affinity scores and best candidates.", Tag: "nodes", QueryParams: []paramMeta{{Name: "hops", Description: "Comma-separated hop hash prefixes", Type: "string", Required: true}}}, diff --git a/cmd/server/relay_candidates_alloc_test.go b/cmd/server/relay_candidates_alloc_test.go new file mode 100644 index 000000000..6e3988c15 --- /dev/null +++ b/cmd/server/relay_candidates_alloc_test.go @@ -0,0 +1,136 @@ +package main + +import ( + "encoding/hex" + "math/rand" + "strings" + "testing" +) + +// relayCandidates runs once per hop of every observation the cold loader +// resolves. With any listener-only observer known it built a filtered copy +// of the candidate nodeInfo structs on every call: about 1 GB allocated over +// one startup on a week of a busy network. The filtered lists are built once +// when the listeners are marked. +func TestRelayCandidatesDoesNotAllocate(t *testing.T) { + pm := buildPrefixMap([]nodeInfo{ + {PublicKey: "a4aa000000000000000000000000000000000000000000000000000000000001", Role: "repeater"}, + {PublicKey: "a4bb000000000000000000000000000000000000000000000000000000000002", Role: "repeater"}, + {PublicKey: "c1cc000000000000000000000000000000000000000000000000000000000003", Role: "repeater"}, + }) + pm.markNonRelay([]string{"A4BB000000000000000000000000000000000000000000000000000000000002"}) + + if got := pm.relayCandidates("A4"); len(got) != 1 || got[0].PublicKey[:4] != "a4aa" { + t.Fatalf("relayCandidates(A4) = %v, want only the a4aa relay", got) + } + if got := pm.relayCandidates("a4bb"); len(got) != 0 { + t.Fatalf("relayCandidates(a4bb) = %v, want none (the only match is a listener)", got) + } + if got := pm.relayCandidates("c1"); len(got) != 1 { + t.Fatalf("relayCandidates(c1) = %v, want one", got) + } + for _, hop := range []string{"a4", "a4bb", "c1", "ff"} { + if n := testing.AllocsPerRun(100, func() { _ = pm.relayCandidates(hop) }); n != 0 { + t.Errorf("relayCandidates(%q) allocates %.0f times per call, want 0", hop, n) + } + } +} + +// Marking more listeners later still filters them. +func TestRelayCandidatesAfterSecondMark(t *testing.T) { + pm := buildPrefixMap([]nodeInfo{ + {PublicKey: "a4aa000000000000000000000000000000000000000000000000000000000001", Role: "repeater"}, + {PublicKey: "a4bb000000000000000000000000000000000000000000000000000000000002", Role: "repeater"}, + }) + pm.markNonRelay([]string{"a4aa000000000000000000000000000000000000000000000000000000000001"}) + pm.markNonRelay([]string{"a4bb000000000000000000000000000000000000000000000000000000000002"}) + if got := pm.relayCandidates("a4"); len(got) != 0 { + t.Fatalf("relayCandidates(a4) = %v, want none after both are marked", got) + } +} + +// markNonRelayFixture is 5,000 repeaters with random keys, three of which +// are marked as listeners, the size of a busy regional network. +func markNonRelayFixture() (*prefixMap, []string) { + rng := rand.New(rand.NewSource(1290)) + nodes := make([]nodeInfo, 5000) + for i := range nodes { + b := make([]byte, 32) + rng.Read(b) + nodes[i] = nodeInfo{PublicKey: hex.EncodeToString(b), Role: "repeater"} + } + return buildPrefixMap(nodes), []string{nodes[7].PublicKey, nodes[1234].PublicKey, strings.ToUpper(nodes[4321].PublicKey)} +} + +// markNonRelay runs on every node cache rebuild, as often as every 30 s. It +// used to copy every prefix's list to find the few that hold a listener; +// those can only be the listeners' own prefixes, so only they are visited. +func TestMarkNonRelayVisitsOnlyListenerPrefixes(t *testing.T) { + _, listeners := markNonRelayFixture() + n := testing.AllocsPerRun(5, func() { + pm, _ := markNonRelayFixture() + pm.markNonRelay(listeners) + }) - testing.AllocsPerRun(5, func() { markNonRelayFixture() }) + // Three listeners have at most 8 prefixes each, plus the maps themselves. + if n > 80 { + t.Errorf("markNonRelay allocates %.0f times for 3 listeners among 5,000 repeaters, want at most 80", n) + } +} + +// Every prefix gives what filtering its list on the spot would. +func TestMarkNonRelayMatchesFilteringEachList(t *testing.T) { + pm, listeners := markNonRelayFixture() + pm.markNonRelay(listeners) + isListener := map[string]bool{} + for _, pk := range listeners { + isListener[strings.ToLower(pk)] = true + } + for key, candidates := range pm.m { + var want []nodeInfo + for _, c := range candidates { + if !isListener[strings.ToLower(c.PublicKey)] { + want = append(want, c) + } + } + got := pm.relayCandidates(key) + if len(got) != len(want) { + t.Fatalf("relayCandidates(%q) has %d candidates, want %d", key, len(got), len(want)) + } + for i := range want { + if got[i].PublicKey != want[i].PublicKey { + t.Fatalf("relayCandidates(%q)[%d] = %s, want %s", key, i, got[i].PublicKey, want[i].PublicKey) + } + } + } +} + +func BenchmarkMarkNonRelay5000(b *testing.B) { + pm, listeners := markNonRelayFixture() + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + pm.nonRelay, pm.relay = nil, nil // as on a fresh rebuild + pm.markNonRelay(listeners) + } +} + +// relayCandidates is the other side of markNonRelay's work: one lookup per +// 2-character prefix, the hop width most paths use, so 256 lookups per op. +func BenchmarkRelayCandidates256(b *testing.B) { + pm, listeners := markNonRelayFixture() + pm.markNonRelay(listeners) + const digits = "0123456789abcdef" + hops := make([]string, 0, 256) + for _, x := range digits { + for _, y := range digits { + hops = append(hops, string(x)+string(y)) + } + } + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + for _, h := range hops { + _ = pm.relayCandidates(h) + } + } +} diff --git a/cmd/server/resolved_index.go b/cmd/server/resolved_index.go index 5b0f05a72..940733dd5 100644 --- a/cmd/server/resolved_index.go +++ b/cmd/server/resolved_index.go @@ -516,38 +516,35 @@ func (s *PacketStore) resolvedPubkeysForEvictionBatch(txIDs []int) map[int][]str continue } + var txID int + var rpJSON sql.NullString for rows.Next() { - var txID int - var rpJSON sql.NullString if err := rows.Scan(&txID, &rpJSON); err != nil { continue } if !rpJSON.Valid || rpJSON.String == "" { continue } - rp := unmarshalResolvedPath(rpJSON.String) - for _, p := range rp { - if p != nil && *p != "" { - result[txID] = append(result[txID], *p) + // De-duplicate as we go: a packet has a handful of keys, so a + // scan of its list is cheaper than a map per packet. + pks := result[txID] + for _, pk := range resolvedPubkeysFromJSON(rpJSON.String) { + dup := false + for _, have := range pks { + if have == pk { + dup = true + break + } + } + if !dup { + pks = append(pks, pk) } } + result[txID] = pks } rows.Close() } - // Deduplicate per-txID - for txID, pks := range result { - seen := make(map[string]bool, len(pks)) - deduped := pks[:0] - for _, pk := range pks { - if !seen[pk] { - seen[pk] = true - deduped = append(deduped, pk) - } - } - result[txID] = deduped - } - return result } diff --git a/cmd/server/resolved_pubkeys_bench_test.go b/cmd/server/resolved_pubkeys_bench_test.go new file mode 100644 index 000000000..70adcb8bd --- /dev/null +++ b/cmd/server/resolved_pubkeys_bench_test.go @@ -0,0 +1,37 @@ +package main + +import ( + "fmt" + "strings" + "testing" +) + +// One resolved_path per observation: 5 relays and 3 unresolved hops, drawn +// from a pool of 2,000 relays, as on a busy instance. +func BenchmarkResolvedPubkeysFromJSON(b *testing.B) { + const pool = 2000 + paths := make([]string, 256) + for i := range paths { + hops := make([]string, 0, 8) + for h := 0; h < 5; h++ { + hops = append(hops, fmt.Sprintf(`"%064x"`, (i*7+h*131)%pool)) + } + hops = append(hops, "null", "null", "null") + paths[i] = "[" + strings.Join(hops, ",") + "]" + } + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + _ = resolvedPubkeysFromJSON(paths[i%len(paths)]) + } +} + +// An eviction batch of 1,000 packets with 3 observations each. +func BenchmarkResolvedPubkeysForEvictionBatch(b *testing.B) { + s, ids := evictionResolvedStore(b, 1000) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + _ = s.resolvedPubkeysForEvictionBatch(ids) + } +} diff --git a/cmd/server/resolved_pubkeys_json_test.go b/cmd/server/resolved_pubkeys_json_test.go new file mode 100644 index 000000000..10e1b689f --- /dev/null +++ b/cmd/server/resolved_pubkeys_json_test.go @@ -0,0 +1,52 @@ +package main + +import ( + "math/rand" + "strings" + "testing" +) + +// The bulk loaders decode each observation's resolved_path (a JSON array of +// pubkeys and nulls, ~420 chars on live data, set on most observations) +// only to collect its non-empty pubkeys. Decoding into []*string allocated +// a string and a pointer per hop plus two slices: about 210MB over one +// startup on a busy week. resolvedPubkeysFromJSON reads the usual array in +// place, and a pubkey it has seen before costs nothing: only the result is +// allocated. +func TestResolvedPubkeysFromJSONAllocations(t *testing.T) { + pk := strings.Repeat("ab", 32) + rp := `["` + pk + `",null,"` + strings.Repeat("cd", 32) + `",null,"` + strings.Repeat("ef", 32) + `"]` + _ = resolvedPubkeysFromJSON(rp) // first sight keeps one copy of each + if n := testing.AllocsPerRun(100, func() { _ = resolvedPubkeysFromJSON(rp) }); n > 1 { + t.Fatalf("resolvedPubkeysFromJSON allocates %.0f times for 3 known pubkeys, want at most 1 (the result)", n) + } +} + +// It returns exactly what decoding and extracting would, odd inputs included. +func TestResolvedPubkeysFromJSONMatchesDecoding(t *testing.T) { + cases := []string{ + "", "[]", "null", "[null]", `["aa"]`, `["aa",null,"bb"]`, ` [ "aa" , null ] `, `[""]`, `["",null,""]`, + `["aa",]`, `["aa"`, `["aa"]x`, `[1]`, `["aa",1]`, `[nul]`, `[nulll]`, `["a\"b"]`, `["é"]`, + `[["aa"]]`, `{"a":1}`, "[\"a\x01\"]", `["aa" "bb"]`, `[,]`, `[null,]`, "[\"\xff\"]", `[true]`, + } + rng := rand.New(rand.NewSource(2)) + alphabet := []string{`[`, `]`, `"`, `,`, ` `, `a`, `1`, `\`, `null`, `nul`, `{`, `:`} + for i := 0; i < 20000; i++ { + var b strings.Builder + for j := rng.Intn(12); j >= 0; j-- { + b.WriteString(alphabet[rng.Intn(len(alphabet))]) + } + cases = append(cases, b.String()) + } + for _, s := range cases { + got, want := resolvedPubkeysFromJSON(s), extractResolvedPubkeys(unmarshalResolvedPath(s)) + if len(got) != len(want) { + t.Fatalf("%q: got %q, want %q", s, got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("%q: got %q, want %q", s, got, want) + } + } + } +} diff --git a/cmd/server/retransmission_pressure_test.go b/cmd/server/retransmission_pressure_test.go index 9226ba272..0456f1412 100644 --- a/cmd/server/retransmission_pressure_test.go +++ b/cmd/server/retransmission_pressure_test.go @@ -675,8 +675,7 @@ func BenchmarkComputeRetransmissionPressureColdTimestamps(b *testing.B) { b.StopTimer() for _, tx := range s.packets { for _, o := range tx.Observations { - o.tsParseOnce = sync.Once{} - o.tsParsed, o.tsParsedOK = time.Time{}, false + o.tsUnixNano.Store(0) } } b.StartTimer() diff --git a/cmd/server/routes.go b/cmd/server/routes.go index 3465aa18e..608e95d5b 100644 --- a/cmd/server/routes.go +++ b/cmd/server/routes.go @@ -7,10 +7,12 @@ import ( "errors" "fmt" "log" + "maps" "math" "net/http" "regexp" "runtime" + "slices" "sort" "strconv" "strings" @@ -2618,8 +2620,8 @@ func (s *Server) handleAnalyticsSubpaths(w http.ResponseWriter, r *http.Request) // Issue #1217: honor the Time window filter on Route Patterns. window := ParseTimeWindow(r) data := s.store.GetAnalyticsSubpathsWithWindow(region, minLen, maxLen, limit, window) - if s.cfg != nil && len(s.cfg.NodeBlacklist) > 0 { - data = s.filterBlacklistedFromSubpaths(data) + if s.subpathFilterActive() { + data, _ = s.dropHiddenPaths(data, "subpaths", "path", "", s.blacklistedNodeNames()) } writeJSON(w, data) return @@ -2684,9 +2686,10 @@ func (s *Server) handleAnalyticsSubpathsBulk(w http.ResponseWriter, r *http.Requ } results := s.store.GetAnalyticsSubpathsBulkWithWindow(region, groups, ParseTimeWindow(r)) - if s.cfg != nil && len(s.cfg.NodeBlacklist) > 0 { + if s.subpathFilterActive() { + names := s.blacklistedNodeNames() for i, r := range results { - results[i] = s.filterBlacklistedFromSubpaths(r) + results[i], _ = s.dropHiddenPaths(r, "subpaths", "path", "", names) } } writeJSON(w, map[string]interface{}{"results": results}) @@ -2722,7 +2725,19 @@ func (s *Server) handleAnalyticsSubpathDetail(w http.ResponseWriter, r *http.Req writeIndexLoading503(w) return } - writeJSON(w, s.store.GetSubpathDetail(rawHops)) + detail := s.store.GetSubpathDetail(rawHops) + // #2184: the raw hops are prefixes, so the check above only catches a + // full key. Check the nodes the store resolved them to instead. + if s.subpathFilterActive() { + // Match each hop by its resolved key, so a blacklisted node without a + // name is caught and a node that only shares its name is not. + if _, dropped := s.dropHiddenPaths(detail, "nodes", "name", "pubkey", nil); dropped > 0 { + writeError(w, 404, "Not found") + return + } + detail, _ = s.dropHiddenPaths(detail, "parentPaths", "path", "", s.blacklistedNodeNames()) + } + writeJSON(w, detail) return } writeJSON(w, SubpathDetailResponse{ @@ -3677,8 +3692,9 @@ func (s *Server) handleMetricsSummary(w http.ResponseWriter, r *http.Request) { return } - since := time.Now().UTC().Add(-dur).Format(time.RFC3339) - summary, err := s.db.GetMetricsSummary(since) + now := time.Now().UTC() + since := now.Add(-dur).Format(time.RFC3339) + summary, err := s.db.GetMetricsSummary(since, now) if err != nil { writeError(w, 500, err.Error()) return @@ -3698,8 +3714,9 @@ func (s *Server) handleMetricsSummary(w http.ResponseWriter, r *http.Request) { summary = filtered } - writeJSON(w, map[string]interface{}{ - "observers": summary, + writeJSON(w, MetricsSummaryResponse{ + Observers: summary, + DutyCycleWarnPct: s.cfg.DutyCycleWarnPctOrDefault(), }) } @@ -3827,33 +3844,62 @@ func (s *Server) filterBlacklistedFromTopology(data map[string]interface{}) map[ return data } -// filterBlacklistedFromSubpaths removes blacklisted node references from -// the subpaths analytics response. -func (s *Server) filterBlacklistedFromSubpaths(data map[string]interface{}) map[string]interface{} { - if subpaths, ok := data["subpaths"]; ok { - if arr, ok := subpaths.([]interface{}); ok { - var filtered []interface{} - for _, item := range arr { - if m, ok := item.(map[string]interface{}); ok { - if hops, ok := m["hops"].([]interface{}); ok { - skip := false - for _, h := range hops { - if hp, ok := h.(string); ok && (s.cfg.IsBlacklisted(hp) || s.isPubkeyHidden(hp)) { - skip = true - break - } - } - if skip { - continue - } - } - } - filtered = append(filtered, item) - } - data["subpaths"] = filtered +// subpathFilterActive reports whether Route Patterns responses need the +// nodeBlacklist / hiddenNamePrefixes filter. +func (s *Server) subpathFilterActive() bool { + return s.cfg != nil && (len(s.cfg.NodeBlacklist) > 0 || len(s.cfg.HiddenNamePrefixes) > 0) +} + +// blacklistedNodeNames returns the names of the blacklisted nodes the store +// knows, for filtering responses that carry node names but no keys. +func (s *Server) blacklistedNodeNames() map[string]bool { + if s.store == nil || s.cfg == nil || len(s.cfg.NodeBlacklist) == 0 { + return nil + } + nodes, _ := s.store.getCachedNodesAndPM() + names := make(map[string]bool) + for _, n := range nodes { + if n.Name != "" && s.cfg.IsBlacklisted(n.PublicKey) { + names[n.Name] = true } } - return data + return names +} + +// subpathPathHidden reports whether a resolved " → " path shows a blacklisted +// or hidden-prefix node. Issue #2184: a subpath carries the names the store +// resolved its hops to, not keys, and a hop prefix alone cannot be pinned to +// one node, so the filter matches what the reader would see. +func (s *Server) subpathPathHidden(path string, blacklisted map[string]bool) bool { + for _, name := range strings.Split(path, " → ") { + if blacklisted[name] || s.cfg.IsNameHidden(name) { + return true + } + } + return false +} + +// dropHiddenPaths removes the entries of data[listKey] whose field (a node +// name, or a resolved " → " path) shows a blacklisted or hidden-prefix node, +// or, when keyField is set, whose key there is blacklisted, and reports how +// many it removed. It returns a copy: data may be the store's cached +// response, shared with other requests. +func (s *Server) dropHiddenPaths(data map[string]interface{}, listKey, field, keyField string, blacklisted map[string]bool) (map[string]interface{}, int) { + entries, ok := data[listKey].([]map[string]interface{}) + if !ok { + return data, 0 + } + kept := slices.Clone(entries)[:0] + for _, e := range entries { + path, _ := e[field].(string) + key, _ := e[keyField].(string) + if !s.subpathPathHidden(path, blacklisted) && (key == "" || !s.cfg.IsBlacklisted(key)) { + kept = append(kept, e) + } + } + out := maps.Clone(data) + out[listKey] = kept + return out, len(entries) - len(kept) } // handleDroppedPackets returns recently dropped packets for investigation. diff --git a/cmd/server/row_scan_allocs_test.go b/cmd/server/row_scan_allocs_test.go new file mode 100644 index 000000000..2e9b32dc8 --- /dev/null +++ b/cmd/server/row_scan_allocs_test.go @@ -0,0 +1,73 @@ +package main + +import ( + "os" + "path/filepath" + "runtime" + "testing" + "time" +) + +// The bulk loaders declared their ~20 scan targets and built the Scan +// argument slice inside the row loop. Each target escapes (its address goes +// into the []interface{}), so every row allocated them all again, plus the +// slice and its growth: about 330MB of the allocations loadChunk made over a +// startup on a week of a busy network. Scan overwrites every target on each +// row, so one set serves the whole loop. On this fixture that took each +// loader from about 89 allocations per row to about 67; the limits sit +// between. + +func rowScanStore(t *testing.T, n int) *PacketStore { + t.Helper() + dir := t.TempDir() + dbPath := filepath.Join(dir, "rows.db") + createTestDBAt(t, dbPath, n) + t.Cleanup(func() { os.RemoveAll(dir) }) + db, err := OpenDB(dbPath) + if err != nil { + t.Fatalf("OpenDB: %v", err) + } + t.Cleanup(func() { db.Close() }) + return NewPacketStore(db, &PacketStoreConfig{}) +} + +func mallocsPerRow(t *testing.T, n int, load func(*PacketStore) error) float64 { + t.Helper() + s := rowScanStore(t, n) + var a, b runtime.MemStats + runtime.GC() + runtime.ReadMemStats(&a) + if err := load(s); err != nil { + t.Fatal(err) + } + runtime.ReadMemStats(&b) + if len(s.packets) != n { + t.Fatalf("loaded %d packets, want %d", len(s.packets), n) + } + return float64(b.Mallocs-a.Mallocs) / float64(n) +} + +func TestBulkLoadersReuseScanTargets(t *testing.T) { + const n = 2000 + cases := []struct { + name string + limit float64 + load func(*PacketStore) error + }{ + {"Load", 78, func(s *PacketStore) error { return s.Load() }}, + {"LoadChunked", 83, func(s *PacketStore) error { return s.LoadChunked(500) }}, + {"loadChunk", 78, func(s *PacketStore) error { + from := time.Date(2025, 12, 31, 0, 0, 0, 0, time.UTC) + return s.loadChunk(from, from.Add(72*time.Hour)) + }}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got := mallocsPerRow(t, n, c.load) + t.Logf("%s: %.1f allocations per row", c.name, got) + if got > c.limit { + t.Errorf("%s makes %.1f allocations per row, limit %.0f: are the scan targets rebuilt per row?", c.name, got, c.limit) + } + }) + } +} diff --git a/cmd/server/store.go b/cmd/server/store.go index acd86c636..ec7b73300 100644 --- a/cmd/server/store.go +++ b/cmd/server/store.go @@ -93,36 +93,82 @@ type StoreObs struct { // #1481 P0-2: cached parsed timestamp. The legacy handlers parse // Timestamp as RFC3339Nano/RFC3339/"2006-01-02 15:04:05" with a // fallback chain on every read; for /api/observers/{id}/analytics - // that fires 60k+ times per request under RLock. ParsedTime returns - // the parsed value once, caching for the lifetime of the StoreObs. - tsParseOnce sync.Once - tsParsed time.Time - tsParsedOK bool + // that fires 60k+ times per request under RLock. ParsedTime parses + // once and keeps the result here in 8 bytes (see tsUnparsed): a + // time.Time, sync.Once and bool took 40 and put each observation in + // the 192-byte size class instead of 160. + tsUnixNano atomic.Int64 } // ParsedTime returns the parsed Timestamp, caching the result. // Tries RFC3339Nano → RFC3339 → "2006-01-02 15:04:05" in order. // Returns (zero, false) if Timestamp is empty or unparseable. -// Thread-safe via sync.Once. +// Safe for concurrent use: callers racing on the first call each parse and +// store the same value. func (o *StoreObs) ParsedTime() (time.Time, bool) { - o.tsParseOnce.Do(func() { - if o.Timestamp == "" { - return - } - if t, err := time.Parse(time.RFC3339Nano, o.Timestamp); err == nil { - o.tsParsed, o.tsParsedOK = t, true - return - } - if t, err := time.Parse(time.RFC3339, o.Timestamp); err == nil { - o.tsParsed, o.tsParsedOK = t, true - return - } - if t, err := time.Parse("2006-01-02 15:04:05", o.Timestamp); err == nil { - o.tsParsed, o.tsParsedOK = t, true - return + switch n := o.tsUnixNano.Load(); n { + case 0: + case tsUnparsed: + return time.Time{}, false + case tsReparse: + return parseObsTimestamp(o.Timestamp) + default: + return time.Unix(0, n).UTC(), true + } + t, ok := parseObsTimestamp(o.Timestamp) + switch { + case !ok: + o.tsUnixNano.Store(tsUnparsed) + case t.Location() != time.UTC || t.Year() < 1678 || t.Year() > 2261 || t.UnixNano() == 0: + // Kept only as UTC nanoseconds; a time with another offset (or one + // UnixNano cannot hold) is parsed again on each call so its + // location is kept. + o.tsUnixNano.Store(tsReparse) + default: + o.tsUnixNano.Store(t.UnixNano()) + return time.Unix(0, t.UnixNano()).UTC(), true + } + return t, ok +} + +// tsUnixNano holds 0 before the first ParsedTime, the UTC time in +// nanoseconds after it, or one of these. +const ( + tsUnparsed = math.MinInt64 // empty or not a time + tsReparse = math.MinInt64 + 1 // parsed, but not held as UTC nanoseconds +) + +func parseObsTimestamp(s string) (time.Time, bool) { + if s == "" { + return time.Time{}, false + } + for _, layout := range []string{time.RFC3339Nano, time.RFC3339, "2006-01-02 15:04:05"} { + if t, err := time.Parse(layout, s); err == nil { + return t, true } - }) - return o.tsParsed, o.tsParsedOK + } + return time.Time{}, false +} + +// decodedCacheDroppedKeys are payload fields no server code reads: the +// ciphertext and MAC of encrypted payloads, advert signatures, the ISO copy +// of a timestamp, raw path data and control-packet details. +// TestDecodedCacheDroppedKeysAreUnread fails if code starts naming one. +var decodedCacheDroppedKeys = []string{ + "encryptedData", "mac", "signature", "signatureValid", "timestampISO", "pathData", + "ctrlPubKey", "ctrlSubtype", "ctrlTag", "ctrlFlags", "ctrlZeroHop", "ctrlLength", +} + +// broadcastPayload returns the packet's whole decoded payload for the live +// feed: the stored JSON as it is, since the feed only serialises it. Copying +// the cache would lose the fields it drops, and decoding again would run +// under the store's write lock; this does neither. nil if the JSON is not +// valid, as before. +func broadcastPayload(tx *StoreTx) json.RawMessage { + if tx.DecodedJSON == "" || !json.Valid([]byte(tx.DecodedJSON)) { + return nil + } + return json.RawMessage(tx.DecodedJSON) } // ParsedDecoded returns the parsed DecodedJSON map, caching the result. @@ -131,6 +177,9 @@ func (tx *StoreTx) ParsedDecoded() map[string]interface{} { tx.decodedOnce.Do(func() { if tx.DecodedJSON != "" { json.Unmarshal([]byte(tx.DecodedJSON), &tx.parsedDecoded) + for _, k := range decodedCacheDroppedKeys { + delete(tx.parsedDecoded, k) + } } }) return tx.parsedDecoded @@ -187,6 +236,10 @@ type PacketStore struct { advertEvidenceCursor int64 advertEvidenceRevision uint64 // cacheMu + // tx_rewrite_feed tail (#2107): in-place rewrites of loaded rows. + txRewriteMu sync.Mutex + txRewriteCursor int64 + mu sync.RWMutex db *DB packets []*StoreTx // sorted by first_seen ASC (oldest first; newest at tail) @@ -784,6 +837,11 @@ func NewPacketStore(db *DB, cfg *PacketStoreConfig, cacheTTLs ...map[string]inte if db.advertEvidencePresent() { _ = db.conn.QueryRow(`SELECT COALESCE(MAX(id),0) FROM advert_route_evidence`).Scan(&ps.advertEvidenceCursor) } + // Same watermark rule for the rewrite feed: rows loaded below already + // carry every rewrite committed before this point. + if db.txRewriteFeedPresent() { + _ = db.conn.QueryRow(`SELECT COALESCE(MAX(id),0) FROM tx_rewrite_feed`).Scan(&ps.txRewriteCursor) + } return ps } @@ -927,31 +985,34 @@ func (s *PacketStore) Load() error { hopsSeen := make(map[string]bool) // reused across observations; cleared per use - for rows.Next() { - var txID int - var rawHex, hash, firstSeen, decodedJSON sql.NullString - var routeType, payloadType, payloadVersion sql.NullInt64 - var obsID sql.NullInt64 - var observerID, observerName, observerIATA, direction, pathJSON, obsTimestamp sql.NullString - var snr, rssi sql.NullFloat64 - var score sql.NullInt64 - var obsRawHex sql.NullString - var resolvedPathStr sql.NullString - var scopeName sql.NullString + // Scan overwrites every target on each row, so one set (and one + // argument slice) serves the whole loop instead of escaping per row. + var txID int + var rawHex, hash, firstSeen, decodedJSON sql.NullString + var routeType, payloadType, payloadVersion sql.NullInt64 + var obsID sql.NullInt64 + var observerID, observerName, observerIATA, direction, pathJSON, obsTimestamp sql.NullString + var snr, rssi sql.NullFloat64 + var score sql.NullInt64 + var obsRawHex sql.NullString + var resolvedPathStr sql.NullString + var scopeName sql.NullString + + scanArgs := []interface{}{&txID, &rawHex, &hash, &firstSeen, &routeType, &payloadType, + &payloadVersion, &decodedJSON, + &obsID, &observerID, &observerName, &observerIATA, &direction, + &snr, &rssi, &score, &pathJSON, &obsTimestamp} + if s.db.hasObsRawHex { + scanArgs = append(scanArgs, &obsRawHex) + } + if s.db.hasResolvedPath { + scanArgs = append(scanArgs, &resolvedPathStr) + } + if s.db.hasScopeName { + scanArgs = append(scanArgs, &scopeName) + } - scanArgs := []interface{}{&txID, &rawHex, &hash, &firstSeen, &routeType, &payloadType, - &payloadVersion, &decodedJSON, - &obsID, &observerID, &observerName, &observerIATA, &direction, - &snr, &rssi, &score, &pathJSON, &obsTimestamp} - if s.db.hasObsRawHex { - scanArgs = append(scanArgs, &obsRawHex) - } - if s.db.hasResolvedPath { - scanArgs = append(scanArgs, &resolvedPathStr) - } - if s.db.hasScopeName { - scanArgs = append(scanArgs, &scopeName) - } + for rows.Next() { if err := rows.Scan(scanArgs...); err != nil { log.Printf("[store] scan error: %v", err) continue @@ -999,10 +1060,10 @@ func (s *PacketStore) Load() error { obs := &StoreObs{ ID: oid, TransmissionID: txID, - ObserverID: obsIDStr, - ObserverName: nullStrVal(observerName), - ObserverIATA: nullStrVal(observerIATA), - Direction: nullStrVal(direction), + ObserverID: internStr(obsIDStr), + ObserverName: internStr(nullStrVal(observerName)), + ObserverIATA: internStr(nullStrVal(observerIATA)), + Direction: internStr(nullStrVal(direction)), SNR: nullFloatPtr(snr), RSSI: nullFloatPtr(rssi), Score: nullIntPtr(score), @@ -1013,8 +1074,7 @@ func (s *PacketStore) Load() error { // Decode-window: extract resolved pubkeys for index, don't store on struct. rpStr := nullStrVal(resolvedPathStr) if rpStr != "" { - rp := unmarshalResolvedPath(rpStr) - pks := extractResolvedPubkeys(rp) + pks := resolvedPubkeysFromJSON(rpStr) // Single point of truth — see indexResolvedPathHops doc + #1558. s.indexResolvedPathHops(tx, pks, hopsSeen) } else if relayPM != nil && obsPJ != "" && obsPJ != "[]" { @@ -1256,31 +1316,34 @@ func (s *PacketStore) loadChunk(from, to time.Time) error { var localMaxTxID int var localMaxObsID int - for rows.Next() { - var txID int - var rawHex, hash, firstSeen, decodedJSON sql.NullString - var routeType, payloadType, payloadVersion sql.NullInt64 - var obsID sql.NullInt64 - var observerID, observerName, observerIATA, direction, pathJSON, obsTimestamp sql.NullString - var snr, rssi sql.NullFloat64 - var score sql.NullInt64 - var obsRawHex sql.NullString - var resolvedPathStr sql.NullString - var scopeName sql.NullString + // Scan overwrites every target on each row, so one set (and one + // argument slice) serves the whole loop instead of escaping per row. + var txID int + var rawHex, hash, firstSeen, decodedJSON sql.NullString + var routeType, payloadType, payloadVersion sql.NullInt64 + var obsID sql.NullInt64 + var observerID, observerName, observerIATA, direction, pathJSON, obsTimestamp sql.NullString + var snr, rssi sql.NullFloat64 + var score sql.NullInt64 + var obsRawHex sql.NullString + var resolvedPathStr sql.NullString + var scopeName sql.NullString + + scanArgs := []interface{}{&txID, &rawHex, &hash, &firstSeen, &routeType, &payloadType, + &payloadVersion, &decodedJSON, + &obsID, &observerID, &observerName, &observerIATA, &direction, + &snr, &rssi, &score, &pathJSON, &obsTimestamp} + if s.db.hasObsRawHex { + scanArgs = append(scanArgs, &obsRawHex) + } + if s.db.hasResolvedPath { + scanArgs = append(scanArgs, &resolvedPathStr) + } + if s.db.hasScopeName { + scanArgs = append(scanArgs, &scopeName) + } - scanArgs := []interface{}{&txID, &rawHex, &hash, &firstSeen, &routeType, &payloadType, - &payloadVersion, &decodedJSON, - &obsID, &observerID, &observerName, &observerIATA, &direction, - &snr, &rssi, &score, &pathJSON, &obsTimestamp} - if s.db.hasObsRawHex { - scanArgs = append(scanArgs, &obsRawHex) - } - if s.db.hasResolvedPath { - scanArgs = append(scanArgs, &resolvedPathStr) - } - if s.db.hasScopeName { - scanArgs = append(scanArgs, &scopeName) - } + for rows.Next() { if err := rows.Scan(scanArgs...); err != nil { log.Printf("[store] loadChunk scan error: %v", err) continue @@ -1321,10 +1384,10 @@ func (s *PacketStore) loadChunk(from, to time.Time) error { obs := &StoreObs{ ID: oid, TransmissionID: txID, - ObserverID: obsIDStr, - ObserverName: nullStrVal(observerName), - ObserverIATA: nullStrVal(observerIATA), - Direction: nullStrVal(direction), + ObserverID: internStr(obsIDStr), + ObserverName: internStr(nullStrVal(observerName)), + ObserverIATA: internStr(nullStrVal(observerIATA)), + Direction: internStr(nullStrVal(direction)), SNR: nullFloatPtr(snr), RSSI: nullFloatPtr(rssi), Score: nullIntPtr(score), @@ -1355,8 +1418,7 @@ func (s *PacketStore) loadChunk(from, to time.Time) error { // they were heard via — Load() does the equivalent inline. rpStr := nullStrVal(resolvedPathStr) if rpStr != "" { - rp := unmarshalResolvedPath(rpStr) - pks := extractResolvedPubkeys(rp) + pks := resolvedPubkeysFromJSON(rpStr) accumulateDedup(localResolvedPKsByTx, localResolvedSeenByTx, txID, pks) } else if relayPM != nil && obsPJ != "" && obsPJ != "[]" { // resolved_path not persisted — reconstruct relay hops from @@ -2908,10 +2970,10 @@ func (s *PacketStore) IngestNewFromDB(sinceID, limit int) ([]map[string]interfac decodedJSON: nullStrVal(decodedJSON), routeType: nullIntPtr(routeType), payloadType: nullIntPtr(payloadType), - observerID: nullStrVal(observerID), - observerName: nullStrVal(observerName), - observerIATA: nullStrVal(observerIATA), - direction: nullStrVal(direction), + observerID: internStr(nullStrVal(observerID)), + observerName: internStr(nullStrVal(observerName)), + observerIATA: internStr(nullStrVal(observerIATA)), + direction: internStr(nullStrVal(direction)), pathJSON: nullStrVal(pathJSON), obsTS: nullStrVal(obsTimestamp), obsRawHex: nullStrVal(obsRawHex), @@ -3025,7 +3087,7 @@ func (s *PacketStore) IngestNewFromDB(sinceID, limit int) ([]map[string]interfac obs := &StoreObs{ ID: oid, TransmissionID: r.txID, - ObserverID: r.observerID, + ObserverID: r.observerID, // interned while scanning, outside s.mu ObserverName: r.observerName, ObserverIATA: r.observerIATA, Direction: r.direction, @@ -3123,14 +3185,8 @@ func (s *PacketStore) IngestNewFromDB(sinceID, limit int) ([]map[string]interfac }, } if tx.DecodedJSON != "" { - if payload := tx.ParsedDecoded(); payload != nil { - // Copy the cached map so broadcast consumers cannot mutate - // the shared cache or cause a data race (#1871 review). - cp := make(map[string]interface{}, len(payload)) - for k, v := range payload { - cp[k] = v - } - decoded["payload"] = cp + if payload := broadcastPayload(tx); payload != nil { + decoded["payload"] = payload } } // For TRACE packets, decode the full packet to include path.hopsCompleted @@ -3221,6 +3277,7 @@ func (s *PacketStore) IngestNewFromDB(sinceID, limit int) ([]map[string]interfac func (s *PacketStore) IngestNewObservations(sinceObsID, limit int) []map[string]interface{} { // Must run even when observation IDs/timestamps have not changed. s.refreshAdvertEvidence() + s.refreshTxRewrites() // #2107: rewrites never move IDs, so poll them here too if limit <= 0 { limit = 500 } @@ -3289,10 +3346,10 @@ func (s *PacketStore) IngestNewObservations(sinceObsID, limit int) []map[string] obsRows = append(obsRows, obsRow{ obsID: oid, txID: txID, - observerID: nullStrVal(observerID), - observerName: nullStrVal(observerName), - observerIATA: nullStrVal(observerIATA), - direction: nullStrVal(direction), + observerID: internStr(nullStrVal(observerID)), + observerName: internStr(nullStrVal(observerName)), + observerIATA: internStr(nullStrVal(observerIATA)), + direction: internStr(nullStrVal(direction)), snr: nullFloatPtr(snr), rssi: nullFloatPtr(rssi), score: nullIntPtr(score), @@ -3341,7 +3398,7 @@ func (s *PacketStore) IngestNewObservations(sinceObsID, limit int) []map[string] obs := &StoreObs{ ID: r.obsID, TransmissionID: r.txID, - ObserverID: r.observerID, + ObserverID: r.observerID, // interned while scanning, outside s.mu ObserverName: r.observerName, ObserverIATA: r.observerIATA, Direction: r.direction, @@ -3396,14 +3453,8 @@ func (s *PacketStore) IngestNewObservations(sinceObsID, limit int) []map[string] }, } if tx.DecodedJSON != "" { - if payload := tx.ParsedDecoded(); payload != nil { - // Copy the cached map so broadcast consumers cannot mutate - // the shared cache or cause a data race (#1871 review). - cp := make(map[string]interface{}, len(payload)) - for k, v := range payload { - cp[k] = v - } - decoded["payload"] = cp + if payload := broadcastPayload(tx); payload != nil { + decoded["payload"] = payload } } // For TRACE packets, decode the full packet to include path.hopsCompleted @@ -7121,6 +7172,10 @@ type prefixMap struct { // contract they do not forward packets. nil/empty preserves the // pre-#1290 behavior (every prefix-matching node is a candidate). nonRelay map[string]struct{} + // relay holds, for each prefix whose candidates include a listener, the + // candidates without them; markNonRelay builds it so relayCandidates, + // called per hop of every observation resolved, never copies. + relay map[string][]nodeInfo } // markNonRelay registers a set of lowercase pubkeys as listener-only. @@ -7136,6 +7191,33 @@ func (pm *prefixMap) markNonRelay(pubkeys []string) { for _, pk := range pubkeys { pm.nonRelay[strings.ToLower(pk)] = struct{}{} } + // A listener can only be in the lists of its own prefixes (and its full + // key), so those are the only lists to filter: at most maxPrefixLen+1 + // per listener, rather than every list in pm.m. markNonRelay runs on + // every node cache rebuild. + pm.relay = make(map[string][]nodeInfo) + for pk := range pm.nonRelay { + for l := 2; l <= len(pk); l++ { + if l > maxPrefixLen && l < len(pk) { + l = len(pk) // past the prefixes, only the full key is a list + } + key := pk[:l] + if _, done := pm.relay[key]; done { + continue + } + candidates, ok := pm.m[key] + if !ok { + continue + } + filtered := candidates[:0:0] + for i := range candidates { + if _, isListener := pm.nonRelay[strings.ToLower(candidates[i].PublicKey)]; !isListener { + filtered = append(filtered, candidates[i]) + } + } + pm.relay[key] = filtered + } + } } // maxPrefixLen caps prefix map entries. MeshCore path hops use 2–6 char @@ -7251,18 +7333,11 @@ func (s *PacketStore) InvalidateNodeCache() { // when the only matching prefix belonged to a listener). Empty pm.nonRelay // preserves the pre-#1290 behavior exactly (back-compat). func (pm *prefixMap) relayCandidates(hop string) []nodeInfo { - candidates := pm.m[strings.ToLower(hop)] - if len(pm.nonRelay) == 0 || len(candidates) == 0 { - return candidates - } - filtered := candidates[:0:0] - for i := range candidates { - if _, isListener := pm.nonRelay[strings.ToLower(candidates[i].PublicKey)]; isListener { - continue - } - filtered = append(filtered, candidates[i]) + h := strings.ToLower(hop) + if filtered, ok := pm.relay[h]; ok { + return filtered } - return filtered + return pm.m[h] } func (pm *prefixMap) resolve(hop string) *nodeInfo { diff --git a/cmd/server/subpaths_blacklist_2184_test.go b/cmd/server/subpaths_blacklist_2184_test.go new file mode 100644 index 000000000..2ff88304a --- /dev/null +++ b/cmd/server/subpaths_blacklist_2184_test.go @@ -0,0 +1,194 @@ +package main + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/gorilla/mux" +) + +// Issue #2184: nodeBlacklist and hiddenNamePrefixes were not applied to Route +// Patterns. The seeded path ["aa","bb"] resolves its first hop to TestRepeater +// (aabbccdd11223344), so the subpath "TestRepeater → bb" is listed. + +const seededRepeaterKey = "aabbccdd11223344" + +func subpathPaths(t *testing.T, body []byte) []string { + t.Helper() + var resp struct { + Subpaths []struct { + Path string `json:"path"` + } `json:"subpaths"` + } + if err := json.Unmarshal(body, &resp); err != nil { + t.Fatalf("unmarshal: %v body=%s", err, body) + } + out := make([]string, 0, len(resp.Subpaths)) + for _, sp := range resp.Subpaths { + out = append(out, sp.Path) + } + return out +} + +func bulkSubpathPaths(t *testing.T, body []byte) []string { + t.Helper() + var resp struct { + Results []struct { + Subpaths []struct { + Path string `json:"path"` + } `json:"subpaths"` + } `json:"results"` + } + if err := json.Unmarshal(body, &resp); err != nil { + t.Fatalf("unmarshal: %v body=%s", err, body) + } + var out []string + for _, r := range resp.Results { + for _, sp := range r.Subpaths { + out = append(out, sp.Path) + } + } + return out +} + +func anyContains(paths []string, name string) bool { + for _, p := range paths { + if strings.Contains(p, name) { + return true + } + } + return false +} + +func getSubpathBody2184(t *testing.T, srv *Server, url string) (int, []byte) { + t.Helper() + req := httptest.NewRequest("GET", url, nil) + w := httptest.NewRecorder() + srv.router.ServeHTTP(w, req) + return w.Code, w.Body.Bytes() +} + +func TestSubpathsBlacklist2184(t *testing.T) { + cases := []struct { + name string + blacklist []string + hidden []string + wantShown bool + }{ + {"no filter lists the repeater", nil, nil, true}, + {"blacklisted key drops it", []string{seededRepeaterKey}, nil, false}, + {"blacklisted key in capitals drops it", []string{strings.ToUpper(seededRepeaterKey)}, nil, false}, + {"hidden name prefix drops it", nil, []string{"TestRep"}, false}, + // Another key with the same 1-byte prefix: the displayed node is not + // the blacklisted one, so the subpath stays. + {"same-prefix key keeps it", []string{"aa00000000000000"}, nil, true}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + srv, _ := setupTestServer(t) + if c.blacklist != nil { + srv.cfg.NodeBlacklist = c.blacklist + srv.cfg.SetNodeBlacklist(c.blacklist) + } + if c.hidden != nil { + srv.cfg.SetHiddenNamePrefixes(c.hidden) + } + + code, body := getSubpathBody2184(t, srv, "/api/analytics/subpaths?minLen=2&maxLen=8") + if code != http.StatusOK { + t.Fatalf("subpaths status=%d body=%s", code, body) + } + if got := anyContains(subpathPaths(t, body), "TestRepeater"); got != c.wantShown { + t.Errorf("subpaths: TestRepeater shown=%v, want %v (%s)", got, c.wantShown, body) + } + + code, body = getSubpathBody2184(t, srv, "/api/analytics/subpaths-bulk?groups=2-2:50,3-8:50") + if code != http.StatusOK { + t.Fatalf("subpaths-bulk status=%d body=%s", code, body) + } + if got := anyContains(bulkSubpathPaths(t, body), "TestRepeater"); got != c.wantShown { + t.Errorf("subpaths-bulk: TestRepeater shown=%v, want %v (%s)", got, c.wantShown, body) + } + + code, body = getSubpathBody2184(t, srv, "/api/analytics/subpath-detail?hops=aa,bb") + wantCode := http.StatusOK + if !c.wantShown { + wantCode = http.StatusNotFound + } + if code != wantCode { + t.Errorf("subpath-detail status=%d, want %d (%s)", code, wantCode, body) + } + }) + } +} + +// The filter must not change the store's cached response: a later request, +// or one after the blacklist is cleared, still sees the full list. +func TestSubpathsBlacklistLeavesCacheIntact2184(t *testing.T) { + srv, _ := setupTestServer(t) + srv.cfg.NodeBlacklist = []string{seededRepeaterKey} + srv.cfg.SetNodeBlacklist(srv.cfg.NodeBlacklist) + if _, body := getSubpathBody2184(t, srv, "/api/analytics/subpaths?minLen=2&maxLen=8"); anyContains(subpathPaths(t, body), "TestRepeater") { + t.Fatalf("blacklisted repeater still listed: %s", body) + } + + srv.cfg.NodeBlacklist = nil + srv.cfg.SetNodeBlacklist(nil) + if _, body := getSubpathBody2184(t, srv, "/api/analytics/subpaths?minLen=2&maxLen=8"); !anyContains(subpathPaths(t, body), "TestRepeater") { + t.Errorf("after clearing the blacklist the repeater is missing: the filter mutated the cached response (%s)", body) + } +} + +// setupSubpathServer2184 is setupTestServer with a SQL statement run on the +// seeded DB before the store loads, so the store sees the changed rows. +func setupSubpathServer2184(t *testing.T, stmt string) *Server { + t.Helper() + db := setupTestDB(t) + seedTestData(t, db) + if _, err := db.conn.Exec(stmt); err != nil { + t.Fatalf("exec %q: %v", stmt, err) + } + srv := NewServer(db, &Config{Port: 3000}, NewHub()) + store := NewPacketStore(db, nil) + if err := store.Load(); err != nil { + t.Fatalf("store.Load failed: %v", err) + } + if !store.WaitIndexesReady(5 * time.Second) { + t.Fatalf("background indexes never became ready") + } + srv.store = store + srv.router = mux.NewRouter() + srv.RegisterRoutes(srv.router) + return srv +} + +// A blacklisted node without a name (the firmware allows one) has no name to +// match, so the detail must check the resolved key. +func TestSubpathDetailBlacklistUnnamedNode2184(t *testing.T) { + srv := setupSubpathServer2184(t, `UPDATE nodes SET name = '' WHERE public_key = '`+seededRepeaterKey+`'`) + srv.cfg.NodeBlacklist = []string{seededRepeaterKey} + srv.cfg.SetNodeBlacklist(srv.cfg.NodeBlacklist) + + code, body := getSubpathBody2184(t, srv, "/api/analytics/subpath-detail?hops=aa,bb") + if code != http.StatusNotFound { + t.Errorf("subpath-detail status=%d, want 404 (%s)", code, body) + } +} + +// Another node that only shares the blacklisted node's name does not hide the +// detail of a path through the seeded repeater. +func TestSubpathDetailBlacklistSameNameOtherNode2184(t *testing.T) { + const other = "ff99887766554433" + srv := setupSubpathServer2184(t, `INSERT INTO nodes (public_key, name, role) VALUES ('`+other+`', 'TestRepeater', 'repeater')`) + srv.cfg.NodeBlacklist = []string{other} + srv.cfg.SetNodeBlacklist(srv.cfg.NodeBlacklist) + + code, body := getSubpathBody2184(t, srv, "/api/analytics/subpath-detail?hops=aa,bb") + if code != http.StatusOK { + t.Errorf("subpath-detail status=%d, want 200 (%s)", code, body) + } +} diff --git a/cmd/server/tx_rewrite_feed.go b/cmd/server/tx_rewrite_feed.go new file mode 100644 index 000000000..1ac8bca12 --- /dev/null +++ b/cmd/server/tx_rewrite_feed.go @@ -0,0 +1,118 @@ +package main + +import ( + "database/sql" + "log" + "sync" +) + +// tx_rewrite_feed (#2107) is the ingestor's change feed for transmissions it +// rewrote in place: a GRP_TXT it can now decrypt, a transport scope it can now +// name (cmd/ingestor/key_backfill.go). IngestNewFromDB only ever looks at +// id > maxTxID, so without this feed a rewritten row stayed stale in memory +// until the next restart. +// +// Same shape as advert_route_evidence: the table is optional (an older +// ingestor never creates it), so absence is re-probed rather than cached, and +// the server tails it with its own cursor captured before Load. A feed row +// carries only the transmission id; the current values are read from +// transmissions, so replaying a row is harmless and the newest write wins. + +// txRewriteFeedPresent reports whether the ingestor has created the feed. +func (db *DB) txRewriteFeedPresent() bool { + if db == nil || db.conn == nil { + return false + } + if db.txRewriteFeedTable.Load() { + return true + } + var exists int + if err := db.conn.QueryRow(`SELECT 1 FROM sqlite_master WHERE type='table' AND name='tx_rewrite_feed'`).Scan(&exists); err != nil { + return false + } + db.txRewriteFeedTable.Store(true) + return true +} + +// pollTxRewrites applies up to limit feed rows to the in-memory store. A +// transmission the store does not hold (evicted, or outside the loaded +// window) is skipped: if it is ever loaded again it is read fresh from the DB. +// A failed read leaves the cursor where it was, so the next poll retries. +func (s *PacketStore) pollTxRewrites(limit int) error { + s.txRewriteMu.Lock() + defer s.txRewriteMu.Unlock() + if !s.db.txRewriteFeedPresent() { + return nil + } + if limit <= 0 || limit > 500 { + limit = 500 + } + scopeCol := "NULL" + if s.db.hasScopeName { + scopeCol = "t.scope_name" + } + rows, err := s.db.conn.Query(` + SELECT f.id, f.tx_id, t.decoded_json, `+scopeCol+` + FROM tx_rewrite_feed f LEFT JOIN transmissions t ON t.id = f.tx_id + WHERE f.id > ? ORDER BY f.id LIMIT ?`, s.txRewriteCursor, limit) + if err != nil { + return err + } + type rewrite struct { + id int64 + txID int + decoded sql.NullString + scope sql.NullString + } + var batch []rewrite + for rows.Next() { + var r rewrite + if err := rows.Scan(&r.id, &r.txID, &r.decoded, &r.scope); err != nil { + rows.Close() + return err + } + batch = append(batch, r) + } + err = rows.Err() + rows.Close() + if err != nil || len(batch) == 0 { + return err + } + + channelChanged := false + s.mu.Lock() + for _, r := range batch { + s.txRewriteCursor = r.id + tx := s.byTxID[r.txID] + if tx == nil { + continue + } + if r.decoded.Valid && r.decoded.String != tx.DecodedJSON { + tx.DecodedJSON = r.decoded.String + // ParsedDecoded caches behind a sync.Once; reset both under the + // write lock so the next reader parses the new JSON. + tx.decodedOnce = sync.Once{} + tx.parsedDecoded = nil + s.trackedBytes += rechargeTx(tx) + if tx.PayloadType != nil && *tx.PayloadType == PayloadGRP_TXT { + channelChanged = true + } + } + if s.db.hasScopeName && r.scope.Valid && (tx.ScopeName == nil || *tx.ScopeName != r.scope.String) { + v := r.scope.String + tx.ScopeName = &v + } + } + s.mu.Unlock() + + if channelChanged { + s.invalidateCachesFor(cacheInvalidation{hasChannelData: true}) + } + return nil +} + +func (s *PacketStore) refreshTxRewrites() { + if err := s.pollTxRewrites(500); err != nil { + log.Printf("[store] tx rewrite feed poll: %v", err) + } +} diff --git a/cmd/server/tx_rewrite_feed_test.go b/cmd/server/tx_rewrite_feed_test.go new file mode 100644 index 000000000..2b07a38cc --- /dev/null +++ b/cmd/server/tx_rewrite_feed_test.go @@ -0,0 +1,165 @@ +package main + +import ( + "database/sql" + "testing" + "time" +) + +// txRewriteFixture is the ingestor's durable output for one GRP_TXT stored +// before its channel key was known: still encrypted, scope unmatched. The +// returned writer plays the ingestor's key backfill. +func txRewriteFixture(t *testing.T, withFeed bool) (*DB, *sql.DB) { + t.Helper() + path := createTestDBWithResolvedPath(t, 1, []string{"fixture-relay"}) + w, err := sql.Open("sqlite3", path+"?_journal_mode=WAL") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { w.Close() }) + stmts := []string{ + `ALTER TABLE transmissions ADD COLUMN scope_name TEXT`, + `ALTER TABLE transmissions ADD COLUMN channel_hash TEXT`, + } + if withFeed { + stmts = append(stmts, txRewriteFeedDDL) + } + for _, stmt := range stmts { + if _, err := w.Exec(stmt); err != nil { + t.Fatal(err) + } + } + now := time.Now().UTC().Add(-time.Minute).Format(time.RFC3339) + if _, err := w.Exec(`UPDATE transmissions SET payload_type=5, route_type=0, first_seen=?, scope_name='', + channel_hash='enc_AB', decoded_json='{"type":"GRP_TXT","channelHashHex":"AB"}'; UPDATE observations SET timestamp=?`, now, now); err != nil { + t.Fatal(err) + } + db, err := OpenDB(path) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { db.conn.Close() }) + return db, w +} + +// Mirrors internal/dbschema/key_backfill.go; the server never creates it. +const txRewriteFeedDDL = `CREATE TABLE tx_rewrite_feed (id INTEGER PRIMARY KEY AUTOINCREMENT, tx_id INTEGER NOT NULL, created_at INTEGER NOT NULL DEFAULT 0)` + +// backfillRewrite does what the ingestor's commit does: rewrite the row and +// append its feed entry in one transaction. +func backfillRewrite(t *testing.T, w *sql.DB, decoded, scope string) { + t.Helper() + if _, err := w.Exec(`BEGIN; UPDATE transmissions SET decoded_json=?, channel_hash='#saar', scope_name=? WHERE id=1; + INSERT INTO tx_rewrite_feed(tx_id) VALUES(1); COMMIT`, decoded, scope); err != nil { + t.Fatal(err) + } +} + +func TestTxRewriteFeedReachesStoreWithoutRestart(t *testing.T) { + db, w := txRewriteFixture(t, true) + s := NewPacketStore(db, &PacketStoreConfig{}) + if err := s.Load(); err != nil { + t.Fatal(err) + } + tx := s.byTxID[1] + if tx == nil { + t.Fatal("fixture transmission not loaded") + } + // Parse once before the rewrite: the cached map must not survive it. + if got := tx.ParsedDecoded()["type"]; got != "GRP_TXT" { + t.Fatalf("fixture decoded type %v", got) + } + + backfillRewrite(t, w, `{"type":"CHAN","channel":"#saar","text":"Alice: moin"}`, "#saar") + s.IngestNewObservations(0, 100) // the server's normal poll path + + if got := tx.ParsedDecoded()["channel"]; got != "#saar" { + t.Fatalf("in-memory decoded still stale after poll: channel=%v json=%s", got, tx.DecodedJSON) + } + if tx.ScopeName == nil || *tx.ScopeName != "#saar" { + t.Fatalf("in-memory scope_name not refreshed: %v", tx.ScopeName) + } + if s.txRewriteCursor != 1 { + t.Fatalf("cursor=%d, want 1", s.txRewriteCursor) + } +} + +// Rows committed before Load are already in what Load read; the watermark +// keeps them from being re-applied (harmless, but wasted work at scale). +func TestTxRewriteFeedCursorStartsAtLoadWatermark(t *testing.T) { + db, w := txRewriteFixture(t, true) + backfillRewrite(t, w, `{"type":"CHAN","channel":"#saar"}`, "#saar") + s := NewPacketStore(db, &PacketStoreConfig{}) + if s.txRewriteCursor != 1 { + t.Fatalf("cursor=%d before Load, want the feed's MAX(id)=1", s.txRewriteCursor) + } +} + +// An older ingestor never creates the feed. The server must not error, and +// must start tailing as soon as a newer ingestor creates it - no restart. +func TestTxRewriteFeedAbsentThenCreated(t *testing.T) { + db, w := txRewriteFixture(t, false) + s := NewPacketStore(db, &PacketStoreConfig{}) + if err := s.Load(); err != nil { + t.Fatal(err) + } + if err := s.pollTxRewrites(500); err != nil { + t.Fatalf("absent feed must be a no-op, got %v", err) + } + if _, err := w.Exec(txRewriteFeedDDL); err != nil { + t.Fatal(err) + } + backfillRewrite(t, w, `{"type":"CHAN","channel":"#saar"}`, "#saar") + if err := s.pollTxRewrites(500); err != nil { + t.Fatal(err) + } + if got := s.byTxID[1].ParsedDecoded()["channel"]; got != "#saar" { + t.Fatalf("feed created after startup was not picked up: %v", got) + } +} + +func TestTxRewriteFeedFailedReadKeepsCursorAndRetries(t *testing.T) { + db, w := txRewriteFixture(t, true) + s := NewPacketStore(db, &PacketStoreConfig{}) + if err := s.Load(); err != nil { + t.Fatal(err) + } + backfillRewrite(t, w, `{"type":"CHAN","channel":"#saar"}`, "#saar") + if _, err := w.Exec(`ALTER TABLE tx_rewrite_feed RENAME TO feed_unavailable`); err != nil { + t.Fatal(err) + } + if err := s.pollTxRewrites(500); err == nil { + t.Fatal("expected failed feed read") + } + if s.txRewriteCursor != 0 { + t.Fatal("failed read advanced the cursor") + } + if _, err := w.Exec(`ALTER TABLE feed_unavailable RENAME TO tx_rewrite_feed`); err != nil { + t.Fatal(err) + } + if err := s.pollTxRewrites(500); err != nil { + t.Fatal(err) + } + if got := s.byTxID[1].ParsedDecoded()["channel"]; got != "#saar" { + t.Fatalf("retry did not apply the rewrite: %v", got) + } +} + +// A feed row for a transmission the store does not hold advances the cursor +// and is otherwise ignored; the row is read fresh if it is ever loaded. +func TestTxRewriteFeedSkipsUnloadedTransmission(t *testing.T) { + db, w := txRewriteFixture(t, true) + s := NewPacketStore(db, &PacketStoreConfig{}) + if err := s.Load(); err != nil { + t.Fatal(err) + } + if _, err := w.Exec(`INSERT INTO tx_rewrite_feed(tx_id) VALUES(999)`); err != nil { + t.Fatal(err) + } + if err := s.pollTxRewrites(500); err != nil { + t.Fatal(err) + } + if s.byTxID[999] != nil || s.txRewriteCursor != 1 { + t.Fatalf("unloaded tx: present=%v cursor=%d", s.byTxID[999] != nil, s.txRewriteCursor) + } +} diff --git a/config.example.json b/config.example.json index daa676f14..74809dee6 100644 --- a/config.example.json +++ b/config.example.json @@ -369,6 +369,8 @@ "criticalMv": 3000, "_comment": "Voltage cutoffs (millivolts) for the per-node battery trend chart on /node-analytics. Latest sample below lowMv shows the node as ⚠️ Low; below criticalMv shows 🪫 Critical. Both default to 3300 / 3000 if omitted. Source data: observer_metrics.battery_mv populated from observer status messages; only nodes that are themselves observers (matching pubkey ↔ observer id) yield a series. Issue #663." }, + "dutyCycleWarnPct": 10, + "_comment_dutyCycleWarnPct": "TX airtime share (percent) over the last hour at which RF Health and the observer list flag an observer. Default 10, the EU 868 MHz g3 sub-band limit. Repeater firmware defaults to a 50% budget (airtime_factor 1.0), so the device does not enforce this itself. Issue #661.", "_comment_mqttSources": "Each source connects to an MQTT broker. Supported schemes: mqtt:// (plain TCP), mqtts:// (TLS), ws:// (WebSocket), wss:// (WebSocket TLS). topics: what to subscribe to. iataFilter: only ingest packets from these regions (optional). region: default IATA region for this source — used when packet/topic doesn't specify one (optional, priority: payload > topic > this field).", "compression": { "gzip": false, diff --git a/docker/entrypoint-go.sh b/docker/entrypoint-go.sh index 9f47efc15..f2f5c1e5e 100644 --- a/docker/entrypoint-go.sh +++ b/docker/entrypoint-go.sh @@ -14,12 +14,15 @@ if [ -f /app/data/theme.json ]; then ln -sf /app/data/theme.json /app/theme.json fi -# Source .env from data volume if present (works with any launch method) -if [ -f /app/data/.env ]; then - set -a - . /app/data/.env - set +a -fi +# Which user runs the two Go services, and what they are handed. supervisord +# itself stays root so that mosquitto and caddy can still bind their ports. +# Unprivileged operation is opt-in via RUN_AS_UID, taken from the container's +# environment before .env is read; .env is then read (with the services +# unprivileged, only if root owns it) and the data directory handed over. The +# order matters and lives in one function: see docker/select-service-user.sh, +# and "Running the Go services unprivileged" in docs/deployment.md. +. /select-service-user.sh +prepare_service_user || exit 1 SUPERVISORD_CONF="/etc/supervisor/conf.d/supervisord.conf" if [ "${DISABLE_MOSQUITTO:-false}" = "true" ] && [ "${DISABLE_CADDY:-false}" = "true" ]; then @@ -34,4 +37,6 @@ elif [ "${DISABLE_CADDY:-false}" = "true" ]; then SUPERVISORD_CONF="/etc/supervisor/conf.d/supervisord-no-caddy.conf" fi +echo "[entrypoint] Go services run as $CORESCOPE_SERVICE_USER (uid=$(id -u "$CORESCOPE_SERVICE_USER") gid=$(id -g "$CORESCOPE_SERVICE_USER"))" + exec /usr/bin/supervisord -c "$SUPERVISORD_CONF" diff --git a/docker/select-service-user.sh b/docker/select-service-user.sh new file mode 100644 index 000000000..3a00ba64c --- /dev/null +++ b/docker/select-service-user.sh @@ -0,0 +1,302 @@ +#!/bin/sh +# select-service-user.sh — decides which user the two Go services run as, and +# hands over the paths they have to write. +# +# Sourced by docker/entrypoint-go.sh, which calls prepare_service_user and +# nothing else from here: the order of the steps (resolve the account, read +# .env, hand over the data directory) is part of what keeps .env from choosing +# the account, so it lives in one function and the test drives that function. +# docker/test-select-service-user.sh replaces getent/adduser/addgroup/deluser/ +# delgroup/id/chown/find/stat with stubs: keep every external command here +# stubbable, keep account resolution free of filesystem writes, and send +# human-readable output to stderr so stdout carries only the account name. +# +# Which user the services run as: +# +# 1. RUN_AS_UID / RUN_AS_GID, when set in the container's environment +# 2. root — what every earlier release did +# +# Both are read before /app/data/.env and never from it: the data directory is +# writable by the service account, and a setting there could be removed by the +# very process it restricts. +# +# There is deliberately no third rule inferring the uid from the mounted data +# directory. Such a rule switches user on upgrade for any deployment whose +# data directory is not root-owned — the ordinary `mkdir data` bind mount — +# while meshcore.db and its -wal/-shm, users.db and the backups inside it stay +# root-owned from the releases that ran as root, leaving the services unable to +# write them. Unprivileged operation is opt-in for that reason. + +# The account baked into the image, at uid 1000. +SERVICE_ACCOUNT=corescope +SERVICE_APP_DIR=/app +SERVICE_DATA_DIR=/app/data +SERVICE_ENV_FILE=/app/data/.env +SERVICE_MOUNTINFO=/proc/self/mountinfo + +svc_log() { + echo "[entrypoint] $*" >&2 +} + +# prepare_service_user +# +# The start-up sequence, in the only order that is safe: +# +# 1. resolve the account, from RUN_AS_UID/RUN_AS_GID in the container's +# environment; +# 2. read /app/data/.env (see load_env_file for when it is trusted); +# 3. hand the data directory to the account, when it is not root. +# +# Returns non-zero, with the reason on stderr, when RUN_AS_UID was set and no +# account could be provided. On success exports CORESCOPE_SERVICE_USER. +# +# The resolved account is kept in the positional parameter, not a variable: the +# sourced .env runs in this shell, and a line in it such as `account=root` would +# otherwise change which account is handed the data directory and exported to +# supervisord. +prepare_service_user() { + local _account + _account=$(resolve_service_account) || return 1 + set -- "$_account" + unset _account + + load_env_file "$SERVICE_ENV_FILE" "$1" + if [ "$1" != root ]; then + claim_service_paths "$1" + fi + CORESCOPE_SERVICE_USER=$1 + export CORESCOPE_SERVICE_USER +} + +# resolve_service_account +# +# Echoes the name of the account the Go services should run as: `root`, or an +# account with uid RUN_AS_UID, created when the image has none at that uid. +# Returns non-zero when RUN_AS_UID was set and no such account could be +# provided — a deployment that asked for unprivileged services must not +# quietly get root ones instead. +resolve_service_account() { + local run_as_uid="${RUN_AS_UID:-}" + local run_as_gid="${RUN_AS_GID:-}" + local account group account_gid + + if [ -z "$run_as_uid" ]; then + svc_log "RUN_AS_UID is unset — the Go services run as root, as in earlier releases." + svc_log "To run them unprivileged, see \"Running the Go services unprivileged\" in docs/deployment.md." + echo root + return 0 + fi + + case "$run_as_uid" in + *[!0-9]*) + svc_log "RUN_AS_UID must be a number, got \"$run_as_uid\"" + return 1 + ;; + esac + : "${run_as_gid:=$run_as_uid}" + case "$run_as_gid" in + *[!0-9]*) + svc_log "RUN_AS_GID must be a number, got \"$run_as_gid\"" + return 1 + ;; + esac + + # Asking for uid 0 is asking for the old behaviour; say so rather than + # treating root as an account to hand files over to. + if [ "$run_as_uid" -eq 0 ]; then + svc_log "RUN_AS_UID=0 — the Go services run as root, as in earlier releases." + echo root + return 0 + fi + + account=$(getent passwd "$run_as_uid" 2>/dev/null | cut -d: -f1) + # The baked-in account in another group is rebuilt below rather than used + # as it is, so RUN_AS_UID=1000 RUN_AS_GID=1001 means gid 1001. An account the + # image needs for something else (caddy, mosquitto) keeps its own group. + if [ "$account" = "$SERVICE_ACCOUNT" ] && [ "$(id -g "$account")" != "$run_as_gid" ]; then + account="" + fi + if [ -z "$account" ]; then + # Nothing usable holds that uid. The baked-in account is not what was + # asked for: free its name, but only now that it is known to be in the + # way. + if getent passwd "$SERVICE_ACCOUNT" >/dev/null 2>&1; then + deluser "$SERVICE_ACCOUNT" >/dev/null 2>&1 || true + fi + if getent group "$SERVICE_ACCOUNT" >/dev/null 2>&1; then + delgroup "$SERVICE_ACCOUNT" >/dev/null 2>&1 || true + fi + group=$(getent group "$run_as_gid" 2>/dev/null | cut -d: -f1) + if [ -z "$group" ] && addgroup -S -g "$run_as_gid" "$SERVICE_ACCOUNT" 2>/dev/null; then + group="$SERVICE_ACCOUNT" + fi + # -H as in the Dockerfile: without it busybox adduser gives the existing + # /app to the account, mode 2755. + if [ -n "$group" ] && adduser -S -u "$run_as_uid" -G "$group" -H \ + -h "$SERVICE_APP_DIR" -s /sbin/nologin "$SERVICE_ACCOUNT" 2>/dev/null; then + account="$SERVICE_ACCOUNT" + fi + fi + + if [ -z "$account" ]; then + svc_log "no account available for uid=$run_as_uid gid=$run_as_gid" + svc_log "RUN_AS_UID was set explicitly — refusing to fall back to root" + return 1 + fi + + account_gid=$(id -g "$account") + if [ "$account_gid" != "$run_as_gid" ]; then + svc_log "note: uid $run_as_uid belongs to the existing account \"$account\"," \ + "whose primary group is $account_gid and not $run_as_gid" + fi + + echo "$account" +} + +# load_env_file +# +# Sources the data directory's .env with every variable exported, as the +# entrypoint always has. When the services run unprivileged that directory is +# theirs, so a .env they could have written or put there is not run as root: it +# is read only when it is a regular file, not a symlink, owned by root and +# writable by no one else. Running as root changes nothing — whoever could +# write the file could already run code as root. +# +# Local names are underscored: the file is sourced into this function. +load_env_file() { + local _env_file="$1" _env_account="$2" + local _uid_before="${RUN_AS_UID-}" _gid_before="${RUN_AS_GID-}" + local _uid_was="${RUN_AS_UID+set}" _gid_was="${RUN_AS_GID+set}" + + [ -f "$_env_file" ] || return 0 + if [ "$_env_account" != root ] && ! env_file_trusted "$_env_file"; then + svc_log "not reading $_env_file: the Go services run as $_env_account and can" + svc_log "write $SERVICE_DATA_DIR, and this file is run as root. Make it a regular" + svc_log "file owned by root and writable only by root to have it read." + return 0 + fi + + set -a + . "$_env_file" + set +a + + # The account was resolved before this file was read, so a RUN_AS_UID or + # RUN_AS_GID in it must not reach supervisord and the services either: put + # back what the container had, set or unset. + if [ "${RUN_AS_UID+set}" != "$_uid_was" ] || [ "${RUN_AS_UID-}" != "$_uid_before" ] || + [ "${RUN_AS_GID+set}" != "$_gid_was" ] || [ "${RUN_AS_GID-}" != "$_gid_before" ]; then + svc_log "ignoring RUN_AS_UID/RUN_AS_GID from $_env_file: set them on the container" + svc_log "(docker run -e, or environment: in compose), where the services cannot remove them." + if [ -n "$_uid_was" ]; then RUN_AS_UID=$_uid_before; export RUN_AS_UID; else unset RUN_AS_UID; fi + if [ -n "$_gid_was" ]; then RUN_AS_GID=$_gid_before; export RUN_AS_GID; else unset RUN_AS_GID; fi + fi +} + +# env_file_trusted +# +# True for a regular file, not a symlink, owned by root and writable by no one +# else. stat without -L describes a symlink itself, so that test is belt and +# braces. +env_file_trusted() { + local _st + [ -L "$1" ] && return 1 + _st=$(stat -c '%u %A' "$1" 2>/dev/null) || return 1 + case "$_st" in + 0\ -????w????|0\ -???????w?) return 1 ;; + 0\ -*) return 0 ;; + esac + return 1 +} + +# service_data_mounts +# +# Mount points inside the data directory, not counting the data directory +# itself, one per line, with the kernel's escapes undone (space, tab, backslash; +# \134 last, so that an escaped backslash followed by digits stays literal). A +# config.json bind-mounted on its own shows up here. +# +# Returns non-zero when a mount point contains a newline: it cannot be listed +# one per line, and the caller must then not hand anything over. +service_data_mounts() { + local _paths _tab + _tab=$(printf '\t') + [ -r "$SERVICE_MOUNTINFO" ] || return 0 + _paths=$(awk -v d="$SERVICE_DATA_DIR/" 'index($5, d) == 1 { print $5 }' "$SERVICE_MOUNTINFO") + case "$_paths" in + *'\012'*) return 1 ;; + esac + printf '%s\n' "$_paths" | sed "s/\\\\040/ /g; s/\\\\011/$_tab/g; s/\\\\134/\\\\/g" +} + +# glob_escape +# +# find -path treats [ ] * ? and \ in its argument as pattern characters; a mount +# point named with one of them must match itself and nothing else. +glob_escape() { + printf '%s\n' "$1" | sed 's/[][*?\\]/\\&/g' +} + +# claim_service_paths +# +# Gives the service account the data directory and what an earlier root-run +# container left in it. /app stays root's: nothing the services write lives +# there, and an operator may run its binaries as root with `docker exec`. +# +# Reached only when RUN_AS_UID named an unprivileged account, and that is what +# makes the hand-over consented to rather than a surprise. It runs on every +# start, so a file later created there as root, by `docker exec` say, is handed +# over at the next one. +# +# Left alone, with what is below them: +# - anything mounted into the data directory on its own, such as a +# config.json bind-mounted as a single file: that is the operator's file on +# the host, and its owner is theirs to choose; +# - .env, which the entrypoint reads as root and so must stay root's. +# +# chown -h re-owns a symlink itself rather than what it points at. A path the +# hand-over could not change — on NFS with root_squash, say — is logged: the +# services would otherwise fail to write it with nothing in the log saying why. +claim_service_paths() { + local account="$1" + local uid gid excluded p left count + + uid=$(id -u "$account") + gid=$(id -g "$account") + + if ! excluded=$(service_data_mounts); then + svc_log "WARNING: a mount point in $SERVICE_DATA_DIR has a newline in its name, so which" + svc_log "files are the operator's cannot be told; not handing $SERVICE_DATA_DIR over." + return 0 + fi + if [ -n "$excluded" ]; then + printf '%s\n' "$excluded" | while IFS= read -r p; do + svc_log "left alone: $p is mounted on its own, so it keeps the owner it has on the host" + done + fi + excluded="$SERVICE_ENV_FILE${excluded:+ +$excluded}" + + # find's prune expression, built in the positional parameters so that + # paths with spaces survive. + set -- + set -f + local IFS=' +' + for p in $excluded; do + [ $# -eq 0 ] || set -- "$@" -o + set -- "$@" -path "$(glob_escape "$p")" + done + unset IFS + set +f + + find "$SERVICE_DATA_DIR" \( "$@" \) -prune -o ! -user "$uid" \ + -exec chown -h "$uid:$gid" {} + 2>/dev/null + + left=$(find "$SERVICE_DATA_DIR" \( "$@" \) -prune -o ! -user "$uid" -print 2>/dev/null) + if [ -n "$left" ]; then + count=$(printf '%s\n' "$left" | wc -l) + svc_log "WARNING: $count path(s) in $SERVICE_DATA_DIR could not be given to uid $uid;" + svc_log "the Go services will fail to write them. First ones:" + printf '%s\n' "$left" | head -n 5 | while IFS= read -r p; do svc_log " $p"; done + fi +} diff --git a/docker/supervisord-go-no-caddy.conf b/docker/supervisord-go-no-caddy.conf index 3ca0af00f..3b1047443 100644 --- a/docker/supervisord-go-no-caddy.conf +++ b/docker/supervisord-go-no-caddy.conf @@ -15,6 +15,7 @@ stderr_logfile=/dev/stderr stderr_logfile_maxbytes=0 [program:corescope-ingestor] +user=%(ENV_CORESCOPE_SERVICE_USER)s command=/app/corescope-ingestor -config /app/config.json directory=/app autostart=true @@ -29,6 +30,7 @@ stderr_logfile=/dev/stderr stderr_logfile_maxbytes=0 [program:corescope-server] +user=%(ENV_CORESCOPE_SERVICE_USER)s command=/app/corescope-server -config-dir /app -db /app/data/meshcore.db -public /app/public -port 3000 directory=/app autostart=true diff --git a/docker/supervisord-go-no-mosquitto-no-caddy.conf b/docker/supervisord-go-no-mosquitto-no-caddy.conf index 661d77196..8c8c98598 100644 --- a/docker/supervisord-go-no-mosquitto-no-caddy.conf +++ b/docker/supervisord-go-no-mosquitto-no-caddy.conf @@ -6,6 +6,7 @@ logfile_maxbytes=0 pidfile=/var/run/supervisord.pid [program:corescope-ingestor] +user=%(ENV_CORESCOPE_SERVICE_USER)s command=/app/corescope-ingestor -config /app/config.json directory=/app autostart=true @@ -20,6 +21,7 @@ stderr_logfile=/dev/stderr stderr_logfile_maxbytes=0 [program:corescope-server] +user=%(ENV_CORESCOPE_SERVICE_USER)s command=/app/corescope-server -config-dir /app -db /app/data/meshcore.db -public /app/public -port 3000 directory=/app autostart=true diff --git a/docker/supervisord-go-no-mosquitto.conf b/docker/supervisord-go-no-mosquitto.conf index fc98edb19..3c673f2ec 100644 --- a/docker/supervisord-go-no-mosquitto.conf +++ b/docker/supervisord-go-no-mosquitto.conf @@ -6,6 +6,7 @@ logfile_maxbytes=0 pidfile=/var/run/supervisord.pid [program:corescope-ingestor] +user=%(ENV_CORESCOPE_SERVICE_USER)s command=/app/corescope-ingestor -config /app/config.json directory=/app autostart=true @@ -20,6 +21,7 @@ stderr_logfile=/dev/stderr stderr_logfile_maxbytes=0 [program:corescope-server] +user=%(ENV_CORESCOPE_SERVICE_USER)s command=/app/corescope-server -config-dir /app -db /app/data/meshcore.db -public /app/public -port 3000 directory=/app autostart=true diff --git a/docker/supervisord-go.conf b/docker/supervisord-go.conf index 315b10ab6..18a8f4d70 100644 --- a/docker/supervisord-go.conf +++ b/docker/supervisord-go.conf @@ -15,6 +15,7 @@ stderr_logfile=/dev/stderr stderr_logfile_maxbytes=0 [program:corescope-ingestor] +user=%(ENV_CORESCOPE_SERVICE_USER)s command=/app/corescope-ingestor -config /app/config.json directory=/app autostart=true @@ -29,6 +30,7 @@ stderr_logfile=/dev/stderr stderr_logfile_maxbytes=0 [program:corescope-server] +user=%(ENV_CORESCOPE_SERVICE_USER)s command=/app/corescope-server -config-dir /app -db /app/data/meshcore.db -public /app/public -port 3000 directory=/app autostart=true diff --git a/docker/test-select-service-user.sh b/docker/test-select-service-user.sh new file mode 100755 index 000000000..7269bb4aa --- /dev/null +++ b/docker/test-select-service-user.sh @@ -0,0 +1,479 @@ +#!/usr/bin/env bash +# test-select-service-user.sh — unit tests for docker/select-service-user.sh, +# the uid selection the container entrypoint performs at start. +# +# Every branch is exercised without docker and without root: getent, adduser, +# addgroup, deluser, delgroup, id, chown, find and stat are replaced by stubs +# on PATH, and the mount table is a fixture file. The account stubs read and +# write a passwd/group pair of files seeded with the accounts the real image has — verified against alpine:3.20 with +# mosquitto, supervisor and caddy installed, plus the Dockerfile's corescope — +# so an account created mid-run is visible to the lookups that follow it. chown +# and find record their arguments rather than touching the filesystem, which is +# what lets the ownership hand-over be asserted exactly. +# +# The driver calls prepare_service_user, the one function docker/entrypoint-go.sh +# calls: resolve the account, read .env, hand over the data directory. A static +# check at the end pins that the entrypoint calls it and nothing else from the +# script, so what gets tested is the sequence the container runs: a non-zero +# return must never be read as root, and .env cannot choose the account. +# +# Run: bash docker/test-select-service-user.sh +# Exits non-zero if any case fails. +set -u + +SCRIPT="$(cd "$(dirname "$0")" && pwd)/select-service-user.sh" +[ -f "$SCRIPT" ] || { echo "FAIL: $SCRIPT missing" >&2; exit 1; } + +# Accounts the image actually has. uid:gid differ for caddy and mosquitto, +# which is what makes the "uid already belongs to an account" branch testable. +IMAGE_PASSWD='root:0:0 +caddy:100:101 +mosquitto:101:102 +corescope:1000:1000' +IMAGE_GROUP='root:0 +caddy:101 +mosquitto:102 +corescope:1000' + +PASS=0 +FAIL=0 +CASE="" + +fail() { + FAIL=$((FAIL + 1)) + echo "FAIL: $CASE — $1" >&2 +} + +assert_eq() { + if [ "$2" = "$3" ]; then PASS=$((PASS + 1)); else + fail "$1 — expected '$2', got '$3'" + fi +} + +assert_has() { + case "$2" in + *"$3"*) PASS=$((PASS + 1)) ;; + *) fail "$1 — expected to find '$3' in: $2" ;; + esac +} + +assert_lacks() { + case "$2" in + *"$3"*) fail "$1 — did not expect '$3' in: $2" ;; + *) PASS=$((PASS + 1)) ;; + esac +} + +# ----- stub environment ------------------------------------------------------ + +make_fixture() { + FIX=$(mktemp -d) + printf '%s\n' "$IMAGE_PASSWD" > "$FIX/passwd" + printf '%s\n' "$IMAGE_GROUP" > "$FIX/group" + : > "$FIX/calls" + : > "$FIX/log" + printf '%s' "${FIX_MOUNTINFO-}" > "$FIX/mountinfo" + if [ -n "${FIX_ENV-}" ]; then + printf '%s\n' "$FIX_ENV" > "$FIX/env.real" + if [ "${FIX_ENV_SYMLINK:-0}" = 1 ]; then + ln -s "$FIX/env.real" "$FIX/env" + [ -L "$FIX/env" ] || { echo "FAIL: fixture could not create the .env symlink" >&2; exit 1; } + else + mv "$FIX/env.real" "$FIX/env" + fi + fi + mkdir -p "$FIX/bin" + + cat > "$FIX/bin/getent" <<'STUB' +#!/bin/sh +case "$1" in + passwd) + while IFS=: read -r name uid gid; do + [ -n "$name" ] || continue + if [ "$2" = "$name" ] || [ "$2" = "$uid" ]; then + echo "$name:x:$uid:$gid::/:/sbin/nologin"; exit 0 + fi + done < "$FIX/passwd" ;; + group) + while IFS=: read -r name gid; do + [ -n "$name" ] || continue + if [ "$2" = "$name" ] || [ "$2" = "$gid" ]; then + echo "$name:x:$gid:"; exit 0 + fi + done < "$FIX/group" ;; +esac +exit 2 +STUB + + cat > "$FIX/bin/adduser" <<'STUB' +#!/bin/sh +[ "${STUB_ADDUSER_FAIL:-0}" = 1 ] && exit 1 +uid=""; grp=""; name=""; flags="" +while [ $# -gt 0 ]; do + case "$1" in + -u) uid="$2"; shift 2 ;; + -G) grp="$2"; shift 2 ;; + -h|-s) shift 2 ;; + -S|-D|-H) flags="$flags $1"; shift ;; + *) name="$1"; shift ;; + esac +done +[ -n "$uid" ] && [ -n "$name" ] && [ -n "$grp" ] || exit 1 +# busybox refuses a uid that is already taken +awk -F: -v u="$uid" '$2==u{found=1} END{exit !found}' "$FIX/passwd" && exit 1 +gid=$(awk -F: -v g="$grp" '$1==g{print $2}' "$FIX/group") +[ -n "$gid" ] || exit 1 +echo "$name:$uid:$gid" >> "$FIX/passwd" +echo "adduser $name uid=$uid gid=$gid flags=$flags" >> "$FIX/calls" +STUB + + cat > "$FIX/bin/addgroup" <<'STUB' +#!/bin/sh +[ "${STUB_ADDGROUP_FAIL:-0}" = 1 ] && exit 1 +gid=""; name="" +while [ $# -gt 0 ]; do + case "$1" in + -g) gid="$2"; shift 2 ;; + -S|-D) shift ;; + *) name="$1"; shift ;; + esac +done +[ -n "$gid" ] && [ -n "$name" ] || exit 1 +awk -F: -v g="$gid" '$2==g{found=1} END{exit !found}' "$FIX/group" && exit 1 +echo "$name:$gid" >> "$FIX/group" +echo "addgroup $name gid=$gid" >> "$FIX/calls" +STUB + + cat > "$FIX/bin/deluser" <<'STUB' +#!/bin/sh +grep -v "^$1:" "$FIX/passwd" > "$FIX/passwd.new" || true +mv "$FIX/passwd.new" "$FIX/passwd" +echo "deluser $1" >> "$FIX/calls" +STUB + + cat > "$FIX/bin/delgroup" <<'STUB' +#!/bin/sh +grep -v "^$1:" "$FIX/group" > "$FIX/group.new" || true +mv "$FIX/group.new" "$FIX/group" +echo "delgroup $1" >> "$FIX/calls" +STUB + + cat > "$FIX/bin/id" <<'STUB' +#!/bin/sh +while IFS=: read -r name uid gid; do + if [ "$2" = "$name" ]; then + case "$1" in + -u) echo "$uid"; exit 0 ;; + -g) echo "$gid"; exit 0 ;; + esac + exit 2 + fi +done < "$FIX/passwd" +exit 1 +STUB + + cat > "$FIX/bin/chown" <<'STUB' +#!/bin/sh +echo "chown $*" >> "$FIX/calls" +STUB + + # The hand-over walk is recorded; the verification walk (-print) answers + # with STUB_LEFTOVER, the paths a failed chown would have left behind. + cat > "$FIX/bin/find" <<'STUB' +#!/bin/sh +echo "find $*" >> "$FIX/calls" +case " $* " in + *" -print "*) [ -n "${STUB_LEFTOVER-}" ] && printf '%s\n' "$STUB_LEFTOVER" ;; +esac +exit 0 +STUB + + # Answers STUB_STAT ("uid mode"), by default a uid 1000 file: the uid + # must never be inferred from that, and the .env check must refuse it. + # Any call is recorded so the test can prove when none happens. + cat > "$FIX/bin/stat" <<'STUB' +#!/bin/sh +echo "stat $*" >> "$FIX/calls" +echo "${STUB_STAT:-1000 -rw-r--r--}" +STUB + + chmod +x "$FIX"/bin/* +} + +# drive [VAR=VAL ...] — run prepare_service_user as docker/entrypoint-go.sh +# does and exit non-zero if it fails. env -i keeps the host environment (a +# RUN_AS_UID of its own, a real PATH) out of the case. FIX_ENV (the .env +# contents), FIX_ENV_SYMLINK and FIX_MOUNTINFO shape the fixture; a .env that +# sets CS_ENV_READ shows up as "env-read" in CALLS. OUT is the account the +# services will run as (CORESCOPE_SERVICE_USER); FINAL_ENV is RUN_AS_UID/GID as +# the services would see them. +drive() { + make_fixture + OUT=$(env -i PATH="$FIX/bin:/usr/bin:/bin" FIX="$FIX" SCRIPT="$SCRIPT" "$@" sh -c ' + . "$SCRIPT" + SERVICE_ENV_FILE="$FIX/env" + SERVICE_MOUNTINFO="$FIX/mountinfo" + prepare_service_user || exit 1 + [ -z "${CS_ENV_READ-}" ] || echo "env-read" >> "$FIX/calls" + echo "RUN_AS_UID=${RUN_AS_UID-} RUN_AS_GID=${RUN_AS_GID-}" > "$FIX/final-env" + echo "$CORESCOPE_SERVICE_USER" + ' 2>"$FIX/log") + RC=$? + LOG=$(cat "$FIX/log") + CALLS=$(cat "$FIX/calls") + FINAL_ENV=$(cat "$FIX/final-env" 2>/dev/null) +} + +# ----- 1. RUN_AS_UID unset: root, exactly as earlier releases ---------------- +CASE="RUN_AS_UID unset" +drive +assert_eq "account" "root" "$OUT" +assert_eq "exit status" 0 "$RC" +assert_eq "nothing was chowned" "" "$CALLS" +assert_has "log explains the default" "$LOG" "RUN_AS_UID is unset" + +# ----- 2. the dropped inferred branch must not come back --------------------- +# The data directory is reported as owned by uid 1000 (`mkdir data` + bind +# mount, the ordinary setup). Earlier revisions of this change ran the services +# as that uid, which broke every upgrade whose data directory held root-owned +# database files. Staying root here is the point of the whole arrangement. +CASE="non-root data dir, RUN_AS_UID unset" +drive +assert_eq "account" "root" "$OUT" +assert_lacks "data dir owner never consulted" "$CALLS" "stat" +assert_eq "nothing was chowned" "" "$CALLS" + +# ----- 3. RUN_AS_UID=1000 reuses the account baked into the image ------------ +CASE="RUN_AS_UID=1000" +drive RUN_AS_UID=1000 RUN_AS_GID=1000 +assert_eq "account" "corescope" "$OUT" +assert_eq "exit status" 0 "$RC" +assert_lacks "/app stays root's" "$CALLS" "chown 1000:1000 /app" +assert_has "data dir handed over, .env left out, symlinks not followed" "$CALLS" \ + "find /app/data ( -path $FIX/env ) -prune -o ! -user 1000 -exec chown -h 1000:1000 {} +" +assert_has "hand-over checked afterwards" "$CALLS" \ + "find /app/data ( -path $FIX/env ) -prune -o ! -user 1000 -print" +assert_lacks "nothing reported left over" "$LOG" "WARNING" +assert_lacks "no account created" "$CALLS" "adduser" +assert_lacks "baked-in account kept" "$CALLS" "deluser" + +# ----- 4. RUN_AS_GID defaults to RUN_AS_UID --------------------------------- +CASE="RUN_AS_UID=1000, RUN_AS_GID unset" +drive RUN_AS_UID=1000 +assert_eq "account" "corescope" "$OUT" +assert_has "gid defaulted to the uid" "$CALLS" "chown -h 1000:1000" +assert_lacks "no group mismatch reported" "$LOG" "note:" + +# ----- 5. a uid nothing holds: the account is recreated at it ---------------- +CASE="RUN_AS_UID=5000" +drive RUN_AS_UID=5000 +assert_eq "account" "corescope" "$OUT" +assert_eq "exit status" 0 "$RC" +assert_has "baked-in account freed" "$CALLS" "deluser corescope" +assert_has "baked-in group freed" "$CALLS" "delgroup corescope" +assert_has "group created at the gid" "$CALLS" "addgroup corescope gid=5000" +assert_has "account created at the uid" "$CALLS" "adduser corescope uid=5000 gid=5000" +assert_has "home directory not created or re-owned" "$CALLS" "flags= -S -H" +assert_has "data dir handed over" "$CALLS" "chown -h 5000:5000" + +# ----- 6. a separate RUN_AS_GID --------------------------------------------- +CASE="RUN_AS_UID=5000, RUN_AS_GID=6000" +drive RUN_AS_UID=5000 RUN_AS_GID=6000 +assert_eq "account" "corescope" "$OUT" +assert_has "group created at the gid" "$CALLS" "addgroup corescope gid=6000" +assert_has "account created with that group" "$CALLS" "adduser corescope uid=5000 gid=6000" +assert_has "data dir handed over" "$CALLS" "chown -h 5000:6000" + +# ----- 7. a uid an image account already holds ------------------------------ +# caddy is uid 100 but gid 101, so the requested gid cannot be honoured; the +# account is used as it is and the mismatch is reported rather than hidden. +CASE="RUN_AS_UID=100 (caddy)" +drive RUN_AS_UID=100 +assert_eq "account" "caddy" "$OUT" +assert_eq "exit status" 0 "$RC" +assert_lacks "existing account untouched" "$CALLS" "adduser" +assert_has "mismatch reported" "$LOG" "primary group is 101 and not 100" +assert_has "account's own gid used" "$CALLS" "chown -h 100:101" + +# ----- 8. RUN_AS_UID=0 is a request for the old behaviour ------------------- +CASE="RUN_AS_UID=0" +drive RUN_AS_UID=0 +assert_eq "account" "root" "$OUT" +assert_eq "exit status" 0 "$RC" +assert_eq "nothing was chowned" "" "$CALLS" + +# ----- 9. non-numeric input is rejected ------------------------------------- +CASE="RUN_AS_UID=nobody" +drive RUN_AS_UID=nobody +assert_eq "exit status" 1 "$RC" +assert_eq "no account on stdout" "" "$OUT" +assert_has "reason given" "$LOG" "RUN_AS_UID must be a number" + +CASE="RUN_AS_GID=wheel" +drive RUN_AS_UID=5000 RUN_AS_GID=wheel +assert_eq "exit status" 1 "$RC" +assert_eq "no account on stdout" "" "$OUT" +assert_has "reason given" "$LOG" "RUN_AS_GID must be a number" + +# ----- 10. an explicit request that cannot be met must not become root ------ +CASE="RUN_AS_UID=5000, adduser fails" +drive RUN_AS_UID=5000 STUB_ADDUSER_FAIL=1 +assert_eq "exit status" 1 "$RC" +assert_eq "no account on stdout" "" "$OUT" +assert_has "refusal is explicit" "$LOG" "refusing to fall back to root" +assert_lacks "nothing handed over" "$CALLS" "chown" + +CASE="RUN_AS_UID=5000, addgroup fails" +drive RUN_AS_UID=5000 STUB_ADDGROUP_FAIL=1 +assert_eq "exit status" 1 "$RC" +assert_eq "no account on stdout" "" "$OUT" +assert_has "refusal is explicit" "$LOG" "refusing to fall back to root" + +# ----- 11. RUN_AS_GID is honoured for the baked-in account ----------------- +# uid 1000 is the image's own corescope account, gid 1000. Asked for gid 1001, +# it is rebuilt in that group rather than used with the wrong one. +CASE="RUN_AS_UID=1000, RUN_AS_GID=1001" +drive RUN_AS_UID=1000 RUN_AS_GID=1001 +assert_eq "account" "corescope" "$OUT" +assert_has "baked-in account rebuilt" "$CALLS" "deluser corescope" +assert_has "group created at the gid" "$CALLS" "addgroup corescope gid=1001" +assert_has "account recreated in it" "$CALLS" "adduser corescope uid=1000 gid=1001" +assert_has "data dir handed to that gid" "$CALLS" "chown -h 1000:1001" +assert_lacks "no group mismatch left to report" "$LOG" "note:" + +# ----- 12. .env as root: read as before, whoever owns it ------------------- +CASE=".env, RUN_AS_UID unset" +FIX_ENV='CS_ENV_READ=1' drive +assert_eq "account" "root" "$OUT" +assert_has ".env read" "$CALLS" "env-read" +assert_lacks "owner not checked as root" "$CALLS" "stat" + +# ----- 13. .env, unprivileged: only a root-owned, root-only-writable file ---- +CASE=".env root-owned 0644, RUN_AS_UID=1000" +FIX_ENV='CS_ENV_READ=1' drive RUN_AS_UID=1000 STUB_STAT='0 -rw-r--r--' +assert_eq "account" "corescope" "$OUT" +assert_has ".env read" "$CALLS" "env-read" + +CASE=".env root-owned 0600, RUN_AS_UID=1000" +FIX_ENV='CS_ENV_READ=1' drive RUN_AS_UID=1000 STUB_STAT='0 -rw-------' +assert_has ".env read" "$CALLS" "env-read" + +CASE=".env owned by the service uid, RUN_AS_UID=1000" +FIX_ENV='CS_ENV_READ=1' drive RUN_AS_UID=1000 STUB_STAT='1000 -rw-r--r--' +assert_eq "account" "corescope" "$OUT" +assert_lacks ".env not read" "$CALLS" "env-read" +assert_has "refusal explained" "$LOG" "not reading $FIX/env" + +CASE=".env root-owned but group-writable, RUN_AS_UID=1000" +FIX_ENV='CS_ENV_READ=1' drive RUN_AS_UID=1000 STUB_STAT='0 -rw-rw-r--' +assert_lacks ".env not read" "$CALLS" "env-read" + +CASE=".env root-owned but world-writable, RUN_AS_UID=1000" +FIX_ENV='CS_ENV_READ=1' drive RUN_AS_UID=1000 STUB_STAT='0 -rw-r--rw-' +assert_lacks ".env not read" "$CALLS" "env-read" + +CASE=".env a symlink, RUN_AS_UID=1000" +FIX_ENV='CS_ENV_READ=1' FIX_ENV_SYMLINK=1 drive RUN_AS_UID=1000 STUB_STAT='0 -rw-r--r--' +assert_lacks ".env not read" "$CALLS" "env-read" +assert_lacks "refused before its owner is asked" "$CALLS" "stat" + +# ----- 14. .env cannot choose the account ----------------------------------- +# A service that can delete .env could otherwise drop its own RUN_AS_UID, and +# one that can write it could set RUN_AS_UID= and be root after a restart. +CASE="RUN_AS_UID in .env, unset on the container" +FIX_ENV='RUN_AS_UID=1000' drive +assert_eq "account" "root" "$OUT" +assert_eq "nothing was chowned" "" "$CALLS" +assert_has "ignored, and said so" "$LOG" "ignoring RUN_AS_UID/RUN_AS_GID" +assert_eq "and not left exported to the services" "RUN_AS_UID= RUN_AS_GID=" "$FINAL_ENV" + +CASE="RUN_AS_UID= in a trusted .env, 1000 on the container" +FIX_ENV='RUN_AS_UID=' drive RUN_AS_UID=1000 STUB_STAT='0 -rw-r--r--' +assert_eq "account" "corescope" "$OUT" +assert_has "ignored, and said so" "$LOG" "ignoring RUN_AS_UID/RUN_AS_GID" +assert_eq "the container's value is put back" "RUN_AS_UID=1000 RUN_AS_GID=" "$FINAL_ENV" + +CASE="RUN_AS_UID=0 and RUN_AS_GID in a trusted .env, 1000 on the container" +FIX_ENV='RUN_AS_UID=0 +RUN_AS_GID=0' drive RUN_AS_UID=1000 RUN_AS_GID=1000 STUB_STAT='0 -rw-r--r--' +assert_eq "account" "corescope" "$OUT" +assert_eq "both put back" "RUN_AS_UID=1000 RUN_AS_GID=1000" "$FINAL_ENV" + +# The account is resolved before .env is read. If it were read first, a +# RUN_AS_UID there would make the resolver create an account at that uid. +CASE="order: .env read after the account is resolved" +FIX_ENV='RUN_AS_UID=5000' drive RUN_AS_UID=1000 STUB_STAT='0 -rw-r--r--' +assert_eq "account" "corescope" "$OUT" +assert_lacks "no account was created at the uid from .env" "$CALLS" "adduser" +assert_lacks "none was freed for it either" "$CALLS" "deluser" +assert_has "the data dir went to the container's uid" "$CALLS" "chown -h 1000:1000" + +# A trusted .env that assigns the names the entrypoint and these functions use +# for the account must not change who the services run as, or who is handed +# the data directory. +CASE=".env assigns the account's variable names" +FIX_ENV='account=root +service_account=root +_account=root +CORESCOPE_SERVICE_USER=root' drive RUN_AS_UID=1000 STUB_STAT='0 -rw-r--r--' +assert_eq "account" "corescope" "$OUT" +assert_has "the data dir was still handed over to it" "$CALLS" "chown -h 1000:1000" + +# ----- 15. a file mounted into the data directory keeps its host owner ----- +# The documented single-file config mount, plus one with a space in its path, +# as the kernel escapes it. The data directory's own mount is not excluded. +CASE="single-file mounts in the data directory" +FIX_MOUNTINFO='600 500 0:50 /srv/data /app/data rw,relatime - ext4 /dev/sda1 rw +601 600 0:50 /srv/config.json /app/data/config.json ro,relatime - ext4 /dev/sda1 rw +602 600 0:50 /srv/my\040keys /app/data/my\040keys rw,relatime - ext4 /dev/sda1 rw +603 500 0:51 / /data/caddy rw,relatime - ext4 /dev/sda1 rw +' drive RUN_AS_UID=1000 +assert_has "mounts pruned along with .env" "$CALLS" \ + "find /app/data ( -path $FIX/env -o -path /app/data/config.json -o -path /app/data/my keys ) -prune -o ! -user 1000 -exec chown -h 1000:1000 {} +" +assert_has "config.json reported as left alone" "$LOG" "left alone: /app/data/config.json" +assert_lacks "data dir itself still handed over" "$CALLS" "-path /app/data )" +assert_lacks "other mounts ignored" "$CALLS" "/data/caddy" + +# Names the kernel escapes (tab) or that find would read as patterns ([ * ?) +# must exclude exactly that path. +CASE="mount points with a tab and glob characters" +FIX_MOUNTINFO='601 600 0:50 /srv/a /app/data/tab\011name rw,relatime - ext4 /dev/sda1 rw +602 600 0:50 /srv/b /app/data/b[1]*? rw,relatime - ext4 /dev/sda1 rw +' drive RUN_AS_UID=1000 +TAB=$(printf '\t') +assert_has "tab decoded" "$CALLS" "-path /app/data/tab${TAB}name" +assert_has "glob characters escaped" "$CALLS" '-path /app/data/b\[1\]\*\?' + +# A newline in a mount point cannot be listed one per line. Failing closed +# means nothing is handed over, not that the file is handed over by mistake. +CASE="mount point with a newline" +FIX_MOUNTINFO='601 600 0:50 /srv/c /app/data/x\012y rw,relatime - ext4 /dev/sda1 rw +' drive RUN_AS_UID=1000 +assert_eq "account" "corescope" "$OUT" +assert_eq "startup continues" 0 "$RC" +assert_lacks "nothing handed over" "$CALLS" "chown" +assert_has "and said why" "$LOG" "has a newline in its name" + +# ----- 16. a hand-over that could not complete is logged ------------------- +CASE="chown refused (root_squash NFS)" +drive RUN_AS_UID=1000 STUB_LEFTOVER='/app/data +/app/data/meshcore.db' +assert_eq "account" "corescope" "$OUT" +assert_eq "startup continues" 0 "$RC" +assert_has "warning names the count" "$LOG" "WARNING: 2 path(s) in /app/data could not be given to uid 1000" +assert_has "and the paths" "$LOG" " /app/data/meshcore.db" + +# ----- 17. the entrypoint runs the sequence through prepare_service_user ---- +# The driver above tests the function; this pins that the entrypoint uses it +# and does not rebuild the sequence from the steps, in an order of its own. +CASE="entrypoint-go.sh" +ENTRYPOINT="$(dirname "$SCRIPT")/entrypoint-go.sh" +EP=$(cat "$ENTRYPOINT") +assert_has "calls prepare_service_user" "$EP" "prepare_service_user || exit 1" +assert_lacks "does not resolve the account itself" "$EP" "resolve_service_account" +assert_lacks "does not read .env itself" "$EP" "load_env_file" +assert_lacks "does not hand over paths itself" "$EP" "claim_service_paths" + +# ----- report --------------------------------------------------------------- +echo "select-service-user: $PASS assertions passed, $FAIL failed" +[ "$FAIL" -eq 0 ] || exit 1 diff --git a/docs/deployment.md b/docs/deployment.md index 4b1142bca..0eb8c2178 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -78,6 +78,8 @@ docker run -d --name corescope \ | `-e DISABLE_MOSQUITTO=true` | No | Skip the internal Mosquitto broker (use your own) | | `-e DISABLE_CADDY=true` | No | Skip the built-in Caddy reverse proxy | | `-e MQTT_BROKER=mqtt://host:1883` | No | Override MQTT broker URL | +| `-e RUN_AS_UID=1000` | No | Run the Go services as this uid instead of root — see [Running the Go services unprivileged](#running-the-go-services-unprivileged) | +| `-e RUN_AS_GID=1000` | No | Group for `RUN_AS_UID` (defaults to it) | #### `/app/data/.env` convenience file @@ -92,6 +94,19 @@ MQTT_BROKER=mqtt://my-broker:1883 The entrypoint sources this file before starting services. This works with any launch method (`docker run`, compose, or manage.sh). +Two exceptions, both because the entrypoint runs this file as root from a +directory the Go services can write once they run unprivileged: + +- `RUN_AS_UID` and `RUN_AS_GID` are never taken from `.env`. Set them with + `-e` or in compose's `environment:`. +- With `RUN_AS_UID` set, `.env` is read only if it is owned by root and + writable by no one else (`sudo chown root:root .env && sudo chmod 644 .env`). + Otherwise the log says it was skipped, and why. "Owned by root" means uid 0 + inside the container. With rootless Docker or a user-namespace remap + (`userns-remap`), host root is not that uid: look at the owner the container + reports (`docker exec stat -c '%u' /app/data/.env`) and give the host + file whichever owner id maps to container uid 0. + ### Docker Compose (legacy alternative) Docker Compose files are maintained for backward compatibility but are no longer the recommended approach. @@ -142,6 +157,84 @@ docker run -d --name corescope ... # same flags as before Data is preserved in the volume — updates are non-destructive. +### Running the Go services unprivileged + +`corescope-server` and `corescope-ingestor` can run as a normal user instead of +root, so that a code-execution bug in either one does not land as root in a +container that has your data directory and `config.json` mounted. `supervisord` +itself stays root, because mosquitto and caddy still need to bind their ports. + +It is opt-in, with `RUN_AS_UID`: + +| `RUN_AS_UID` | Services run as | +|--------------|-----------------| +| unset (the default) | root — as in earlier releases | +| set | `RUN_AS_UID` / `RUN_AS_GID` | + +Nothing about an upgrade changes until you set it: the container does not pick a +uid for you, and does not change the owner of anything on your host. That is +deliberate rather than cautious. Every release before this one ran as root, so +the `meshcore.db` in your data directory — with its `-wal` and `-shm`, `users.db` +and the backups beside them — is root-owned. Switching user without also handing +those files over leaves the services unable to write the database, so the switch +has to be something you ask for. + +**To turn it on**, pick the uid you want and restart. Setting `RUN_AS_UID` is +taken as permission to `chown` the data directory, and what an earlier root-run +container left in it, to that uid. This happens at every start, so a file +created there later as root (with `docker exec`, say) is handed over at the +next restart: + +```bash +docker run -d --name corescope \ + -e RUN_AS_UID=1000 -e RUN_AS_GID=1000 \ + -v /your/data:/app/data \ + ... # same flags as before + ghcr.io/kpa-clawbot/corescope:latest +``` + +Check which user it picked: + +```bash +docker logs corescope 2>&1 | grep '\[entrypoint\] Go services run as' +# [entrypoint] Go services run as corescope (uid=1000 gid=1000) +``` + +If you would rather the container never `chown` anything, do the hand-over +yourself first. Then there is nothing left for it to change: + +```bash +docker stop corescope +sudo chown -R 1000:1000 /your/data # uid of your own account is usually 1000 +docker start corescope # with RUN_AS_UID=1000 set as above +``` + +To go back, unset `RUN_AS_UID` and `chown -R root:root /your/data`. + +Notes: + +- `RUN_AS_GID` defaults to `RUN_AS_UID` when omitted. `RUN_AS_UID=0` is the + same as leaving it unset. +- If the uid already belongs to another account inside the image (`caddy` is + uid 100, `mosquitto` 101), that account is used as it is, keeping its own + group, and the log says so. Pick a different uid if you need `RUN_AS_GID` + honoured. +- With `RUN_AS_UID` set to a uid the container cannot use, startup fails rather + than quietly falling back to root. +- What the hand-over leaves alone: anything mounted into the data directory on + its own, such as `-v ./config.json:/app/data/config.json:ro`, which keeps the + owner it has on your host (make sure the uid can read it), and `.env` (see + above). Each mount it skips is named in the log. +- If the hand-over cannot change a file, for example on NFS with + `root_squash`, the log says how many and which. Fix their owner on the host. +- `/app` stays root's. The services write only to the data directory and + `/tmp`. +- A fresh install is root-owned and so starts as root, like any other. Set + `RUN_AS_UID` from the first run to avoid a hand-over later. + +The uid selection lives in `docker/select-service-user.sh`, with its branches +covered by `docker/test-select-service-user.sh`. + --- ## Configuration Reference diff --git a/internal/dbschema/dbschema.go b/internal/dbschema/dbschema.go index ba0533ae7..c6885b02a 100644 --- a/internal/dbschema/dbschema.go +++ b/internal/dbschema/dbschema.go @@ -111,9 +111,15 @@ func Apply(rw *sql.DB, logf Logger) error { if err := ensureAdvertEvidence(rw); err != nil { return fmt.Errorf("ensure advert route evidence: %w", err) } + if err := ensureKeyBackfill(rw); err != nil { + return fmt.Errorf("ensure key backfill: %w", err) + } // Advert evidence is intentionally not required by AssertReady: a legacy // read-only server reports unknown until this table/backfill is available, // and re-probes absence instead of caching the startup race permanently. + // tx_rewrite_feed (#2107) follows the same rule for the same reason: a + // server without it simply never sees in-place rewrites until restart, + // which is exactly the behaviour before the feed existed. return nil } diff --git a/internal/dbschema/key_backfill.go b/internal/dbschema/key_backfill.go new file mode 100644 index 000000000..0934cc7ab --- /dev/null +++ b/internal/dbschema/key_backfill.go @@ -0,0 +1,52 @@ +package dbschema + +import "database/sql" + +// ensureKeyBackfill creates the tables behind the key-set backfill (#2107). +// +// tx_rewrite_feed is the change feed: the ingestor appends one row per +// transmission it rewrote in place (a GRP_TXT it can now decrypt, a transport +// scope it can now name), in the same transaction as the rewrite itself. The +// read-only server tails it with its own cursor, exactly like +// advert_route_evidence, so a rewritten row reaches the in-memory store without +// a restart. AUTOINCREMENT keeps feed IDs monotonic after pruning; the +// created_at index serves the age-based prune, and the tx_id index serves the +// ON DELETE CASCADE, so retention deleting a transmission does not scan the +// whole feed. +// +// key_backfill_state holds at most one in-flight run per kind. run_fp is the +// fingerprint of the key set the run is working with: a run whose fingerprint +// no longer matches the live key set is restarted, which is what makes a key +// added mid-run safe. cursor advances in the same transaction as the rewrites +// it covers, so a cancelled run resumes exactly where it committed. +// +// key_backfill_done records which keys have had a complete pass over history, +// so a restart with an unchanged key set does no work at all. +// +// PREFLIGHT: async=true reason="creates three empty tables and two indexes on the empty feed; the backfill itself runs batched in the background after live ingest is ready" +func ensureKeyBackfill(rw *sql.DB) error { + _, err := rw.Exec(` + CREATE TABLE IF NOT EXISTS tx_rewrite_feed ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + tx_id INTEGER NOT NULL REFERENCES transmissions(id) ON DELETE CASCADE, + created_at INTEGER NOT NULL DEFAULT (CAST(strftime('%s','now') AS INTEGER)) + ); + CREATE INDEX IF NOT EXISTS idx_tx_rewrite_feed_created ON tx_rewrite_feed(created_at); + CREATE INDEX IF NOT EXISTS idx_tx_rewrite_feed_tx ON tx_rewrite_feed(tx_id); + CREATE TABLE IF NOT EXISTS key_backfill_state ( + kind TEXT PRIMARY KEY CHECK(kind IN ('channel','region')), + run_fp TEXT NOT NULL, + cursor INTEGER NOT NULL DEFAULT 0, + upper INTEGER NOT NULL DEFAULT 0, + rewritten INTEGER NOT NULL DEFAULT 0, + started_at INTEGER NOT NULL DEFAULT (CAST(strftime('%s','now') AS INTEGER)) + ); + CREATE TABLE IF NOT EXISTS key_backfill_done ( + kind TEXT NOT NULL, + key_fp TEXT NOT NULL, + done_at INTEGER NOT NULL DEFAULT (CAST(strftime('%s','now') AS INTEGER)), + PRIMARY KEY(kind, key_fp) + ) WITHOUT ROWID; + `) + return err +} diff --git a/public/analytics.js b/public/analytics.js index 7851bdb2c..1bfb12b89 100644 --- a/public/analytics.js +++ b/public/analytics.js @@ -4330,6 +4330,7 @@ function destroy() { _stopRolesRefresh(); _stopScopesRefresh(); _stopDistanceRet avg: ${avgNf} max: ${maxNf} ${obs.sample_count} samples + TX 1h: ${ObserverAirtime.badge(obs.tx_airtime_pct_1h, summaryData.duty_cycle_warn_pct)} `; }).join(''); diff --git a/public/index.html b/public/index.html index a6c417e29..b845beafd 100644 --- a/public/index.html +++ b/public/index.html @@ -225,6 +225,7 @@ + diff --git a/public/observer-airtime.js b/public/observer-airtime.js new file mode 100644 index 000000000..90eaf0047 --- /dev/null +++ b/public/observer-airtime.js @@ -0,0 +1,36 @@ +/* #661 — rolling 1 h TX airtime share per observer, shared by the RF Health + * tiles (analytics.js) and the observer list (observers.js). The value and + * the warning threshold come from /api/observers/metrics/summary + * (tx_airtime_pct_1h, duty_cycle_warn_pct). */ +window.ObserverAirtime = (function () { + 'use strict'; + var DEFAULT_WARN_PCT = 10; + + function warnPct(w) { + return w > 0 ? w : DEFAULT_WARN_PCT; + } + + function format(pct) { + if (pct == null) return '—'; + var n = Number(pct); + if (n > 0 && n < 0.1) return '<0.1%'; + return n.toFixed(1) + '%'; + } + + // Compared at the one decimal format() shows, so "10.0%" is never unflagged. + function isOver(pct, w) { + return pct != null && Math.round(Number(pct) * 10) / 10 >= warnPct(w); + } + + function badge(pct, w) { + if (pct == null) { + return '—'; + } + var over = isOver(pct, w); + var title = 'TX airtime over the last hour. Duty-cycle warning at ' + warnPct(w) + '%'; + return '' + + format(pct).replace('<', '<') + ''; + } + + return { format: format, isOver: isOver, badge: badge }; +})(); diff --git a/public/observers.js b/public/observers.js index 90f3c6075..7caf780ae 100644 --- a/public/observers.js +++ b/public/observers.js @@ -140,6 +140,8 @@ window.preserveCompareSelection = function preserveCompareSelection(prevIds, tbo let _fetchedAt = 0; // #1562: ms epoch when the current `observers` payload was received let _loadObserversReqId = 0; // #1563: monotonic id; resolutions older than the latest are discarded let obsSkewMap = {}; // observerID → {offsetSec, samples} + let obsAirtimeMap = {}; // #661 observerID → tx_airtime_pct_1h + let obsAirtimeWarnPct; // #661 duty_cycle_warn_pct from the summary let wsHandler = null; let refreshTimer = null; let regionChangeHandler = null; @@ -255,6 +257,7 @@ window.preserveCompareSelection = function preserveCompareSelection(prevIds, tbo _mqttPanelTeardown = null; observers = []; obsSkewMap = {}; + obsAirtimeMap = {}; } async function loadObservers(opts) { @@ -265,13 +268,20 @@ window.preserveCompareSelection = function preserveCompareSelection(prevIds, tbo // (or vice versa) with stale data + a misleading "0s ago" pill. var myId = ++_loadObserversReqId; try { - const [data, skewData] = await Promise.all([ + const [data, skewData, airData] = await Promise.all([ api('/observers', { ttl: CLIENT_TTL.observers, bust: bust }), - api('/observers/clock-skew', { ttl: 30000 }).catch(function() { return []; }) + api('/observers/clock-skew', { ttl: 30000 }).catch(function() { return []; }), + // #661 — rolling 1 h TX airtime; one bulk call, not per observer. + api('/observers/metrics/summary?window=1h', { ttl: 30000 }).catch(function() { return {}; }) ]); if (myId !== _loadObserversReqId) return; // stale resolve, newer in-flight observers = data.observers || []; _fetchedAt = Date.now(); // #1562: stamp freshness for the header label + obsAirtimeMap = {}; + obsAirtimeWarnPct = airData && airData.duty_cycle_warn_pct; + ((airData && airData.observers) || []).forEach(function(a) { + if (a && a.observer_id) obsAirtimeMap[a.observer_id] = a.tx_airtime_pct_1h; + }); obsSkewMap = {}; (Array.isArray(skewData) ? skewData : []).forEach(function(s) { if (s && s.observerID) obsSkewMap[s.observerID] = s; @@ -384,7 +394,7 @@ window.preserveCompareSelection = function preserveCompareSelection(prevIds, tbo Observer status and statistics StatusNameRegionLast StatusLast Packet - Packet HealthTotal PacketsPackets/HourClock OffsetUptimeFirmwareClient + Packet HealthTotal PacketsPackets/HourTX 1hClock OffsetUptimeFirmwareClient Select for compare ${filtered.map(o => { @@ -419,6 +429,7 @@ window.preserveCompareSelection = function preserveCompareSelection(prevIds, tbo ${packetBadge(o)} ${(o.packet_count || 0).toLocaleString()} ${sparkBar(o.packetsLastHour || 0, maxPktsHr)} + ${window.ObserverAirtime.badge(obsAirtimeMap[o.id], obsAirtimeWarnPct)} ${(function() { var sk = obsSkewMap[o.id]; if (!sk || sk.samples == null || sk.samples === 0) return '—'; diff --git a/public/style.css b/public/style.css index 5a51f748e..30f91f88c 100644 --- a/public/style.css +++ b/public/style.css @@ -4361,6 +4361,11 @@ button.region-pill-active:hover { opacity: 0.85; color: var(--text-on-accent); } .rf-cell-sparkline { height: 24px; margin: 2px 0; overflow: hidden; } .rf-cell-stats { display: flex; gap: 8px; font-size: 10px; color: var(--text-muted); } +/* #661 — a fourth stat ("TX 1h") wraps between items, never inside one */ +.rf-cell-stats { flex-wrap: wrap; row-gap: 0; } +.rf-cell-stats > span { white-space: nowrap; } +/* #661 — rolling 1 h TX airtime at or over the duty-cycle threshold */ +.airtime-over { color: var(--status-red-text); font-weight: 600; } /* Side panel for observer detail */ .rf-health-detail { diff --git a/scripts/e2e-coverage-hook.js b/scripts/e2e-coverage-hook.js new file mode 100644 index 000000000..dee310a3e --- /dev/null +++ b/scripts/e2e-coverage-hook.js @@ -0,0 +1,18 @@ +'use strict'; +// Preload for the E2E shards (#2203): `NODE_OPTIONS=--require ./scripts/e2e-coverage-hook.js`. +// Every Chromium the suite launches is tracked by scripts/e2e-coverage.js, and +// the summed coverage is written to .nyc_output/ when the process exits. A +// suite that never loads an instrumented page writes nothing. + +const fs = require('fs'); +const path = require('path'); +const { chromium } = require('playwright'); +const { createCollector, writeCoverage } = require('./e2e-coverage'); + +const collector = createCollector(); +collector.wrapLaunch(chromium); + +process.on('exit', () => { + writeCoverage(fs, path, collector.store, path.resolve(__dirname, '..', '.nyc_output'), + path.basename(process.argv[1] || 'node', '.js'), process.pid); +}); diff --git a/scripts/e2e-coverage.js b/scripts/e2e-coverage.js new file mode 100644 index 000000000..1115986b3 --- /dev/null +++ b/scripts/e2e-coverage.js @@ -0,0 +1,154 @@ +'use strict'; +// Collects window.__coverage__ from every page a Playwright E2E suite opens +// (#2203). Loaded through scripts/e2e-coverage-hook.js; kept free of +// Playwright and fs so tests/unit/test-e2e-coverage.js can drive it with fakes. +// +// A page's counters live until its document is replaced, so they are read +// (and zeroed) before every call that can replace or end the document. A +// full load the suite does not start through one of these calls, such as a +// click on a link that leaves the SPA, is not counted. + +const PAGE_CALLS = ['goto', 'reload', 'setContent', 'goBack', 'goForward', 'close']; + +// A read must never hold up the call it runs before: a page stuck in a busy +// loop would otherwise keep browser.close() in a suite's cleanup waiting for +// as long as the loop runs. Past this, the read is given up. +const READ_TIMEOUT_MS = 5000; + +// Runs inside the page. Returns the hit counts since the last read and zeroes +// them, so a document read twice is not counted twice. A file the caller has +// not seen yet comes back whole (with its statement, function and branch maps). +// The result is a JSON string: Playwright serialises a returned object node by +// node, which took about 10 s for the roughly 6 MB of maps of the first read. +function readAndReset(known) { + const cov = window.__coverage__; + if (!cov) return null; + const seen = new Set(known); + const out = {}; + for (const file of Object.keys(cov)) { + const fc = cov[file]; + out[file] = seen.has(file) ? { s: fc.s, f: fc.f, b: fc.b } : fc; + } + const json = JSON.stringify(out); + for (const file of Object.keys(cov)) { + const fc = cov[file]; + for (const k in fc.s) fc.s[k] = 0; + for (const k in fc.f) fc.f[k] = 0; + for (const k in fc.b) fc.b[k].fill(0); + } + return json; +} + +// Adds a snapshot from readAndReset to store, summing the hit counts. +function mergeInto(store, snapshot) { + for (const file of Object.keys(snapshot)) { + const add = snapshot[file]; + const into = store[file]; + if (!into) { + store[file] = add; + continue; + } + for (const k in add.s) into.s[k] = (into.s[k] || 0) + add.s[k]; + for (const k in add.f) into.f[k] = (into.f[k] || 0) + add.f[k]; + for (const k in add.b) { + const sum = into.b[k] || add.b[k].map(() => 0); + add.b[k].forEach((n, i) => { sum[i] = (sum[i] || 0) + n; }); + into.b[k] = sum; + } + } +} + +function createCollector({ readTimeoutMs = READ_TIMEOUT_MS } = {}) { + const store = {}; + const pages = new WeakSet(); + const contexts = new WeakSet(); + + async function collect(page) { + if (page.isClosed()) return; + let timer; + const timeout = new Promise(resolve => { timer = setTimeout(resolve, readTimeoutMs, null); }); + let snapshot; + try { + snapshot = await Promise.race([page.evaluate(readAndReset, Object.keys(store)), timeout]); + } catch { + return; // page crashed or navigated away mid-read: nothing to add + } finally { + clearTimeout(timer); + } + if (snapshot) mergeInto(store, JSON.parse(snapshot)); + } + + function readFirst(obj, names, read) { + for (const name of names) { + const orig = obj[name]; + if (typeof orig !== 'function') continue; + obj[name] = async function (...args) { + await read(); + return orig.apply(this, args); + }; + } + } + + function trackPage(page) { + if (pages.has(page)) return page; + pages.add(page); + readFirst(page, PAGE_CALLS, () => collect(page)); + return page; + } + + function trackContext(context) { + if (contexts.has(context)) return context; + contexts.add(context); + context.pages().forEach(trackPage); + context.on('page', trackPage); + readFirst(context, ['close'], () => Promise.all(context.pages().map(collect))); + const newPage = context.newPage; + context.newPage = async function (...args) { + return trackPage(await newPage.apply(this, args)); + }; + return context; + } + + function trackBrowser(browser) { + const newContext = browser.newContext; + browser.newContext = async function (...args) { + return trackContext(await newContext.apply(this, args)); + }; + const newPage = browser.newPage; + browser.newPage = async function (...args) { + const page = await newPage.apply(this, args); + trackContext(page.context()); + return trackPage(page); + }; + readFirst(browser, ['close'], () => + Promise.all(browser.contexts().flatMap(c => c.pages()).map(collect))); + return browser; + } + + function wrapLaunch(browserType) { + const launch = browserType.launch; + browserType.launch = async function (...args) { + return trackBrowser(await launch.apply(this, args)); + }; + } + + return { store, collect, trackPage, trackContext, trackBrowser, wrapLaunch }; +} + +// Writes store as one nyc input file, or nothing when it is empty. Never +// throws: it runs in an exit handler, where a throw would change the suite's +// exit code. Returns the file written, or null. +function writeCoverage(fs, path, store, outDir, suite, pid) { + if (Object.keys(store).length === 0) return null; + try { + fs.mkdirSync(outDir, { recursive: true }); + const file = path.join(outDir, `e2e-${suite}-${pid}.json`); + fs.writeFileSync(file, JSON.stringify(store)); + return file; + } catch (e) { + console.error(`e2e-coverage: not written: ${e.message}`); + return null; + } +} + +module.exports = { readAndReset, mergeInto, createCollector, writeCoverage }; diff --git a/scripts/non-unit-tests.json b/scripts/non-unit-tests.json index 1b0977c3e..2388c1a55 100644 --- a/scripts/non-unit-tests.json +++ b/scripts/non-unit-tests.json @@ -68,6 +68,7 @@ "tests/e2e/test-issue-1599-replay-freeze-e2e.js", "tests/e2e/test-issue-1630-reach-mobile-e2e.js", "tests/e2e/test-issue-1639-observers-sort-e2e.js", + "tests/e2e/test-issue-661-airtime-e2e.js", "tests/e2e/test-issue-1640-compare-discovery-e2e.js", "tests/e2e/test-issue-1648-m1-icons-e2e.js", "tests/e2e/test-issue-1648-m2-icons-e2e.js", diff --git a/test-all.sh b/test-all.sh index 2b17ee0a7..5efc9a9af 100755 --- a/test-all.sh +++ b/test-all.sh @@ -48,6 +48,7 @@ node tests/unit/test-confidence-indicator.js node tests/unit/test-coverage-gate.js node tests/unit/test-customizer-v2.js node tests/unit/test-drag-manager.js +node tests/unit/test-e2e-coverage.js node tests/unit/test-direct-rf-heard-by.js node tests/unit/test-embed-mode-1369.js node tests/unit/test-fetch-all-nodes-pagination.js @@ -101,6 +102,7 @@ node tests/unit/test-issue-1509-nav-active-bg.js node tests/unit/test-issue-1518-home-url.js node tests/unit/test-issue-1532-live-fullscreen.js node tests/unit/test-issue-1562-observers-summary.js +node tests/unit/test-issue-661-observer-airtime.js node tests/unit/test-issue-1563-aggregate-and-inflight.js node tests/unit/test-issue-1574-live-map-max-nodes.js node tests/unit/test-issue-1606-pagination.js diff --git a/tests/e2e/test-e2e-playwright.js b/tests/e2e/test-e2e-playwright.js index 44984552e..43a5a1ddc 100644 --- a/tests/e2e/test-e2e-playwright.js +++ b/tests/e2e/test-e2e-playwright.js @@ -2603,19 +2603,6 @@ async function run() { // ─── End #1468 tests ────────────────────────────────────────────────────── - // Extract frontend coverage if instrumented server is running - try { - const coverage = await page.evaluate(() => window.__coverage__); - if (coverage) { - const fs = require('fs'); - const path = require('path'); - const outDir = path.join(REPO_ROOT, '.nyc_output'); - if (!fs.existsSync(outDir)) fs.mkdirSync(outDir, { recursive: true }); - fs.writeFileSync(path.join(outDir, 'e2e-coverage.json'), JSON.stringify(coverage)); - console.log(`Frontend coverage from E2E: ${Object.keys(coverage).length} files`); - } - } catch {} - // --- Group: Deep linking (#536) --- // Test: nodes tab deep link diff --git a/tests/e2e/test-issue-661-airtime-e2e.js b/tests/e2e/test-issue-661-airtime-e2e.js new file mode 100644 index 000000000..7eaa6e69e --- /dev/null +++ b/tests/e2e/test-issue-661-airtime-e2e.js @@ -0,0 +1,111 @@ +/** + * E2E test (#661): rolling 1 h TX airtime per observer. + * + * The fixture DB has no observer_metrics inside the last hour, so the + * summary endpoint is intercepted with page.route and given known values for + * the fixture's own observer ids: one over the duty-cycle threshold, one + * under, the rest unknown. + * + * Usage: BASE_URL=http://localhost:13581 node test-issue-661-airtime-e2e.js + */ +const { chromium } = require('playwright'); + +const BASE = process.env.BASE_URL || 'http://localhost:3000'; + +async function test(name, fn) { + try { + await fn(); + console.log(` ✅ ${name}`); + } catch (err) { + console.log(` ❌ ${name}: ${err.message}`); + process.exit(1); + } +} + +function assert(cond, msg) { + if (!cond) throw new Error(msg || 'Assertion failed'); +} + +function summaryFor(ids, overId, underId) { + return { + duty_cycle_warn_pct: 10, + observers: ids.map((id) => ({ + observer_id: id, + observer_name: id, + current_noise_floor: -110, + avg_noise_floor_24h: -111, + max_noise_floor_24h: -105, + battery_mv: null, + sample_count: 12, + sparkline: [-112, -110, -108], + tx_airtime_pct_1h: id === overId ? 12.5 : (id === underId ? 2 : null), + rx_airtime_pct_1h: null, + })), + }; +} + +async function run() { + const browser = await chromium.launch({ + headless: true, + executablePath: process.env.CHROMIUM_PATH || undefined, + args: ['--no-sandbox', '--disable-gpu', '--disable-dev-shm-usage'] + }); + const ctx = await browser.newContext({ viewport: { width: 1400, height: 900 } }); + const page = await ctx.newPage(); + page.setDefaultTimeout(15000); + + console.log(`\nRunning #661 observer airtime E2E tests against ${BASE}\n`); + + const obsResp = await page.request.get(`${BASE}/api/observers`); + const ids = ((await obsResp.json()).observers || []).map((o) => o.id); + assert(ids.length >= 2, `fixture needs >=2 observers, got ${ids.length}`); + const [overId, underId] = ids; + + await page.route('**/api/observers/metrics/summary*', (route) => route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify(summaryFor(ids, overId, underId)), + })); + + await test('observer list has a sortable numeric TX 1h column', async () => { + await page.goto(`${BASE}/#/observers`, { waitUntil: 'domcontentloaded' }); + await page.evaluate(() => { try { localStorage.removeItem('meshcore-observers-sort'); } catch (_) {} }); + await page.reload({ waitUntil: 'domcontentloaded' }); + await page.waitForSelector('#obsTable tbody tr'); + const th = await page.$('#obsTable thead th[data-sort-key="airtime_1h"]'); + assert(th, 'TX 1h must carry data-sort-key="airtime_1h"'); + assert(await th.getAttribute('data-type') === 'numeric', 'TX 1h must be data-type="numeric"'); + }); + + await test('over-threshold observer is flagged, under-threshold is not, unknown is a dash', async () => { + const cell = (id) => page.$eval(`#obsTable tbody tr[data-observer-id="${id}"] td[data-testid="obs-cell-airtime"]`, + (td) => ({ html: td.innerHTML, text: td.textContent.trim(), value: td.getAttribute('data-value') })); + const over = await cell(overId); + assert(over.html.includes('airtime-over') && over.text === '12.5%', `over cell: ${JSON.stringify(over)}`); + const under = await cell(underId); + assert(!under.html.includes('airtime-over') && under.text === '2.0%', `under cell: ${JSON.stringify(under)}`); + if (ids.length > 2) { + const unknown = await cell(ids[2]); + assert(unknown.text === '—' && unknown.value === '', `unknown cell: ${JSON.stringify(unknown)}`); + } + }); + + await test('clicking TX 1h sorts the highest share first', async () => { + await page.click('#obsTable thead th[data-sort-key="airtime_1h"]'); + const first = await page.$eval('#obsTable tbody tr', (tr) => tr.getAttribute('data-observer-id')); + assert(first === overId, `first row after sort = ${first}, want ${overId}`); + }); + + await test('RF Health tiles show the TX 1h share with the warning', async () => { + await page.goto(`${BASE}/#/analytics?tab=rf-health`, { waitUntil: 'domcontentloaded' }); + await page.waitForSelector('.rf-cell'); + const over = await page.$eval(`.rf-cell[data-observer="${overId}"] [data-testid="rf-cell-airtime"]`, + (el) => ({ html: el.innerHTML, text: el.textContent.trim() })); + assert(over.html.includes('airtime-over') && over.text === 'TX 1h: 12.5%', `tile: ${JSON.stringify(over)}`); + }); + + await browser.close(); + console.log('\nAll #661 airtime E2E tests passed.'); +} + +run().catch((e) => { console.error(e); process.exit(1); }); diff --git a/tests/unit/test-e2e-coverage.js b/tests/unit/test-e2e-coverage.js new file mode 100644 index 000000000..255d7c252 --- /dev/null +++ b/tests/unit/test-e2e-coverage.js @@ -0,0 +1,223 @@ +/* scripts/e2e-coverage.js (#2203): every E2E page's __coverage__ is read before + * its document is replaced or closed, summed, and never counted twice. Driven + * with fake pages, so no browser is needed. */ +'use strict'; +const REPO_ROOT = require('path').resolve(__dirname, '..', '..'); + +const assert = require('assert'); +const fs = require('fs'); +const path = require('path'); +const os = require('os'); +const { readAndReset, mergeInto, createCollector, writeCoverage } = require('../../scripts/e2e-coverage'); + +// Playwright calls that replace a page's document. Listed here, not taken from +// the module, so dropping one there fails this test. +const NEW_DOCUMENT = ['goto', 'reload', 'setContent', 'goBack', 'goForward']; + +// One instrumented file as istanbul shapes it: statement, function and branch +// counters plus the maps that only need to travel once. +function freshFile() { + return { + path: 'public/app.js', + statementMap: { 0: {}, 1: {} }, fnMap: { 0: {} }, branchMap: { 0: {} }, + s: { 0: 0, 1: 0 }, f: { 0: 0 }, b: { 0: [0, 0] }, + }; +} + +// A fake Playwright page: evaluate runs the function against this page's +// window, and goto/reload/setContent/goBack/goForward load a new document +// (fresh counters), as a real full load does. +function fakePage(context) { + const page = { + win: { __coverage__: { 'public/app.js': freshFile() } }, + closed: false, + isClosed() { return this.closed; }, + async evaluate(fn, arg) { + const saved = global.window; + global.window = this.win; + try { return fn(arg); } finally { global.window = saved; } + }, + context() { return context; }, + hit(stmt, times) { this.win.__coverage__['public/app.js'].s[stmt] += times; }, + }; + for (const name of NEW_DOCUMENT) { + page[name] = async function () { + this.win = { __coverage__: { 'public/app.js': freshFile() } }; + }; + } + page.close = async function () { this.closed = true; }; + return page; +} + +function fakeContext() { + const listeners = []; + const ctx = { + list: [], + pages() { return this.list.filter(p => !p.closed); }, + on(event, fn) { if (event === 'page') listeners.push(fn); }, + // A page the context opens by itself (window.open): only the 'page' + // event announces it. + open() { + const p = fakePage(ctx); + this.list.push(p); + listeners.forEach(fn => fn(p)); + return p; + }, + async newPage() { return this.open(); }, + async close() { this.list.forEach(p => { p.closed = true; }); }, + }; + return ctx; +} + +function fakeBrowser() { + const browser = { + ctxs: [], + contexts() { return this.ctxs; }, + async newContext() { const c = fakeContext(); this.ctxs.push(c); return c; }, + async newPage() { const c = await this.newContext(); return c.newPage(); }, + async close() { this.ctxs.forEach(c => c.list.forEach(p => { p.closed = true; })); }, + }; + return browser; +} + +async function main() { + // readAndReset: a new file comes back whole, a known one as counts only, + // and the page's counters are zero afterwards. + { + global.window = { __coverage__: { 'public/app.js': freshFile() } }; + window.__coverage__['public/app.js'].s[0] = 3; + window.__coverage__['public/app.js'].b[0][1] = 2; + const first = JSON.parse(readAndReset([])); + assert.ok(first['public/app.js'], 'an instrumented file comes back'); + assert.strictEqual(first['public/app.js'].s[0], 3); + assert.deepStrictEqual(first['public/app.js'].b[0], [0, 2]); + assert.ok(first['public/app.js'].statementMap, 'an unseen file must carry its maps'); + assert.deepStrictEqual(window.__coverage__['public/app.js'].s, { 0: 0, 1: 0 }, 'statement counters are zeroed'); + assert.deepStrictEqual(window.__coverage__['public/app.js'].b[0], [0, 0], 'branch counters are zeroed'); + const again = JSON.parse(readAndReset(['public/app.js'])); + assert.strictEqual(again['public/app.js'].statementMap, undefined, 'a known file travels as counts only'); + assert.strictEqual(again['public/app.js'].s[0], 0, 'a second read of the same document adds nothing'); + global.window = {}; + assert.strictEqual(readAndReset([]), null, 'an uninstrumented page gives nothing'); + delete global.window; + } + + // mergeInto sums every counter and copies files it has not seen. + { + const store = {}; + const a = freshFile(); a.s[0] = 1; a.b[0] = [1, 0]; + mergeInto(store, { 'public/app.js': a }); + mergeInto(store, { 'public/app.js': { s: { 0: 2, 1: 5 }, f: { 0: 1 }, b: { 0: [0, 4] } } }); + assert.deepStrictEqual(store['public/app.js'].s, { 0: 3, 1: 5 }); + assert.deepStrictEqual(store['public/app.js'].f, { 0: 1 }); + assert.deepStrictEqual(store['public/app.js'].b, { 0: [1, 4] }); + } + + // Every document a page shows is counted once: before each call that + // replaces it, and before close. + for (const name of NEW_DOCUMENT) { + const c = createCollector(); + const browser = fakeBrowser(); + c.trackBrowser(browser); + const page = await browser.newPage(); + page.hit(0, 1); + await page[name](); + page.hit(0, 2); + page.hit(1, 1); + await page.close(); + assert.deepStrictEqual(c.store['public/app.js'].s, { 0: 3, 1: 1 }, name + ': both documents counted once'); + await browser.close(); + assert.deepStrictEqual(c.store['public/app.js'].s, { 0: 3, 1: 1 }, name + ': a closed page is not read again'); + } + + // Pages left open are read when their context or browser closes, including + // pages from browser.newContext() and ones the page itself opened. + { + const c = createCollector(); + const browser = fakeBrowser(); + c.trackBrowser(browser); + const ctx = await browser.newContext(); + const p1 = await ctx.newPage(); + p1.hit(0, 1); + await ctx.close(); + assert.strictEqual(c.store['public/app.js'].s[0], 1, 'context.close reads its pages'); + + const ctx2 = await browser.newContext(); + const popup = ctx2.open(); + popup.hit(1, 2); + await popup.goto(); + assert.strictEqual(c.store['public/app.js'].s[1], 2, 'a page opened by the page itself is tracked too'); + const p2 = await ctx2.newPage(); + p2.hit(1, 4); + await browser.close(); + assert.strictEqual(c.store['public/app.js'].s[1], 6, 'browser.close reads the pages still open'); + } + + // wrapLaunch tracks what launch returns. + { + const c = createCollector(); + const browserType = { async launch() { return fakeBrowser(); } }; + c.wrapLaunch(browserType); + const browser = await browserType.launch(); + const page = await browser.newPage(); + page.hit(0, 7); + await browser.close(); + assert.strictEqual(c.store['public/app.js'].s[0], 7); + } + + // A read that fails or hangs is given up: the call it runs before still + // happens, promptly, and nothing is added. + { + const c = createCollector({ readTimeoutMs: 50 }); + const browser = fakeBrowser(); + c.trackBrowser(browser); + const broken = await browser.newPage(); + broken.evaluate = async () => { throw new Error('Target crashed'); }; + await broken.close(); + assert.ok(broken.closed, 'close still runs after a failed read'); + + const busy = await browser.newPage(); + busy.evaluate = () => new Promise(() => {}); // a page stuck in a busy loop + const started = Date.now(); + await busy.close(); + assert.ok(busy.closed, 'close still runs after a read that never answers'); + assert.ok(Date.now() - started < 2000, 'the read is given up after the timeout, took ' + (Date.now() - started) + ' ms'); + assert.deepStrictEqual(c.store, {}, 'a failed or hung read adds nothing'); + } + + // writeCoverage: one file when there is coverage, none without, and never + // a throw (it runs in an exit handler). + { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'e2e-cov-')); + const out = path.join(dir, 'nyc'); + assert.strictEqual(writeCoverage(fs, path, {}, out, 'suite', 1), null, 'nothing to write'); + assert.ok(!fs.existsSync(out), 'no directory without coverage'); + const store = { 'public/app.js': freshFile() }; + const file = writeCoverage(fs, path, store, out, 'test-x-e2e', 42); + assert.strictEqual(path.basename(file), 'e2e-test-x-e2e-42.json'); + assert.deepStrictEqual(JSON.parse(fs.readFileSync(file, 'utf8')), store); + const fullDisk = { mkdirSync() {}, writeFileSync() { throw new Error('ENOSPC'); } }; + const savedError = console.error; + console.error = () => {}; + try { + assert.strictEqual(writeCoverage(fullDisk, path, store, out, 'suite', 1), null, 'a failed write returns null'); + } finally { + console.error = savedError; + } + } + + finished = true; + console.log('test-e2e-coverage: ok'); +} + +// A read that waits forever leaves nothing for node to wait on, so the +// process would exit 0 half-way through the test without this check. +let finished = false; +process.on('exit', code => { + if (code === 0 && !finished) { + console.error('test-e2e-coverage: stopped before the end (a read never returned)'); + process.exitCode = 1; + } +}); + +main().catch(e => { console.error(e); process.exit(1); }); diff --git a/tests/unit/test-issue-661-observer-airtime.js b/tests/unit/test-issue-661-observer-airtime.js new file mode 100644 index 000000000..f84b03faa --- /dev/null +++ b/tests/unit/test-issue-661-observer-airtime.js @@ -0,0 +1,78 @@ +/** + * #661 — rolling 1 h TX airtime per observer: the shared badge helper used + * by the RF Health tiles (analytics.js) and the observer list (observers.js). + * Loads the real public/observer-airtime.js in a vm sandbox. + */ +'use strict'; +const fs = require('fs'); +const path = require('path'); +const vm = require('vm'); +const assert = require('assert'); + +const REPO_ROOT = path.resolve(__dirname, '..', '..'); +let passed = 0, failed = 0; +function t(name, fn) { + try { fn(); passed++; console.log(' ✓ ' + name); } + catch (e) { failed++; console.error(' ✗ ' + name + ': ' + e.message); } +} + +const ctx = { window: {}, Number, Math, String }; +vm.createContext(ctx); +vm.runInContext(fs.readFileSync(path.join(REPO_ROOT, 'public', 'observer-airtime.js'), 'utf8'), ctx); +const A = ctx.window.ObserverAirtime; + +console.log('observer-airtime.js'); + +t('exposes window.ObserverAirtime', () => { + assert.ok(A && typeof A.format === 'function' && typeof A.isOver === 'function' && typeof A.badge === 'function'); +}); + +t('format: null and undefined read as unknown', () => { + assert.strictEqual(A.format(null), '—'); + assert.strictEqual(A.format(undefined), '—'); +}); + +t('format: one decimal, tiny non-zero shares as <0.1%', () => { + assert.strictEqual(A.format(0), '0.0%'); + assert.strictEqual(A.format(0.03), '<0.1%'); + assert.strictEqual(A.format(5.18), '5.2%'); + assert.strictEqual(A.format(100), '100.0%'); +}); + +t('isOver: at or above the threshold warns, unknown never does', () => { + assert.strictEqual(A.isOver(10, 10), true); + assert.strictEqual(A.isOver(9.9, 10), false); + assert.strictEqual(A.isOver(null, 10), false); +}); + +t('isOver: a share that displays as the threshold warns', () => { + assert.strictEqual(A.format(9.96), '10.0%'); + assert.strictEqual(A.isOver(9.96, 10), true); + assert.strictEqual(A.isOver(9.94, 10), false); +}); + +t('isOver: a missing threshold falls back to 10%', () => { + assert.strictEqual(A.isOver(10, undefined), true); + assert.strictEqual(A.isOver(9, 0), false); +}); + +t('badge: over threshold carries airtime-over and names the limit', () => { + const html = A.badge(12.5, 10); + assert.ok(/class="airtime-pct airtime-over"/.test(html), html); + assert.ok(html.includes('12.5%'), html); + assert.ok(html.includes('10%'), html); +}); + +t('badge: under threshold has no warning class', () => { + const html = A.badge(2, 10); + assert.ok(!html.includes('airtime-over'), html); + assert.ok(html.includes('2.0%'), html); +}); + +t('badge: unknown renders a muted dash', () => { + const html = A.badge(null, 10); + assert.ok(html.includes('text-muted') && html.includes('—'), html); +}); + +console.log(`\n${passed} passed, ${failed} failed`); +process.exit(failed ? 1 : 0); diff --git a/tests/unit/test-observers-headings.js b/tests/unit/test-observers-headings.js index 825f08761..a6d945a4a 100644 --- a/tests/unit/test-observers-headings.js +++ b/tests/unit/test-observers-headings.js @@ -59,7 +59,7 @@ test('expected headings present and ordered', () => { let m; while ((m = re.exec(thead)) !== null) labels.push(m[1].trim()); const expected = ['Status', 'Name', 'Region', 'Last Status', 'Last Packet', - 'Packet Health', 'Total Packets', 'Packets/Hour', 'Clock Offset', 'Uptime', + 'Packet Health', 'Total Packets', 'Packets/Hour', 'TX 1h', 'Clock Offset', 'Uptime', 'Firmware', 'Client']; assert.deepStrictEqual(labels, expected, `Headings out of sync.\nGot: ${JSON.stringify(labels)}\nExpected: ${JSON.stringify(expected)}`);