deploy.yml

name: CI/CD Pipeline

on:
  push:
    branches: [master]
  pull_request:
    branches: [master]
  workflow_dispatch:

permissions:
  contents: read
  packages: write

concurrency:
  group: ci-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

env:
  FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
  STAGING_COMPOSE_FILE: docker-compose.staging.yml
  STAGING_SERVICE: staging-go
  STAGING_CONTAINER: corescope-staging-go

# Pipeline: measured Go and E2E shards, split checks and image validation run in parallel.
# The stable build-and-publish check requires every shard and its report; only then may
# an upstream push publish to GHCR and deploy. Fork runs never publish/deploy.
#
# Fork guard: everything with an external side effect (GHCR login/push, release
# upload, staging deploy, badge commits) additionally requires
# github.repository == 'Kpa-clawbot/CoreScope'. On a fork (e.g. dborup/CoreScope)
# only the tests, the local image build and CI artifact uploads run.

jobs:
  # ───────────────────────────────────────────────────────────────
  # 1. Go Build & Test
  # ───────────────────────────────────────────────────────────────
  go-shards-check:
    name: "📋 Verify Go Test Assignments"
    runs-on: ubuntu-latest
    permissions:
      contents: read
    timeout-minutes: 20
    steps:
      - name: Checkout code
        uses: actions/checkout@v5

      - name: Set up Node.js 22
        uses: actions/setup-node@v5
        with:
          node-version: '22'

      - name: Clean Go module cache
        run: rm -rf ~/go/pkg/mod 2>/dev/null || true

      - name: Set up Go 1.27.1
        uses: actions/setup-go@v6
        with:
          go-version: '1.27.1'
          cache-dependency-path: |
            cmd/server/go.sum
            cmd/ingestor/go.sum

      - name: Guard every Go test and fuzz seed target
        run: |
          node scripts/run-go-server-shard.js --validate
          node scripts/run-go-server-shard.js --runtime-list


  go-server-1:
    name: "✅ Go Server Tests (1/3)"
    runs-on: ubuntu-latest
    permissions:
      contents: read
    timeout-minutes: 40
    env:
      GO_SERVER_SHARD: '1'
    steps: &go-server-shard-steps
      - name: Checkout code
        uses: actions/checkout@v5

      - name: Set up Node.js 22
        uses: actions/setup-node@v5
        with:
          node-version: '22'

      - name: Clean Go module cache
        run: rm -rf ~/go/pkg/mod 2>/dev/null || true

      - name: Set up Go 1.27.1
        uses: actions/setup-go@v6
        with:
          go-version: '1.27.1'
          cache-dependency-path: |
            cmd/server/go.sum
            cmd/ingestor/go.sum

      - name: Run assigned Go server tests (race and coverage)
        run: node scripts/run-go-server-shard.js

      - name: Upload Go server coverage and execution proof
        if: always()
        uses: actions/upload-artifact@v6
        with:
          name: go-server-${{ env.GO_SERVER_SHARD }}
          overwrite: true
          path: .ci-go/server-${{ env.GO_SERVER_SHARD }}/
          retention-days: 1
          if-no-files-found: error
          include-hidden-files: true


  go-server-2:
    name: "✅ Go Server Tests (2/3)"
    runs-on: ubuntu-latest
    permissions:
      contents: read
    timeout-minutes: 40
    env:
      GO_SERVER_SHARD: '2'
    steps: *go-server-shard-steps


  go-server-3:
    name: "✅ Go Server Tests (3/3)"
    runs-on: ubuntu-latest
    permissions:
      contents: read
    timeout-minutes: 40
    env:
      GO_SERVER_SHARD: '3'
    steps: *go-server-shard-steps


  go-ingestor:
    name: "✅ Go Ingestor Tests"
    runs-on: ubuntu-latest
    permissions:
      contents: read
    timeout-minutes: 30
    steps:
      - name: Checkout code
        uses: actions/checkout@v5

      - name: Set up Node.js 22
        uses: actions/setup-node@v5
        with:
          node-version: '22'

      - name: Clean Go module cache
        run: rm -rf ~/go/pkg/mod 2>/dev/null || true

      - name: Set up Go 1.27.1
        uses: actions/setup-go@v6
        with:
          go-version: '1.27.1'
          cache-dependency-path: |
            cmd/server/go.sum
            cmd/ingestor/go.sum

      - name: Build and test Go ingestor (with coverage)
        run: |
          set -e -o pipefail
          cd cmd/ingestor
          go build .
          go test -timeout 20m -coverprofile=ingestor-coverage.out ./... 2>&1 | tee ingestor-test.log
          echo "--- Go Ingestor Coverage ---"
          go tool cover -func=ingestor-coverage.out | tail -1

      # The full ingestor suite runs without -race (too slow), so the
      # force-reconnect tests (upstream #1897) get their own -race pass: a
      # Disconnect moved into a goroutine races paho's retry loop, and the
      # race detector backs up the behavioural real-paho tests.

      - name: Race-check Go ingestor force-reconnect
        run: |
          set -e -o pipefail
          cd cmd/ingestor
          go test -race -count=1 -timeout 5m -run 'ForceReconnect' ./...

      - name: Record ingestor coverage identity
        run: node scripts/report-go-server-shards.js --ingestor-receipt

      - name: Upload ingestor coverage
        uses: actions/upload-artifact@v6
        with:
          name: go-ingestor-coverage
          overwrite: true
          path: .ci-go/ingestor/
          retention-days: 1
          if-no-files-found: error
          include-hidden-files: true


  go-libraries:
    name: "✅ Go Libraries & CLI"
    runs-on: ubuntu-latest
    permissions:
      contents: read
    timeout-minutes: 20
    steps:
      - name: Checkout code
        uses: actions/checkout@v5

      - name: Clean Go module cache
        run: rm -rf ~/go/pkg/mod 2>/dev/null || true

      - name: Set up Go 1.27.1
        uses: actions/setup-go@v6
        with:
          go-version: '1.27.1'
          cache-dependency-path: |
            cmd/server/go.sum
            cmd/ingestor/go.sum

      - name: Build and test channel library + decrypt CLI
        run: |
          set -e -o pipefail
          cd internal/channel
          go test ./...
          echo "--- Channel library tests passed ---"
          cd ../../cmd/decrypt
          CGO_ENABLED=0 go build -ldflags="-s -w" -o corescope-decrypt .
          go test ./...
          echo "--- Decrypt CLI tests passed ---"

      - name: Test shared channel registry + schema packages
        run: |
          set -e -o pipefail
          (cd internal/channelregistry && go test ./...)
          (cd internal/dbschema && go test ./...)
          (cd internal/brokerurl && go test ./...)
          (cd internal/pprofbind && go test ./...)
          # internal/regions is its own module: the server/ingestor steps
          # above compile it but never run its tests, and it now holds the
          # hashRegions/hashRegionsPath loader both processes share (#360).
          (cd internal/regions && go test ./...)

      # internal/anomaly is its own module (not wired into any binary), so the
      # server/ingestor steps above do not reach it. No -fuzz and no -bench:
      # fuzz targets run their in-code seeds only, benchmarks do not run.

      - name: Vet and test anomaly library (internal/anomaly, with -race)
        run: |
          set -e -o pipefail
          cd internal/anomaly
          go vet ./...
          go test ./...
          go test -race -v ./... 2>&1 | tee anomaly-race-test.log
          passed=$(grep -c -- '^--- PASS: ' anomaly-race-test.log || true)
          echo "--- Anomaly library: ${passed} top-level tests passed under -race ---"
          test "${passed}" -gt 0

      # cmd/anomaly is the offline replay/export tool for internal/anomaly
      # (#457): its own module, not built into any image or release.
      - name: Vet and test anomaly offline tool (cmd/anomaly)
        run: |
          set -e -o pipefail
          cd cmd/anomaly
          go vet ./...
          go test ./...


  js-test:
    name: "✅ JavaScript Unit Tests & Lint"
    runs-on: ubuntu-latest
    permissions:
      contents: read
    timeout-minutes: 20
    steps:
      - name: Checkout code
        uses: actions/checkout@v5
        with:
          fetch-depth: 0

      - name: Set up Node.js 22
        uses: actions/setup-node@v5
        with:
          node-version: '22'

      - name: Clean Go module cache
        run: rm -rf ~/go/pkg/mod 2>/dev/null || true

      - name: Set up Go 1.27.1
        uses: actions/setup-go@v6
        with:
          go-version: '1.27.1'
          cache-dependency-path: |
            cmd/server/go.sum
            cmd/ingestor/go.sum

      - name: Lint CSS variables (issue #1128)
        run: |
          set -e
          node scripts/check-css-vars.js
          node scripts/test-check-css-vars.js

      - name: Lint sqlite3 CLI busy timeout (issue #606)
        run: |
          set -e
          node scripts/check-sqlite3-busy-timeout.js
          node scripts/test-check-sqlite3-busy-timeout.js

      - name: Run JS unit tests (test-all.sh)
        # One list: test-all.sh runs every unit file and prints a summary of
        # the failed ones; the step fails if any file fails (#174).
        run: |
          node --version
          sh test-all.sh

      - name: 🛡️ Preflight XSS gate — actual --diff check (PR only)
        # The fixture self-test above (test-preflight-xss-gate.js) only
        # asserts the script's behavior against fixtures. It does NOT scan
        # the PR's own changes. This step closes that gap by running the
        # gate against added lines in public/**/*.{js,html} on the PR.
        # Gate is PR-scoped only (per djb finding: merge commits would
        # slip an opt-out otherwise). Master pushes skip this step.
        if: github.event_name == 'pull_request'
        env:
          PR_BODY: ${{ github.event.pull_request.body }}
          PREFLIGHT_PR_LABELS: ${{ join(github.event.pull_request.labels.*.name, ' ') }}
        run: |
          set -e
          git fetch origin master --depth=50 2>&1 | tail -3 || true
          # Materialize PR body to a file for the opt-out parser.
          printf '%s' "$PR_BODY" > /tmp/pr-body.md
          PREFLIGHT_PR_BODY=/tmp/pr-body.md bash scripts/check-xss-sinks.sh --diff origin/master

      - name: 🧹 Frontend lint (eslint no-undef) — issue #1342
        run: |
          set -e
          # Use eslint@8 (legacy .eslintrc.json). Don't migrate to flat-config / eslint@9.
          # --no-save: avoid touching package.json / no committed node_modules.
          npm install --no-save --no-audit --no-fund eslint@8
          npx eslint public/*.js


  script-test:
    name: "✅ Script & Container Unit Checks"
    runs-on: ubuntu-latest
    permissions:
      contents: read
    timeout-minutes: 20
    steps:
      - name: Checkout code
        uses: actions/checkout@v5

      - name: Verify Dockerfile COPY invariants (issue #1316)
        run: bash scripts/check-dockerfile-internal-pkgs.sh

      - name: Staging disk-monitor unit tests (issue #1684)
        run: bash scripts/staging/test-disk-monitor.sh

      - name: Non-root container helper unit tests + ShellCheck (issue #339)
        run: |
          bash scripts/test-nonroot-helper.sh
          shellcheck docker/nonroot.sh scripts/test-nonroot-helper.sh scripts/test-docker-nonroot.sh scripts/test-nonroot-entrypoint.sh
          # SC1091: the entrypoint sources the data volume's .env at run time.
          shellcheck -e SC1091 docker/entrypoint-go.sh

      # The real entrypoint and helper as root in the runtime image's pinned
      # Alpine base, supervisord stubbed: off-mode start, ownership changes,
      # the check before root uses files in the data dir, rollback records.

      - name: Non-root container start path, supervisord stubbed (issue #339)
        run: bash scripts/test-nonroot-entrypoint.sh

      - name: Decode-backfill measurement script unit tests (issue #380)
        run: bash scripts/test-decode-backfill-estimate.sh

      - name: ShellCheck decode-backfill measurement scripts (issue #380)
        run: shellcheck scripts/decode-backfill-estimate.sh scripts/test-decode-backfill-estimate.sh

      - name: QA blacklist script unit tests (issues #1977, #83)
        run: bash qa/scripts/test-blacklist-sql.sh

      - name: ShellCheck QA blacklist scripts (issue #83)
        run: shellcheck -x -P SCRIPTDIR qa/scripts/blacklist-test.sh qa/scripts/test-blacklist-sql.sh

      - name: Verify proto syntax
        timeout-minutes: 5
        run: |
          set -e
          # Bounded apt: retry transient mirror errors, give up on a stalled
          # connection after 30s instead of hanging (#156).
          APT_OPTS="-o Acquire::Retries=3 -o Acquire::http::Timeout=30"
          sudo DEBIAN_FRONTEND=noninteractive apt-get $APT_OPTS update -qq
          sudo DEBIAN_FRONTEND=noninteractive apt-get $APT_OPTS install -y protobuf-compiler
          command -v protoc >/dev/null || { echo "::error::protoc not found after install"; exit 1; }
          for proto in proto/*.proto; do
            echo "  ✓ $(basename "$proto")"
            protoc --proto_path=proto --descriptor_set_out=/dev/null "$proto"
          done
          echo "✅ All .proto files are syntactically valid"


  go-test:
    name: "✅ Go Build & Test"
    runs-on: ubuntu-latest
    permissions:
      contents: read
    timeout-minutes: 20
    needs: [go-shards-check, go-server-1, go-server-2, go-server-3, go-ingestor, go-libraries, js-test, script-test]
    if: ${{ !cancelled() }}
    steps:
      - name: Require all Go and split checks
        env:
          GO_GUARD_RESULT: ${{ needs.go-shards-check.result }}
          GO_SERVER_1_RESULT: ${{ needs.go-server-1.result }}
          GO_SERVER_2_RESULT: ${{ needs.go-server-2.result }}
          GO_SERVER_3_RESULT: ${{ needs.go-server-3.result }}
          GO_INGESTOR_RESULT: ${{ needs.go-ingestor.result }}
          GO_LIBRARIES_RESULT: ${{ needs.go-libraries.result }}
          JS_RESULT: ${{ needs.js-test.result }}
          SCRIPTS_RESULT: ${{ needs.script-test.result }}
        run: |
          for result in "$GO_GUARD_RESULT" "$GO_SERVER_1_RESULT" "$GO_SERVER_2_RESULT" "$GO_SERVER_3_RESULT" "$GO_INGESTOR_RESULT" "$GO_LIBRARIES_RESULT" "$JS_RESULT" "$SCRIPTS_RESULT"; do
            if [ "$result" != success ]; then
              echo "Required CI job did not pass: $result"
              exit 1
            fi
          done

      - name: Checkout code
        uses: actions/checkout@v5

      - name: Set up Node.js 22
        uses: actions/setup-node@v5
        with:
          node-version: '22'

      - name: Clean Go module cache
        run: rm -rf ~/go/pkg/mod 2>/dev/null || true

      - name: Set up Go 1.27.1
        uses: actions/setup-go@v6
        with:
          go-version: '1.27.1'
          cache-dependency-path: |
            cmd/server/go.sum
            cmd/ingestor/go.sum

      - name: Download server shard coverage and execution proof
        uses: actions/download-artifact@v6
        with:
          pattern: go-server-*
          path: .ci-go/downloaded
          merge-multiple: false

      - name: Download ingestor coverage
        uses: actions/download-artifact@v6
        with:
          name: go-ingestor-coverage
          path: .ci-go/ingestor-downloaded

      - name: Verify execution and merge coverage profiles
        env:
          GO_GUARD_RESULT: ${{ needs.go-shards-check.result }}
          GO_SERVER_1_RESULT: ${{ needs.go-server-1.result }}
          GO_SERVER_2_RESULT: ${{ needs.go-server-2.result }}
          GO_SERVER_3_RESULT: ${{ needs.go-server-3.result }}
          GO_INGESTOR_RESULT: ${{ needs.go-ingestor.result }}
          GO_LIBRARIES_RESULT: ${{ needs.go-libraries.result }}
          JS_RESULT: ${{ needs.js-test.result }}
          SCRIPTS_RESULT: ${{ needs.script-test.result }}
        run: |
          node scripts/report-go-server-shards.js
          echo "--- Go Server Coverage ---"
          (cd cmd/server && go tool cover -func=server-coverage.out | tail -1)
          echo "--- Go Ingestor Coverage ---"
          (cd cmd/ingestor && go tool cover -func=ingestor-coverage.out | tail -1)

      - name: Generate Go coverage badges
        if: success()
        run: |
          mkdir -p .badges

          SERVER_COV="0"
          if [ -f cmd/server/server-coverage.out ]; then
            SERVER_COV=$(cd cmd/server && go tool cover -func=server-coverage.out | tail -1 | grep -oP '[\d.]+(?=%)')
          fi
          SERVER_COLOR="red"
          if [ "$(echo "$SERVER_COV >= 80" | bc -l 2>/dev/null)" = "1" ]; then SERVER_COLOR="green"
          elif [ "$(echo "$SERVER_COV >= 60" | bc -l 2>/dev/null)" = "1" ]; then SERVER_COLOR="yellow"; fi
          echo "{\"schemaVersion\":1,\"label\":\"go server coverage\",\"message\":\"${SERVER_COV}%\",\"color\":\"${SERVER_COLOR}\"}" > .badges/go-server-coverage.json

          INGESTOR_COV="0"
          if [ -f cmd/ingestor/ingestor-coverage.out ]; then
            INGESTOR_COV=$(cd cmd/ingestor && go tool cover -func=ingestor-coverage.out | tail -1 | grep -oP '[\d.]+(?=%)')
          fi
          INGESTOR_COLOR="red"
          if [ "$(echo "$INGESTOR_COV >= 80" | bc -l 2>/dev/null)" = "1" ]; then INGESTOR_COLOR="green"
          elif [ "$(echo "$INGESTOR_COV >= 60" | bc -l 2>/dev/null)" = "1" ]; then INGESTOR_COLOR="yellow"; fi
          echo "{\"schemaVersion\":1,\"label\":\"go ingestor coverage\",\"message\":\"${INGESTOR_COV}%\",\"color\":\"${INGESTOR_COLOR}\"}" > .badges/go-ingestor-coverage.json

          echo "## Go Coverage" >> $GITHUB_STEP_SUMMARY
          echo "| Module | Coverage |" >> $GITHUB_STEP_SUMMARY
          echo "|--------|----------|" >> $GITHUB_STEP_SUMMARY
          echo "| Server | ${SERVER_COV}% |" >> $GITHUB_STEP_SUMMARY
          echo "| Ingestor | ${INGESTOR_COV}% |" >> $GITHUB_STEP_SUMMARY

      - name: Upload Go coverage badges
        if: success()
        uses: actions/upload-artifact@v6
        with:
          name: go-badges
          overwrite: true
          path: .badges/go-*.json
          retention-days: 1
          if-no-files-found: ignore
          include-hidden-files: true

      - name: Upload merged Go coverage and report
        uses: actions/upload-artifact@v6
        with:
          name: go-coverage
          overwrite: true
          path: |
            cmd/server/server-coverage.out
            cmd/ingestor/ingestor-coverage.out
            .ci-go/server-shards-report.json
          retention-days: 1
          if-no-files-found: error
          include-hidden-files: true

  # ───────────────────────────────────────────────────────────────
  # 2. Playwright E2E Tests (against Go server with fixture DB)
  # ───────────────────────────────────────────────────────────────
  e2e-test-1:
    name: "🎭 Playwright E2E Tests (1/3)"
    runs-on: ubuntu-latest
    timeout-minutes: 150
    defaults:
      run:
        shell: bash
    env:
      E2E_SHARD: '1'
      E2E_PORT: '13581'
      BASE_URL: http://localhost:13581
    steps: &e2e-shard-steps
      - name: Checkout code
        uses: actions/checkout@v5
        with:
          fetch-depth: 0

      - name: Set up Node.js 22
        uses: actions/setup-node@v5
        with:
          node-version: '22'

      - name: Validate E2E shard assignments
        run: node scripts/run-e2e-shard.js --validate

      - name: Identify the core coverage shard
        id: shard-meta
        run: echo "core_shard=$(node scripts/run-e2e-shard.js --core-shard)" >> "$GITHUB_OUTPUT"

      - name: Clean Go module cache
        run: rm -rf ~/go/pkg/mod 2>/dev/null || true

      - name: Set up Go 1.27.1
        uses: actions/setup-go@v6
        with:
          go-version: '1.27.1'
          cache-dependency-path: cmd/server/go.sum

      - name: Build Go server
        run: |
          cd cmd/server
          go build -o ../../corescope-server .
          echo "Go server built successfully"

      - name: Build Go migrate tool
        run: |
          cd cmd/migrate
          go build -o ../../corescope-migrate .
          echo "Go migrate tool built successfully"

      - name: Build Go ingestor (channel proposals E2E)
        run: |
          cd cmd/ingestor
          go build -o ../../corescope-ingestor .
          echo "Go ingestor built successfully"

      - name: Install npm dependencies
        run: npm ci --production=false

      - name: Install Playwright browser
        run: |
          npx playwright install chromium 2>/dev/null || true
          npx playwright install-deps chromium 2>/dev/null || true

      - name: Instrument frontend JS for coverage
        run: sh scripts/instrument-frontend.sh

      - name: Freshen fixture timestamps
        run: bash tools/freshen-fixture.sh test-fixtures/e2e-fixture.db

      - name: Seed grouped-packet row for #1486 collapse test
        # The committed fixture has 499 packets, each with exactly ONE
        # observation, so the packets-page renders only flat
        # (select-hash) rows. The #1486 repro needs at least one grouped
        # (toggle-select) row. Insert a NEW transmission with 3
        # observations.
        #
        # The server's async hash-migrate (cmd/server/hash_migrate.go)
        # recomputes `transmissions.hash` from `raw_hex` via
        # ComputeContentHash(), so the inserted hash MUST equal that
        # function's output for the chosen raw_hex — otherwise the row
        # gets relabelled and the E2E can't find it.
        #
        # raw_hex 15000102030405060708090a0b0c0d0e0f
        #   → header=0x15 (route_type=1, payload_type=5)
        #   → ComputeContentHash(...) = fae0c9e6d357a814
        #
        # The first_seen / observation timestamps are pinned to a date
        # within retentionHours but outside the default 15-min UI
        # window so the row is hidden in the default view (keeping
        # test-e2e-playwright's first-10-rows hex-pane test
        # unaffected) and reachable via the explicit ?timeWindow=0
        # deep-link the #1486 test uses.
        run: |
          sqlite3 test-fixtures/e2e-fixture.db <<'SQL'
          -- Sort the seeded row LAST in BOTH default packets views:
          --   • flat view sorts by transmissions.id DESC → id=0 puts it last
          --   • grouped view (#default for the packets page) sorts by
          --     MAX(observations.timestamp) DESC → we must keep our obs
          --     timestamps OLDER than every other fixture observation.
          -- Fixture (after freshen) has obs timestamps spanning
          --   2026-05-17 16:01:39Z .. 2026-05-28 00:00:00Z (max).
          --   Note: freshen only shifts transmissions.first_seen forward
          --   to ~now; observation.timestamp is left alone except for
          --   the timestamp=0 case.
          -- Use 2026-05-15 (~2 days older than the oldest fixture obs)
          -- so our row sorts LAST in the grouped view too, keeping
          -- test-e2e-playwright's first-10-rows hex-pane test
          -- unaffected. The #1486 test still reaches the row via the
          -- explicit hash + ?timeWindow=0 deep-link.
          INSERT INTO transmissions(id,raw_hex,hash,first_seen,route_type,payload_type,payload_version,decoded_json,channel_hash,from_pubkey)
            VALUES (0,'15000102030405060708090a0b0c0d0e0f','fae0c9e6d357a814','2026-05-15T00:00:00Z',1,5,0,'{"type":"CHAN","channel":"#test","text":"#1486 fixture"}',NULL,NULL);
          INSERT INTO observations(transmission_id,observer_idx,direction,snr,rssi,score,path_json,timestamp,resolved_path) VALUES
            (0,1,'rx',5.0,-95,0,'["AA"]',CAST(strftime('%s','2026-05-15T00:00:00Z') AS INTEGER),'["aa00000000000000000000000000000000000000000000000000000000000000"]'),
            (0,2,'rx',5.5,-92,0,'["BB"]',CAST(strftime('%s','2026-05-15T00:00:00Z') AS INTEGER),'["bb00000000000000000000000000000000000000000000000000000000000000"]'),
            (0,3,'rx',6.0,-90,0,'["CC"]',CAST(strftime('%s','2026-05-15T00:00:00Z') AS INTEGER),'["cc00000000000000000000000000000000000000000000000000000000000000"]');
          -- #1791 fixture: a single GRP_DATA (payload_type=6) packet so the
          -- E2E "Group Data filter" test has at least one row to filter on.
          -- Use an obs timestamp within the default UI window so the row
          -- appears with no time-window override.
          --
          -- raw_hex header byte 0x19 = bits 5-2 (payload)=0110=6 (GRP_DATA),
          --                            bits 1-0 (route)=01=1 (FLOOD).
          -- path_len byte 0x00 = hash_size=1, hash_count=0 (zero-hop on-wire,
          -- typical GRP_DATA going FLOOD). path_json/resolved_path are kept
          -- EMPTY so the rendered hop-row count matches the hex-path byte
          -- count (a prior fixture used path_json=["AA"] but raw_hex
          -- path_len=0, which broke the "hex strip Path range matches hop
          -- row count" E2E).
          --
          -- Note: id=-1000000 is a deliberately out-of-band sentinel id so
          -- this synthetic fixture row cannot collide with real ingested
          -- transmissions (real ids are positive autoincrement values).
          INSERT INTO transmissions(id,raw_hex,hash,first_seen,route_type,payload_type,payload_version,decoded_json,channel_hash,from_pubkey)
            VALUES (-1000000,'19000102030405060708090a0b0c0d0e0f','17910000deadbeef',strftime('%Y-%m-%dT%H:%M:%SZ','now'),1,6,0,'{"type":"GRP_DATA","channel":"#test","raw":"deadbeef"}',NULL,NULL);
          INSERT INTO observations(transmission_id,observer_idx,direction,snr,rssi,score,path_json,timestamp,resolved_path) VALUES
            (-1000000,1,'rx',7.0,-88,0,'[]',CAST(strftime('%s','now') AS INTEGER),'[]');
          SQL

      - name: Migrate fixture DB to current schema (#1287)
        # Server now ASSERTs schema is migrated and refuses to start
        # otherwise (cmd/server/main.go: dbschema.AssertReady). In prod
        # the ingestor owns dbschema.Apply, but CI starts only the
        # server against the committed e2e fixture — so we run the
        # standalone migrate tool here to bring the fixture up to the
        # required shape before the server boots.
        run: ./corescope-migrate -db test-fixtures/e2e-fixture.db

      - name: Seed route-classified adverts for #2073 E2E
        # Runs after the migration because it sets transmissions.route_mask
        # (#89). See the file header for the rows and expected classes.
        run: sqlite3 test-fixtures/e2e-fixture.db < test-fixtures/seed-2073-route-adverts.sql

      - name: Seed an inactive observer node for #199 E2E
        # Also after the migration: the fixture has no inactive_nodes table
        # until corescope-migrate creates it.
        run: sqlite3 test-fixtures/e2e-fixture.db < test-fixtures/seed-199-inactive-observer.sql

      - name: Seed adverts with known gaps for #245 E2E
        # After the migration (sets transmissions.route_mask). See the file
        # header for the rows and the expected interval estimates.
        run: sqlite3 test-fixtures/e2e-fixture.db < test-fixtures/seed-245-advert-intervals.sql

      - name: Seed reference-to-many overlap data for #122 E2E
        # The fixture's observations.direction is NULL throughout (it
        # predates that field), which the #122 endpoint correctly treats
        # as "not RX" rather than guessing -- so without this seed every
        # comparison would be all-zero. See the file header for the exact
        # overlap numbers (matches the issue's own 60%/75% example).
        run: sqlite3 test-fixtures/e2e-fixture.db < test-fixtures/seed-122-observer-comparison.sql

      - name: Start Go server with fixture DB
        run: |
          fuser -k "$E2E_PORT"/tcp 2>/dev/null || true
          sleep 1
          ./corescope-server -port "$E2E_PORT" -db test-fixtures/e2e-fixture.db -public public-instrumented &
          echo $! > .server.pid
          for i in $(seq 1 30); do
            if curl -sf "$BASE_URL/api/healthz" > /dev/null 2>&1; then
              echo "Server ready after ${i}s"
              break
            fi
            if [ "$i" -eq 30 ]; then
              echo "Server failed to start within 30s"
              exit 1
            fi
            sleep 1
          done

      - name: Run assigned Playwright suites and repeat gate (fail-fast)
        run: node scripts/run-e2e-shard.js

      - name: Collect frontend coverage (parallel)
        if: success() && github.event_name == 'push' && steps.shard-meta.outputs.core_shard == env.E2E_SHARD
        run: |
          node scripts/collect-frontend-coverage.js 2>&1 | tee fe-coverage-output.txt || true

      - name: Bundle core frontend coverage
        if: success()
        run: |
          if [ -d .nyc_output ]; then
            cp -R .nyc_output ".ci-e2e/shard-$E2E_SHARD/coverage"
          fi

      - name: Stop test server
        if: always()
        run: |
          if [ -f .server.pid ]; then
            kill $(cat .server.pid) 2>/dev/null || true
            rm -f .server.pid
          fi

      - name: Upload E2E shard receipt and logs
        if: always()
        uses: actions/upload-artifact@v6
        with:
          name: e2e-shard-${{ env.E2E_SHARD }}
          overwrite: true
          path: .ci-e2e/shard-${{ env.E2E_SHARD }}/
          retention-days: 1
          if-no-files-found: error
          include-hidden-files: true

  e2e-test-2:
    name: "🎭 Playwright E2E Tests (2/3)"
    runs-on: ubuntu-latest
    timeout-minutes: 150
    defaults:
      run:
        shell: bash
    env:
      E2E_SHARD: '2'
      E2E_PORT: '13582'
      BASE_URL: http://localhost:13582
    steps: *e2e-shard-steps

  e2e-test-3:
    name: "🎭 Playwright E2E Tests (3/3)"
    runs-on: ubuntu-latest
    timeout-minutes: 150
    defaults:
      run:
        shell: bash
    env:
      E2E_SHARD: '3'
      E2E_PORT: '13583'
      BASE_URL: http://localhost:13583
    steps: *e2e-shard-steps

  e2e-report:
    name: "📊 Verify E2E Shards & Coverage"
    needs: [e2e-test-1, e2e-test-2, e2e-test-3]
    if: ${{ !cancelled() }}
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash
    env:
      E2E_1_RESULT: ${{ needs.e2e-test-1.result }}
      E2E_2_RESULT: ${{ needs.e2e-test-2.result }}
      E2E_3_RESULT: ${{ needs.e2e-test-3.result }}
    steps:
      - name: Require every E2E shard to succeed
        run: |
          for result in "$E2E_1_RESULT" "$E2E_2_RESULT" "$E2E_3_RESULT"; do
            if [ "$result" != success ]; then
              echo "E2E shard did not succeed: '$result'"
              exit 1
            fi
          done

      - name: Checkout code
        uses: actions/checkout@v5

      - name: Set up Node.js 22
        uses: actions/setup-node@v5
        with:
          node-version: '22'

      - name: Install npm dependencies
        run: npm ci --production=false

      - name: Download E2E shard receipts and logs
        uses: actions/download-artifact@v6
        with:
          pattern: e2e-shard-*
          path: .ci-e2e/downloaded
          merge-multiple: false

      - name: Verify every assigned suite executed successfully
        run: node scripts/report-e2e-shards.js

      - name: Generate frontend coverage badges
        if: success()
        run: |
          # Aggregate per-suite PASS/FAIL across every test-*-e2e.js summary.
          # The previous regex (grep -oP '[0-9]+(?=/)' | tail -1) caught a
          # stray digits-before-slash like the '2' in '2/3 tests passed' from
          # some sub-output and stamped the badge as '2 passed'. See #1296.
          eval "$(bash scripts/aggregate-e2e-pass.sh e2e-output.txt)"
          E2E_PASS=${PASS:-0}
          E2E_FAIL=${FAIL:-0}

          mkdir -p .badges
          if [ -f .nyc_output/frontend-coverage.json ] || [ -f .nyc_output/e2e-coverage.json ]; then
            npx nyc report --reporter=text-summary --reporter=text 2>&1 | tee fe-report.txt
            FE_COVERAGE=$(grep 'Statements' fe-report.txt | head -1 | grep -oP '[\d.]+(?=%)' || echo "0")
            FE_COVERAGE=${FE_COVERAGE:-0}
            FE_COLOR="red"
            [ "$(echo "$FE_COVERAGE > 50" | bc -l 2>/dev/null)" = "1" ] && FE_COLOR="yellow"
            [ "$(echo "$FE_COVERAGE > 80" | bc -l 2>/dev/null)" = "1" ] && FE_COLOR="brightgreen"
            echo "{\"schemaVersion\":1,\"label\":\"frontend coverage\",\"message\":\"${FE_COVERAGE}%\",\"color\":\"${FE_COLOR}\"}" > .badges/frontend-coverage.json
            echo "## Frontend: ${FE_COVERAGE}% coverage" >> $GITHUB_STEP_SUMMARY
          fi
          if [ "${E2E_FAIL:-0}" -gt 0 ]; then
            E2E_MSG="${E2E_PASS:-0} passed, ${E2E_FAIL} failed"
            E2E_COLOR="red"
          else
            E2E_MSG="${E2E_PASS:-0} passed"
            E2E_COLOR="brightgreen"
          fi
          echo "{\"schemaVersion\":1,\"label\":\"e2e tests\",\"message\":\"${E2E_MSG}\",\"color\":\"${E2E_COLOR}\"}" > .badges/e2e-tests.json

      - name: Upload E2E badges
        if: success()
        uses: actions/upload-artifact@v6
        with:
          name: e2e-badges
          overwrite: true
          path: .badges/
          retention-days: 1
          if-no-files-found: ignore
          include-hidden-files: true

  # ───────────────────────────────────────────────────────────────
  # 3. Validate the staging image independently of tests
  # ───────────────────────────────────────────────────────────────
  image-check:
    name: "🧱 Validate Docker Image"
    runs-on: ubuntu-latest
    # Local build only: no registry access, and a hung build must not hold the
    # required gate pending for the 360 minute default.
    permissions:
      contents: read
    timeout-minutes: 20
    steps:
      - name: Checkout code
        uses: actions/checkout@v5

      - name: Verify staging compose file
        run: docker compose -f "$STAGING_COMPOSE_FILE" config --quiet

      - name: Build Go Docker image (local staging)
        run: |
          GIT_COMMIT="${GITHUB_SHA::7}" \
          APP_VERSION=edge \
          BUILD_TIME="$(date -u '+%Y-%m-%dT%H:%M:%SZ')" \
          docker compose -f "$STAGING_COMPOSE_FILE" -p corescope-staging build "$STAGING_SERVICE"
          echo "Built Go staging image ✅"

  # ───────────────────────────────────────────────────────────────
  # 4. Stable required gate and GHCR publisher
  # ───────────────────────────────────────────────────────────────
  build-and-publish:
    name: "🏗️ Build & Publish Docker Image"
    needs: [go-shards-check, go-server-1, go-server-2, go-server-3, go-ingestor, go-libraries, js-test, script-test, go-test, e2e-test-1, e2e-test-2, e2e-test-3, e2e-report, image-check]
    # A failed dependency normally skips this job. A skipped required check
    # can count as passing in branch protection, so run and fail explicitly.
    # A cancelled run reports this job as cancelled (not skipped), which does
    # not pass either. Later steps keep the implicit success(), so nothing
    # publishes after a failed gate.
    if: ${{ !cancelled() }}
    runs-on: ubuntu-latest
    steps:
      - name: Require tests and image validation
        env:
          GO_GUARD_RESULT: ${{ needs.go-shards-check.result }}
          GO_SERVER_1_RESULT: ${{ needs.go-server-1.result }}
          GO_SERVER_2_RESULT: ${{ needs.go-server-2.result }}
          GO_SERVER_3_RESULT: ${{ needs.go-server-3.result }}
          GO_INGESTOR_RESULT: ${{ needs.go-ingestor.result }}
          GO_LIBRARIES_RESULT: ${{ needs.go-libraries.result }}
          JS_RESULT: ${{ needs.js-test.result }}
          SCRIPTS_RESULT: ${{ needs.script-test.result }}
          GO_RESULT: ${{ needs.go-test.result }}
          E2E_1_RESULT: ${{ needs.e2e-test-1.result }}
          E2E_2_RESULT: ${{ needs.e2e-test-2.result }}
          E2E_3_RESULT: ${{ needs.e2e-test-3.result }}
          E2E_REPORT_RESULT: ${{ needs.e2e-report.result }}
          IMAGE_RESULT: ${{ needs.image-check.result }}
        run: |
          for result in "$GO_GUARD_RESULT" "$GO_SERVER_1_RESULT" "$GO_SERVER_2_RESULT" "$GO_SERVER_3_RESULT" "$GO_INGESTOR_RESULT" "$GO_LIBRARIES_RESULT" "$JS_RESULT" "$SCRIPTS_RESULT" "$GO_RESULT" "$E2E_1_RESULT" "$E2E_2_RESULT" "$E2E_3_RESULT" "$E2E_REPORT_RESULT" "$IMAGE_RESULT"; do
            if [ "$result" != success ]; then
              echo "Required CI job did not pass: $result"
              exit 1
            fi
          done

      - name: Checkout code
        uses: actions/checkout@v5

      - name: Compute build metadata
        id: meta
        run: |
          BUILD_TIME=$(date -u '+%Y-%m-%dT%H:%M:%SZ')
          GIT_COMMIT="${GITHUB_SHA::7}"
          if [[ "$GITHUB_REF" == refs/tags/v* ]]; then
            APP_VERSION="${GITHUB_REF#refs/tags/}"
          else
            APP_VERSION="edge"
          fi
          echo "build_time=$BUILD_TIME" >> "$GITHUB_OUTPUT"
          echo "git_commit=$GIT_COMMIT" >> "$GITHUB_OUTPUT"
          echo "app_version=$APP_VERSION" >> "$GITHUB_OUTPUT"
          echo "Build: version=$APP_VERSION commit=$GIT_COMMIT time=$BUILD_TIME"

      - name: Set up Docker Buildx
        if: github.event_name == 'push' && github.repository == 'Kpa-clawbot/CoreScope'
        uses: docker/setup-buildx-action@v3

      - name: Set up QEMU (arm64 runtime stage)
        if: github.event_name == 'push' && github.repository == 'Kpa-clawbot/CoreScope'
        uses: docker/setup-qemu-action@v3

      - name: Log in to GHCR
        if: github.event_name == 'push' && github.repository == 'Kpa-clawbot/CoreScope'
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Extract Docker metadata
        if: github.event_name == 'push' && github.repository == 'Kpa-clawbot/CoreScope'
        id: docker-meta
        uses: docker/metadata-action@v5
        with:
          images: ghcr.io/kpa-clawbot/corescope
          tags: |
            type=semver,pattern=v{{version}}
            type=semver,pattern=v{{major}}.{{minor}}
            type=semver,pattern=v{{major}}
            type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v') }}
            type=edge,branch=master

      - name: Build and push to GHCR
        if: github.event_name == 'push' && github.repository == 'Kpa-clawbot/CoreScope'
        uses: docker/build-push-action@v6
        with:
          context: .
          push: true
          platforms: linux/amd64,linux/arm64
          tags: ${{ steps.docker-meta.outputs.tags }}
          labels: ${{ steps.docker-meta.outputs.labels }}
          build-args: |
            APP_VERSION=${{ steps.meta.outputs.app_version }}
            GIT_COMMIT=${{ steps.meta.outputs.git_commit }}
            BUILD_TIME=${{ steps.meta.outputs.build_time }}
          cache-from: type=gha
          cache-to: type=gha,mode=max

  # ───────────────────────────────────────────────────────────────
  # 4. Release Artifacts (tags only)
  # ───────────────────────────────────────────────────────────────
  release-artifacts:
    name: "📦 Release Artifacts"
    # Fork guard: release uploads only on the upstream repository.
    if: startsWith(github.ref, 'refs/tags/v') && github.repository == 'Kpa-clawbot/CoreScope'
    needs: [go-test]
    runs-on: ubuntu-latest
    permissions:
      contents: write
    steps:
      - name: Checkout code
        uses: actions/checkout@v5

      - name: Set up Go 1.27.1
        uses: actions/setup-go@v6
        with:
          go-version: '1.27.1'

      - name: Build corescope-decrypt (static, linux/amd64)
        run: |
          cd cmd/decrypt
          CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-s -w -X main.version=${{ github.ref_name }}" -o ../../corescope-decrypt-linux-amd64 .

      - name: Build corescope-decrypt (static, linux/arm64)
        run: |
          cd cmd/decrypt
          CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -ldflags="-s -w -X main.version=${{ github.ref_name }}" -o ../../corescope-decrypt-linux-arm64 .

      - name: Upload release assets
        uses: softprops/action-gh-release@v2
        with:
          files: |
            corescope-decrypt-linux-amd64
            corescope-decrypt-linux-arm64

  # ───────────────────────────────────────────────────────────────
  # 4b. Deploy Staging (master only)
  # ───────────────────────────────────────────────────────────────
  deploy:
    name: "🚀 Deploy Staging"
    # Fork guard (last clause): never scheduled on a fork, so no fork run can
    # reach the self-hosted staging runner.
    if: |
      (github.event_name == 'push' || github.event_name == 'workflow_dispatch')
      && github.ref == 'refs/heads/master'
      && github.repository == 'Kpa-clawbot/CoreScope'
    needs: [build-and-publish]
    runs-on: [self-hosted, meshcore-runner-2]
    steps:
      - name: Checkout code
        uses: actions/checkout@v5

      - name: Pull latest image from GHCR
        run: |
          # Try to pull the edge image from GHCR and tag for docker-compose compatibility
          if docker pull ghcr.io/kpa-clawbot/corescope:edge; then
            docker tag ghcr.io/kpa-clawbot/corescope:edge corescope-go:latest
            echo "Pulled and tagged GHCR edge image ✅"
          else
            echo "⚠️ GHCR pull failed — falling back to locally built image"
          fi

      - name: Deploy staging
        run: |
          # Force-remove the staging container regardless of how it was created
          # (compose-managed OR manually created via docker run)
          docker stop corescope-staging-go 2>/dev/null || true
          docker rm -f corescope-staging-go 2>/dev/null || true
          docker compose -f "$STAGING_COMPOSE_FILE" -p corescope-staging down --timeout 30 2>/dev/null || true

          # Wait for container to be fully gone and OS to reclaim memory (3GB limit)
          for i in $(seq 1 15); do
            if ! docker ps -a --format '{{.Names}}' | grep -q 'corescope-staging-go'; then
              break
            fi
            sleep 1
          done
          sleep 5  # extra pause for OS memory reclaim

          # Ensure staging data dir exists (config.json lives here, no separate file mount)
          STAGING_DATA="${STAGING_DATA_DIR:-$HOME/meshcore-staging-data}"
          mkdir -p "$STAGING_DATA"

          # If no config exists, copy the example (CI doesn't have a real prod config)
          if [ ! -f "$STAGING_DATA/config.json" ]; then
            echo "Staging config missing — copying config.example.json"
            cp config.example.json "$STAGING_DATA/config.json" 2>/dev/null || true
          fi

          docker compose -f "$STAGING_COMPOSE_FILE" -p corescope-staging up -d staging-go

      - name: Healthcheck staging container
        run: |
          for i in $(seq 1 120); do
            HEALTH=$(docker inspect corescope-staging-go --format '{{.State.Health.Status}}' 2>/dev/null || echo "starting")
            if [ "$HEALTH" = "healthy" ]; then
              echo "Staging healthy after ${i}s"
              break
            fi
            if [ "$i" -eq 120 ]; then
              echo "Staging failed health check after 120s"
              docker logs corescope-staging-go --tail 50
              exit 1
            fi
            sleep 1
          done

      - name: Smoke test staging API
        run: |
          PORT="${STAGING_GO_HTTP_PORT:-80}"
          if curl -sf "http://localhost:${PORT}/api/stats" | grep -q engine; then
            echo "Staging verified — engine field present ✅"
          else
            echo "Staging /api/stats did not return engine field (port ${PORT})"
            exit 1
          fi

      - name: Clean up old Docker images
        if: always()
        run: |
          # Remove dangling images and images older than 24h (keeps current build)
          echo "--- Docker disk usage before cleanup ---"
          docker system df
          docker image prune -af --filter "until=24h" 2>/dev/null || true
          docker builder prune -f --keep-storage=1GB 2>/dev/null || true
          echo "--- Docker disk usage after cleanup ---"
          docker system df

  # ───────────────────────────────────────────────────────────────
  # 5. Publish Badges & Summary (master only)
  # ───────────────────────────────────────────────────────────────
  publish:
    name: "📝 Publish Badges & Summary"
    # Fork guard: badge files are committed to master only on the upstream repository.
    if: github.event_name == 'push' && github.repository == 'Kpa-clawbot/CoreScope'
    needs: [deploy]
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v5

      - name: Download Go coverage badges
        continue-on-error: true
        uses: actions/download-artifact@v6
        with:
          name: go-badges
          path: .badges/

      - name: Download E2E badges
        continue-on-error: true
        uses: actions/download-artifact@v6
        with:
          name: e2e-badges
          path: .badges/

      - name: Publish coverage badges to repo
        continue-on-error: true
        env:
          GH_TOKEN: ${{ secrets.BADGE_PUSH_TOKEN }}
        run: |
          # GITHUB_TOKEN cannot push to protected branches (required status checks).
          # Use admin PAT (BADGE_PUSH_TOKEN) via GitHub Contents API instead.
          for badge in .badges/*.json; do
            FILENAME=$(basename "$badge")
            FILEPATH=".badges/$FILENAME"
            CONTENT=$(base64 -w0 "$badge")
            CURRENT_SHA=$(gh api "repos/${{ github.repository }}/contents/$FILEPATH" --jq '.sha' 2>/dev/null || echo "")
            if [ -n "$CURRENT_SHA" ]; then
              gh api "repos/${{ github.repository }}/contents/$FILEPATH" \
                -X PUT \
                -f message="ci: update $FILENAME [skip ci]" \
                -f content="$CONTENT" \
                -f sha="$CURRENT_SHA" \
                -f branch="master" \
                --silent 2>&1 || echo "Failed to update $FILENAME"
            else
              gh api "repos/${{ github.repository }}/contents/$FILEPATH" \
                -X PUT \
                -f message="ci: update $FILENAME [skip ci]" \
                -f content="$CONTENT" \
                -f branch="master" \
                --silent 2>&1 || echo "Failed to create $FILENAME"
            fi
          done
          echo "Badge publish complete"

      - name: Post deployment summary
        run: |
          echo "## Staging Deployed ✓" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "**Commit:** \`$(git rev-parse --short HEAD)\` — $(git log -1 --format=%s)" >> $GITHUB_STEP_SUMMARY