diff --git a/.gitattributes b/.gitattributes index dfe07704..82303ef6 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,5 @@ # Auto detect text files and perform LF normalization * text=auto + +# Bench setup scripts run as POSIX sh on the Pi; CRLF would break the shebang. +dev/hil/*.sh text eol=lf diff --git a/.github/workflows/build_release.yml b/.github/workflows/build_release.yml index d2f7da51..763363ce 100644 --- a/.github/workflows/build_release.yml +++ b/.github/workflows/build_release.yml @@ -15,10 +15,10 @@ jobs: pull-requests: write steps: - name: Check out code - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Set up Python - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.10" @@ -71,7 +71,7 @@ jobs: - name: Upload update artifacts if: ${{ github.event_name == 'pull_request' }} - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: update-files path: pr-artifacts @@ -88,7 +88,7 @@ jobs: - name: Publish raw update files if: ${{ github.event_name == 'pull_request' }} id: publish_raw_updates - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 with: # GitHub's git-data API intermittently returns 5xx; retry instead # of failing the whole build on a transient server error. @@ -227,7 +227,7 @@ jobs: - name: Comment with update artifact links if: ${{ github.event_name == 'pull_request' }} - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 env: RAW_LINKS_JSON: ${{ steps.publish_raw_updates.outputs.links }} with: @@ -347,7 +347,7 @@ jobs: - name: Create or update GitHub Release if: ${{ env.SHOULD_RELEASE == 'true' }} id: create_release - uses: softprops/action-gh-release@a74c6b72af54cfa997e81df42d94703d6313a2d0 # v2.0.6 + uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2.6.2 with: tag_name: ${{ env.VERSION }} name: ${{ env.VERSION }} diff --git a/.github/workflows/deploy_docs.yml b/.github/workflows/deploy_docs.yml index d387bc7b..d262ef0c 100644 --- a/.github/workflows/deploy_docs.yml +++ b/.github/workflows/deploy_docs.yml @@ -13,10 +13,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.11" diff --git a/.github/workflows/docs-on-pr.yml b/.github/workflows/docs-on-pr.yml index dcf62faf..5c140016 100644 --- a/.github/workflows/docs-on-pr.yml +++ b/.github/workflows/docs-on-pr.yml @@ -15,14 +15,14 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: repository: ${{ github.event.pull_request.head.repo.full_name }} fetch-depth: 0 ref: ${{ github.head_ref }} - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: '3.11' diff --git a/.github/workflows/hil-flash-check.yml b/.github/workflows/hil-flash-check.yml new file mode 100644 index 00000000..a5948dd3 --- /dev/null +++ b/.github/workflows/hil-flash-check.yml @@ -0,0 +1,87 @@ +name: HIL flash and health check + +# Builds firmware for each attached board, flashes it, and health-checks the +# API over both USB and HTTP once the board boots. +# +# Deliberately no `pull_request` trigger — this targets a self-hosted runner on +# a private network and physically reflashes hardware. Fork gating is designed +# in dev/hil/DESIGN.md §4 and is not built yet. +# +# The `push` trigger exists so this can be validated before merging, since a +# workflow_dispatch workflow is not dispatchable until it reaches the default +# branch. Drop the push trigger once this is on main. + +on: + workflow_dispatch: + inputs: + skip_http: + description: "USB checks only (skip the network stack)" + type: boolean + default: false + skip_flash: + description: "Health-check what is already flashed, do not reflash" + type: boolean + default: false + inventory_only: + description: "Just print each board's chip id, for building VECTOR_HIL_BOARD_MAP" + type: boolean + default: false + identify: + description: "Blink each board in turn so you can see which physical board is which" + type: boolean + default: false + push: + branches: + - claude/hil-testing-design-s564ln + paths: + - dev/hil/flash_and_check.py + - .github/workflows/hil-flash-check.yml + +permissions: + contents: read + +# Flashing is destructive and the bench is one set of boards. Never interleave. +concurrency: + group: hil-bench + cancel-in-progress: false + +jobs: + flash-and-check: + runs-on: [self-hosted, vector-hil] + timeout-minutes: 45 + + steps: + # actions/checkout is safe here ONLY because every trigger above is + # repo-internal. If a fork-reachable trigger is ever added, this must go + # back to running from the runner's pinned clone ($VECTOR_HIL_REPO) — + # otherwise a fork PR would execute its own harness code on the bench. + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + + - name: Prepare environment + run: | + test -x "$VECTOR_HIL_VENV/bin/python" || { echo "bench venv missing"; exit 1; } + echo "$VECTOR_HIL_VENV/bin" >> "$GITHUB_PATH" + + # VECTOR_HIL_BOARD_MAP, VECTOR_HIL_WIFI_* and friends arrive from the + # runner's .env, which the runner process exports into every job. They + # are deliberately NOT restated in an `env:` block here: the `env` + # context only covers workflow/job/step-level vars, so `${{ env.X }}` + # would evaluate to empty and shadow the real value. + - name: Flash and health-check every board + run: | + args="" + if [ "${{ inputs.skip_http }}" = "true" ]; then args="$args --skip-http"; fi + if [ "${{ inputs.skip_flash }}" = "true" ]; then args="$args --skip-flash"; fi + if [ "${{ inputs.inventory_only }}" = "true" ]; then args="$args --inventory-only"; fi + if [ "${{ inputs.identify }}" = "true" ]; then args="$args --identify"; fi + python dev/hil/flash_and_check.py $args + + - name: Board serial logs on failure + if: failure() + run: | + # A board that failed its health check may still be printing something + # useful; grab a few seconds of console from each. + for dev in /dev/ttyACM*; do + echo "--- $dev" + timeout 8 cat "$dev" || true + done diff --git a/.github/workflows/hil-smoke.yml b/.github/workflows/hil-smoke.yml new file mode 100644 index 00000000..ca4c9cbc --- /dev/null +++ b/.github/workflows/hil-smoke.yml @@ -0,0 +1,135 @@ +name: HIL smoke + +# Proves the bench runner is wired up correctly: the runner picks up jobs, the +# bench environment reaches them, the boards enumerate, and each one answers +# over serial. It does not test firmware — see dev/hil/DESIGN.md for that. +# +# Deliberately no `pull_request` trigger. This targets a self-hosted runner on +# a private network, and a pull_request trigger would let any fork PR run code +# against it. Fork gating is designed in DESIGN.md §4 and is not built yet. +# +# The `push` trigger exists so this can be validated before merging, since a +# workflow_dispatch workflow is not dispatchable until it reaches the default +# branch. Drop the push trigger once this is on main. + +on: + workflow_dispatch: + push: + branches: + - claude/hil-testing-design-s564ln + +permissions: {} + +# The bench is a single set of physical boards. Queue, never interleave. +concurrency: + group: hil-bench + cancel-in-progress: false + +jobs: + smoke: + runs-on: [self-hosted, vector-hil] + timeout-minutes: 15 + + # No actions/checkout on purpose: under the design's trust model the bench + # runs harness code from a trusted checkout the runner already has, not + # from the PR. $VECTOR_HIL_REPO points at it. + + steps: + - name: Runner host + run: | + echo "user: $(whoami)" + echo "kernel: $(uname -srm)" + echo "arch: $(dpkg --print-architecture)" + free -h | awk '/Mem:/ {print "memory: " $2 " total, " $7 " available"}' + echo "uptime:$(uptime -p | sed 's/^up//')" + + - name: Bench environment + run: | + fail=0 + for var in VECTOR_HIL_REPO VECTOR_HIL_VENV VECTOR_HIL_WIFI_SSID VECTOR_HIL_WIFI_PASSWORD; do + eval "val=\${$var:-}" + if [ -z "$val" ]; then + echo "MISSING: $var" + fail=1 + else + echo "set: $var" + fi + done + [ "$fail" -eq 0 ] || { + echo "The runner's .env is incomplete. It is read at service start:" + echo " cd ~/actions-runner && sudo ./svc.sh stop && sudo ./svc.sh start" + exit 1 + } + test -x "$VECTOR_HIL_VENV/bin/python" || { echo "venv python missing at $VECTOR_HIL_VENV"; exit 1; } + test -d "$VECTOR_HIL_REPO/.git" || { echo "no clone at $VECTOR_HIL_REPO"; exit 1; } + + # dev/detect_boards.py shells out to a bare `mpremote` from PATH and + # silently returns {} when it is missing, so the venv must be ON PATH + # for later steps - invoking its python by absolute path is not enough. + echo "$VECTOR_HIL_VENV/bin" >> "$GITHUB_PATH" + + - name: Serial access + run: | + if ! ls /dev/ttyACM* >/dev/null 2>&1; then + echo "no /dev/ttyACM* devices - check the USB hub and power" + exit 1 + fi + ls -l /dev/ttyACM* + id -nG | grep -qw dialout || { + echo "runner user is not in the dialout group; reboot or restart the service" + exit 1 + } + + - name: mpremote sees the boards + run: | + # Checked separately because detect_boards.py collapses "mpremote is + # missing", "mpremote failed", and "no boards attached" into the same + # empty result, which makes a real failure indistinguishable from an + # empty bench. + command -v mpremote || { echo "mpremote not on PATH"; exit 1; } + mpremote version + + echo "--- mpremote devs ---" + mpremote devs | tee /tmp/devs.txt + test -s /tmp/devs.txt || { + echo "mpremote sees no devices, though /dev/ttyACM* exist." + echo "Boards may be busy or in an odd state; power-cycle the hub and retry." + exit 1 + } + + - name: Detect and query boards + run: | + # detect_boards.py prints progress text before the JSON, so the + # machine-readable part is the last line only. + raw=$(python "$VECTOR_HIL_REPO/dev/detect_boards.py") + echo "$raw" + json=$(printf '%s\n' "$raw" | tail -1) + + python - "$json" <<'PY' + import json, subprocess, sys + + boards = json.loads(sys.argv[1]) + + if not boards: + sys.exit("no boards detected") + + rc = 0 + for kind, ports in sorted(boards.items()): + for port in ports: + probe = subprocess.run( + ["mpremote", "connect", port, "exec", + "import systemConfig; print(systemConfig.vectorSystem, systemConfig.SystemVersion)"], + capture_output=True, text=True, timeout=60) + if probe.returncode: + print(f"FAIL {kind:12} {port:16} {probe.stderr.strip()}") + rc = 1 + else: + print(f"ok {kind:12} {port:16} {probe.stdout.strip()}") + + print(f"\n{sum(len(p) for p in boards.values())} board(s) responding") + sys.exit(rc) + PY + + - name: Summary + if: always() + run: echo "Bench smoke ${{ job.status }}" diff --git a/.github/workflows/specialfeatures-on-pr.yml b/.github/workflows/specialfeatures-on-pr.yml index b660928c..c13866f6 100644 --- a/.github/workflows/specialfeatures-on-pr.yml +++ b/.github/workflows/specialfeatures-on-pr.yml @@ -16,14 +16,14 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: repository: ${{ github.event.pull_request.head.repo.full_name }} fetch-depth: 0 ref: ${{ github.head_ref }} - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: '3.11' diff --git a/.github/workflows/validate-json-configs.yml b/.github/workflows/validate-json-configs.yml index a2525152..ea9eb2b5 100644 --- a/.github/workflows/validate-json-configs.yml +++ b/.github/workflows/validate-json-configs.yml @@ -14,10 +14,10 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.11" diff --git a/.github/workflows/version-bump-guard.yml b/.github/workflows/version-bump-guard.yml index 5a6976b3..df1894ce 100644 --- a/.github/workflows/version-bump-guard.yml +++ b/.github/workflows/version-bump-guard.yml @@ -16,14 +16,14 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout PR branch - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: repository: ${{ github.event.pull_request.head.repo.full_name }} fetch-depth: 0 ref: ${{ github.head_ref }} - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.11" @@ -59,4 +59,4 @@ jobs: git config user.email "41898282+github-actions[bot]@users.noreply.github.com" git add src/common/SharedState.py src/em/systemConfig.py src/sys11/systemConfig.py src/wpc/systemConfig.py src/data_east/systemConfig.py src/whitestar/systemConfig.py src/classic/systemConfig.py git commit -m "ci: auto-bump required patch versions" - git push origin HEAD:${{ github.head_ref }} \ No newline at end of file + git push origin HEAD:${{ github.head_ref }} diff --git a/dev/hil/DESIGN.md b/dev/hil/DESIGN.md new file mode 100644 index 00000000..cf0466b9 --- /dev/null +++ b/dev/hil/DESIGN.md @@ -0,0 +1,495 @@ +# Hardware-in-the-Loop (HIL) Testing — Design + +**Status:** design proposal; bench bring-up and the flash/health-check harness are implemented and running +**Scope:** a self-hosted GitHub Actions runner driving real Vector boards, safely, from a public repository. + +--- + +## 1. Goals + +The first four things we want covered: + +| # | Goal | Tier | +|---|---|---| +| G1 | The code boots and the boards run normally | every PR | +| G2 | We can connect to the boards and hit the API | every PR | +| G3 | Every available config can be parsed and boot | every PR | +| G4 | Updating from the last 5 versions to the proposed version works | every PR | + +G4 runs on every PR by decision: the update path is critical infrastructure, it is where we most suspect latent problems, and we would rather pay the wall-clock cost than find out at release time. §8 quantifies the cost and §11 lists the levers to dial it back if it becomes painful. + +Non-goals for v1: gameplay/bus-level correctness (bare boards, no machine attached), AP-mode setup flow (needs a wire to GPIO22), long-duration soak testing. + +## 2. Decisions taken + +| Decision | Choice | +|---|---| +| Harness location | All in `warped-pinball/vector`. Bench topology and credentials live on the Pi, not in git. | +| Bench hardware | Bare boards, no machine or bus emulator attached. | +| Board networking | Dedicated VLAN on the existing LAN. | +| Trigger policy | Automatic for branches in `warped-pinball/vector`; forks require a maintainer gate. | +| Board recovery | Software reset only (`machine.reset()` over `mpremote`). | +| Update signing | `skip_signature_check: true` for the upgrade path, plus negative tests that the signature gate still rejects bad packages. | +| Initial inventory | `sys11`, `wpc`, `data_east`. More systems added later. | +| Update matrix cadence | Every PR. Updates are the priority; dial back later only if wall clock becomes a problem. | + +--- + +## 3. Threat model + +The repo is public and anyone can open a PR. A PR is, by construction, *untrusted code that we want to run on hardware*. The design has to make that safe rather than avoid it. + +### What we are protecting + +1. **The LAN.** Anything reachable from the runner or the boards. +2. **`WARPED_PINBALL_PRIVATE_KEY`.** This is the highest-value asset in the org. The matching public key is hardcoded in `src/common/update.py:107`, so a leak means an attacker can sign an update that every Vector in the field will accept. It must never be reachable from a job that runs on the self-hosted runner. +3. **The runner host** and its registration token. +4. **The boards** — recoverable, and the least of the four. + +### Attack surfaces, ranked + +**(a) Host-side code execution on the Pi.** The big one, and it is broader than it looks. Any of these executes PR-authored code on the runner: + +- an obvious `run: python dev/build.py` +- `pip install -r dev/requirements.txt` — a PR can repoint a requirement at a malicious package, or add a `setup.py` that runs at install time +- `pre-commit` hooks — the config names arbitrary repos and revisions +- **`pytest` collection** — `conftest.py` is imported automatically before any test runs. If the harness ever runs from a PR checkout, a PR that only adds a `conftest.py` gets code execution with zero test code executed +- workflow files themselves, if the workflow runs from the PR ref + +The mitigation is a single rule, and everything else follows from it: + +> **The Pi never checks out or executes PR-authored files. It executes harness code from a trusted ref, and consumes only build *artifacts* from the PR.** + +**(b) Board-side code execution.** Intentional — that's the test. `update.py:write_files()` honors `"execute": true` per file and `__import__`s it, so an update package is arbitrary code by design. Contained by assuming every board on the bench is fully compromised at all times, and isolating it accordingly. + +**(c) Runner persistence between jobs.** A compromised job leaves something behind for the next one. Mitigated with ephemeral runners and job hooks that wipe state. + +**(d) Secret exfiltration.** Mitigated by putting no secrets on the HIL workflow at all. + +### A note on the repo question + +Putting the harness in a separate repo would **not**, by itself, make any of this safer. A separate repo whose workflow still checked out the PR and ran `pytest` on the Pi is exactly as compromised as an in-repo one. The property that makes this safe is *trusted-ref execution*, not the repo boundary. + +So the repo choice is an ergonomics decision, and `vector` is the right home: the harness tests the API, and the API changes in this repo. Keeping them together means an endpoint change and its test ship in one PR and can't drift. The one carve-out is that the *instantiated* bench manifest — serial numbers, IPs, game passwords, WiFi credentials — stays on the Pi. The repo carries the schema and an example, not the instance. That is a "don't publish a map of the lab" measure, not a security boundary. + +--- + +## 4. Execution model + +Two stages, split by trust. + +### Stage A — build (untrusted, GitHub-hosted) + +This already exists: `.github/workflows/build_release.yml` runs on `pull_request`, builds all seven targets, and uploads a `update-files` artifact. Two additions needed: + +1. Also upload the full `build/` tree per target (tarball). The update packages are only good for OTA; flashing a board from scratch needs the file tree. +2. Write a `pr_meta.json` into the artifact: PR number, head SHA, head repo full name, and the per-target versions. Stage B needs these and cannot reliably get them from the event payload for fork PRs. + +No secrets, no self-hosted labels, unchanged trust posture. + +### Stage B — hardware (trusted, self-hosted) + +A new `.github/workflows/hil.yml`, triggered by **`workflow_run`** on completion of "Build and Deploy". + +`workflow_run` is the key primitive. GitHub always runs a `workflow_run` workflow *from the default branch, using the default branch's code*, regardless of what the triggering PR contains. A PR therefore cannot modify `hil.yml`, `dev/hil/**`, `conftest.py`, or the pinned dependency set that the Pi will execute. It gets to supply exactly one thing: the artifact bytes. + +``` +pull_request (fork or branch) + │ + ▼ + Build and Deploy ── GitHub-hosted, untrusted code, no secrets + │ artifact: update packages + build trees + pr_meta.json + ▼ + workflow_run: completed + │ + ├─ gate job (ubuntu-latest): decide auto vs. approval + │ head repo == warped-pinball/vector → proceed + │ fork → environment `hardware-lab`, + │ required reviewers, job blocks + ▼ + hil job (runs-on: [self-hosted, vector-hil]) + │ workflow + harness from default branch ← trusted + │ artifact downloaded from the build run ← untrusted payload only + ▼ + boards on isolated VLAN + │ + ▼ + check run posted against the PR head SHA +``` + +**Gating.** Auto for same-repo branches, approval for forks. Two mechanisms, both auditable: + +- a `hardware-lab` GitHub Environment with required reviewers — the job literally pauses until a maintainer clicks approve, and the approval is logged +- a `safe-to-hil` label as a secondary signal, **auto-removed on every new push** by a tiny `pull_request.synchronize` workflow, so a contributor can't get a clean diff approved and then push a dirty one + +Belt and braces, but the label alone is not enough — label state and head SHA can desync — and the environment gate alone gives no signal on the PR itself. + +**Reporting.** `workflow_run` jobs have write permissions, so the result is posted as a check run keyed to the PR's head SHA via the Checks API. That's what makes it eligible to be a required check later. + +--- + +## 5. Physical and network layout + +### Runner host + +The existing Pi already runs the Actions runner with a USB hub attached, so the hardware question is largely settled. Worth noting for the record: **that Pi is a Zero 2 W, not a Zero 1 W.** The GitHub Actions runner ships only for Linux x64, ARM64, and ARM32 (ARMv7); the original Zero/Zero W is ARMv6 (`ARM1176`) and the runner will not start on it at all. A runner that runs is a Zero 2 W (Cortex-A53). No need to verify further — it's proven by the fact that it works. + +Constraints to design around on that hardware: + +- **512 MB RAM** shared between the runner, Python, and three concurrent `mpremote` sessions. Workable, but keep per-board test processes lean and avoid loading whole build trees into memory. Swapping to the SD card will hurt badly and will show up as timing flake. +- **Single-core-ish throughput.** Board work is I/O-bound on serial round-trips rather than CPU-bound, which is what makes 3-way parallelism viable at all — but orchestration overhead is not free at this scale. +- **USB hub already present.** Confirm it is self-powered; three Picos plus enumeration churn on a Zero 2 W's OTG port is more than the bus-powered case wants to supply. + +If the combined per-PR matrix (§8) turns out too slow, the first thing to try is more boards rather than a bigger Pi — the work is serial-I/O-bound, not compute-bound, so a faster host buys much less than a second WPC board does. + +### Board network + +Boards live on a dedicated VLAN. Rules: + +- **Allow:** board subnet → Pi's HTTP file server (serves update packages for OTA tests). +- **Allow:** intra-VLAN broadcast, so `discovery.py` peer discovery is actually exercised. +- **Deny:** board subnet → rest of the LAN. +- **Deny:** board subnet → internet. +- The Pi needs an interface on the VLAN (tagged sub-interface on a USB ethernet adapter, or a second SSID) *plus* uplink to github.com. +- Firewall the runner from the rest of the LAN too. It is a CI box that pulls untrusted artifacts; it is not a trusted host. + +One wrinkle: `/api/update/check` fetches `http://software.warpedpinball.com/vector//latest.json` (`src/*/systemConfig.py:2`). Rather than punching an egress hole, **override that hostname in DNS on the VLAN to point at the Pi** and serve a canned `latest.json`. That keeps egress at zero and makes the update-check test deterministic — you can assert behavior against a pinned "latest" instead of whatever is live. + +### Board addressing + +`/dev/ttyACM*` ordering is not stable across reboots or re-enumeration. Use udev rules keyed on each Pico's unique USB serial number to get stable paths: + +``` +/dev/vector-sys11-a +/dev/vector-wpc-a +/dev/vector-data-east-a +``` + +Static DHCP leases on the VLAN, keyed by MAC, for the HTTP side. + +--- + +## 6. Bench manifest + +The inventory is going to grow (`em`, `whitestar`, `classic`, and probably a second WPC board). Nothing about board count or identity should be hardcoded. + +`dev/hil/bench.example.yaml` in the repo; the real one at `/etc/vector-hil/bench.yaml` on the Pi, located via `$VECTOR_HIL_BENCH`. + +```yaml +boards: + - id: sys11-a + target: sys11 # matches an id in dev/ci/targets.json + hardware: sys11 # build tree to flash + serial: /dev/vector-sys11-a + ip: 10.42.7.11 + - id: wpc-a + target: wpc + hardware: wpc + serial: /dev/vector-wpc-a + ip: 10.42.7.12 + - id: data-east-a + target: data_east + hardware: data_east + serial: /dev/vector-data-east-a + ip: 10.42.7.13 + +network: + update_server: http://10.42.7.1:8080 +secrets_file: /etc/vector-hil/secrets.yaml # game password, wifi creds +``` + +Tests parameterize over `manifest ∩ dev/ci/targets.json`. Adding `whitestar` later is a udev rule plus three lines of YAML — no test changes. Multiple boards with the same `target` are treated as a pool and sharded across, which is how you buy throughput later. + +`sys11_tiny` is the same hardware as `sys11`, so it runs as a second firmware pass on the `sys11-a` board rather than needing its own. + +--- + +## 7. Harness structure + +``` +dev/hil/ + bench.py # manifest load + validation + board.py # Board: transports, reset, wait_for_boot, flash, seed_config + transports/ + usb.py # JSON-over-serial client for usb_comms.py + http.py # HTTP client with challenge/HMAC auth + flashing.py # wipe + mpremote fs cp from a build tree; OTA via /api/update/apply + update_server.py # local HTTP server for OTA payloads + conftest.py # fixtures: board pool, per-test lease, artifact discovery + tests/ + test_boot.py + test_api_contract.py + test_configs.py + test_update_matrix.py + bench.example.yaml + README.md +``` + +### Two transports, and an important asymmetry + +Vector exposes the same routes over HTTP (phew) and over USB serial (`usb_comms.py` replays into `phew.server._routes`). Running tests over both catches regressions in either bridge. + +But note `backend.py:280-287`: **the USB transport bypasses authentication entirely.** Requests tagged `is_usb_transport` skip the HMAC check. Two consequences: + +- **Auth tests must run over HTTP.** Over USB every route is reachable unauthenticated, so USB can't tell you anything about the auth layer. +- **USB is a convenient privileged setup channel.** Seeding config, scores, and adjustment profiles for a test doesn't need credentials at all — just drive it over serial. That simplifies fixtures considerably. + +The HTTP client mirrors `src/common/web/js/utils.js:64-72`: `GET /api/auth/challenge`, then `hmac_sha256(password, challenge + path + query + body)` hex-encoded into `X-Auth-HMAC`, with the challenge in `X-Auth-challenge`. + +--- + +## 8. Test design + +### G1 — boots and runs normally + +Per board, per target firmware: wipe → flash build tree via `mpremote` → seed bench config → `machine.reset()` → capture serial console. + +Assertions: + +- Banner and `Version ` on the console within the timeout +- `/api/version` matches the version built from the PR head SHA +- `/api/fault` contains only bench-expected faults (see below) +- No `SFTW01` (drop-through) — that fault means `backend.go()` returned, which it never should +- **Soak, 60–120s:** poll `/api/memory-snapshot` and `/api/game/status`. Assert free memory doesn't trend toward zero and the scheduler still answers. This catches `SFTW02: async loop interrupted` and slow leaks, which are exactly the failures a single boot check misses. +- `/api/wifi/status` connected with the expected IP; `/api/network/peers` sees the other bench boards (exercises `discovery.py`) + +#### ⚠️ Bare-board boot behavior — characterize this first + +This is the most likely source of a flaky suite, and it needs measuring during bring-up before any test is written. + +`main.py:44-66` runs `bus_activity_fault_check()`: it samples GPIO14–21 (the data lines) for 800 ms and raises **`HDWR01: Early Bus Activity`** if it counts more than 250 transitions. On a bare board those eight pins are floating inputs. If they pick up enough noise to cross the threshold, the board takes a completely different boot path — `GameDefsLoad.go(safe_mode=True)`, `MemoryMain` skipped, error LED — and every downstream assertion changes meaning. + +Separately, `adr_activity_ok()` raises **`HDWR02: No Bus Activity`** when shadow RAM lamp columns don't change, which on a bare board is the correct and permanent state. + +**Measured on the bench (2026-08-17), and both predictions were wrong:** + +- `HDWR01` did **not** fire. Three boards, freshly flashed for their real targets, all booted + with `faults: none`. The floating-pin worry did not materialise — no resistor pack needed, + and the config assertions are not silently vacuous. +- `HDWR02` did not fire either, and cannot: **`adr_activity_ok()` is defined but never called** + (`src/common/main.py:69`, `src/data_east/main.py:82`). The fault is unreachable code. + +That second point is a finding about the firmware rather than the bench. `HDWR02: No Bus +Activity` is exactly the diagnostic a customer with a dead bus needs, and today nothing can +raise it. Worth a separate issue: either wire the check into the boot path or drop the fault +code, but leaving a dead diagnostic in `faults.py` is the worst of both. + +The allowlist keeps tolerating `HDWR02` for now — harmless, and it costs nothing if the check +is ever reconnected. `HDWR01` stays a warning that suppresses the active-config assertion, +since if it ever does fire the board is in safe mode and the config genuinely was not loaded. + +Bring-up task: boot each bare board 50 times, record the fault set each time, and confirm it is identical every time. Until that holds, nothing else is worth automating. + +### G2 — connect and hit the API + +**Generated contract tests.** `tools/gen_api_docs.py` already parses the `@api` docstrings out of `backend.py`. Reuse that parser as the test inventory rather than hand-maintaining a list: + +- every documented unauthenticated route returns 200 and parseable JSON whose top-level shape matches the documented example +- every `auth: true` route returns 401 over HTTP without credentials, and succeeds with them +- **every route the server actually registers is documented** — walk `phew.server._routes` over the REPL and diff against the parsed docs + +That last one is the valuable one. It turns the API docs into a load-bearing artifact and makes "shipped an undocumented endpoint" a build failure instead of a discovery. + +**Auth behavior** (HTTP only): wrong password rejected; a replayed challenge rejected (`backend.py` deletes the challenge on use — worth a regression test); expired challenge rejected; more than 10 outstanding challenges returns 429. + +**Round-trip state:** set an adjustment profile name → read back; import scores → export → compare; set tournament mode → reset → still set (proves FRAM persistence, which is what update tests depend on later). + +**Static serving:** `/`, `/index.html`, gzip `Content-Encoding`, and the ETag/304 path. + +**Known gap:** AP-mode setup can't be tested. `check_ap_button()` reads GPIO22, and with software-only control we can't hold it. One wire per board from GPIO22 to a Pi GPIO would unlock it; worth doing only if AP-mode regressions start to bite. + +### G3 — every config parses and boots + +Full matrix on every PR, per the decision above. Per board, loop over that hardware's configs: + +1. Interrupt to REPL, write `gamename` into `SPI_DataStore` `configuration` record, `machine.reset()` +2. Wait for boot +3. Assert: + - no `CONF00` / `CONF01` fault + - `/api/game/active_config` is the config we set + - `/api/game/name` matches `GameInfo.GameName` **from the source JSON in the repo** — this cross-checks the on-board `config/all.jsonl.z` against the source and catches build-time config-packing bugs, not just parse errors + - `/api/leaders` and `/api/adjustments/status` both return 200 — proves the parsed definition is *usable*, not merely loadable + - free memory after load is above a floor + +That last assertion is the one that earns its keep. `sys11_tiny` exists because RAM is tight; a config that parses fine but leaves too little heap is the failure that actually reaches customers. + +**Throughput.** Measured on the bench, per board, from the flash/health-check harness: + +| Stage | sys11 | wpc | data_east | +|---|---|---|---| +| build | 13.7s | 21.7s | 19.8s | +| flash (wipe + copy + config + reset) | 26.4s | 28.0s | 22.5s | +| boot → API answering + full health check | 11.7s | 14.8s | 10.7s | + +So a full flash-and-verify of all three boards is **~3.5 min end to end**, and build cost is +not a concern on the Zero 2 W — the caching this design worried about is unnecessary. + +The number that drives the config matrix is the last row: a board answers its API within +roughly 10s of reset, and the health check itself accounts for most of that 10–15s. The +15–25s per-config estimate below therefore still holds, but is an estimate — a per-config +reboot loop has not been measured yet. + +Boot cycle is roughly 15–25s (`main.py` alone has an 0.8s bus check plus ~4.5s of sleeps +before WiFi comes up), times config count: + +| Board | Configs | Serial estimate | +|---|---|---| +| wpc | 63 | ~16–26 min | +| sys11 | 39 | ~10–16 min | +| data_east | 28 | ~7–12 min | +| em | 1 | trivial | + +Boards run in parallel, so wall clock ≈ the WPC leg, **~16–26 min per PR**. That's a real cost and the bench is a singleton. Recommended mitigations: + +- `concurrency: { group: hil-bench, cancel-in-progress: false }` — queue, don't interleave. Two jobs sharing one bench will corrupt each other's state. +- Path filters so docs-only and workflow-only PRs skip HIL entirely. +- Order the loop so configs touched by the diff run first — fail fast on the likely culprit. +- Poll readiness over **USB rather than HTTP**; the USB bridge is answering well before WiFi associates, which shaves seconds off every one of 130 iterations. +- When it gets annoying, add a second WPC board. The manifest already supports pools, and the runner shards across them — that alone roughly halves wall clock. + +### G4 — update from the last 5 versions + +Matrix: each board target × the last 5 **stable** releases. Note the release list is mostly prereleases (`1.11.28-beta1746` etc. are PR builds); filter to `prerelease == false`, which currently gives `1.11.28`, `1.11.27`, and so on. + +Getting a board *to* an old version is the interesting part. Rather than building historical source — which drags in period-correct `mpy-cross` and MicroPython versions and is genuinely painful: + +1. Flash the current build once. +2. Apply version *V*'s **signed** release asset via `/api/update/apply` with no skip flag. This gets us to V using the real OTA mechanism, and as a bonus exercises the *signed* code path on the way down. +3. Confirm `/api/version == V`. + +Caveat worth stating: that's a downgrade, which no user performs. But the resulting filesystem is exactly what V's update package produces, which is what the upgrade is going to be applied to — so the fidelity that matters is preserved. + +Then the actual test: + +4. Seed realistic state — scores, players, adjustment profiles, settings. `dev/test_data.json` already exists for this and `flash.py --test-data` already knows how to write it. +5. Serve the PR's `-update.json` from the Pi. +6. `POST /api/update/apply` with `skip_signature_check: true`. Consume the streamed progress JSON; assert percent is monotonic and no error lines appear. +7. After reboot, assert: + - version == the PR version + - **all seeded state survived** — scores, players, adjustments, tournament mode, claim methods, WiFi credentials, gamename. Data migration is the real risk in an upgrade, and it's invisible to a version check. + - no faults, API healthy, free memory sane + - **no stale files** — the update runs a `remove_extra_files.py` execute-step, so diff an `ls` over the REPL against the expected build tree. A file the cleanup misses is a file that shadows a module in a later version. + +#### Negative tests + +The update path is the most security-sensitive code we ship, and these are cheap: + +- unsigned PR package applied **without** `skip_signature_check` → must be rejected, **and the board must still boot afterward** +- one byte flipped in the body (hash mismatch) → rejected +- valid hash, garbage signature → rejected +- **interrupted mid-update** — reset the board at ~50% and see what happens. This is the "user pulled the plug" scenario. Either it recovers to something bootable, or we learn precisely how it fails and how to talk a customer through it. Right now nobody knows which. + +The negative tests are what buy back the coverage lost by using `skip_signature_check` for the happy path. + +#### Additional cases worth having, given updates are the suspected problem area + +- **Repeated updates in one power cycle.** Apply an update, then apply another without a power cycle in between. `apply_update()` runs inside `LowMemoryMode`, which halts the phew scheduler and closes the discovery sockets on entry and rebuilds them on exit (`update.py:157-196`). If `__exit__` doesn't fully restore that state, the second update is where it shows. Field users do sometimes update twice in a row. +- **Chained vs. direct upgrades.** The common case is a direct jump from an old version to the newest, and that's what the main matrix covers. A chained walk (V-5 → V-4 → … → proposed) additionally catches migration-ordering bugs, where each individual hop works but the sequence doesn't. Worth running on release tags even if it's too slow for every PR. +- **Bytecode compatibility (see the toolchain note below).** After an update, assert every `.mpy` on the board actually imports. A bytecode-version mismatch produces a board that updates "successfully" and then fails to boot — which looks exactly like a mysterious update bug. + +#### ⚠️ Toolchain finding: `mpy-cross` was unpinned + +Worth surfacing here because it lands squarely in the "we suspect there might be issues with updates" category. + +`dev/requirements.txt` pinned `mpremote==1.23.0` but left **`mpy-cross` unpinned**, so CI resolved whatever was newest — currently `1.28.0.post2`. Meanwhile the MicroPython in the shipped UF2s is not uniform: + +| Firmware | MicroPython | +|---|---| +| `Vector_WPC_v5.uf2` | v1.26.0-preview.255 | +| `Vector_DataEast_v1.uf2` | v1.26.0-preview.255 | +| `vector_system_11_and_9_v4.uf2` | v1.24.1 | + +`.mpy` files carry a bytecode version in their header, and a board's MicroPython refuses to import a `.mpy` whose version it doesn't know. So the build was compiling bytecode with a 1.28 toolchain and shipping it to firmware three to four minor versions behind, with nothing asserting the pairing is valid. + +It happens to work today — verified empirically that `mpy-cross` 1.23.0 and 1.28.0.post2 both emit `mpy_version=6, flags=0x00`, and a full `sys11` build produces 41 `.mpy` files all at version 6. So this is not a live bug. But it is unpinned, undocumented, and load-bearing: the day `mpy-cross` bumps to bytecode version 7, every build silently produces modules that no deployed board can import, and every OTA update bricks on the next boot. That failure would be very hard to diagnose from the symptom. + +`mpy-cross` is now pinned to `1.28.0.post2` — deliberately the version CI was already resolving, so the pin freezes current behavior rather than changing it. **The open question is what it *should* be pinned to**, which is a hardware question we can't answer from the repo: it should match the MicroPython in each target's UF2, and today those differ per target while the build uses one toolchain for all of them. See §12. + +This is also the single best argument for the update matrix running on every PR: it's exactly the class of problem where the build is green, the unit tests pass, and only real hardware tells you. + +**Runtime:** 5 versions × 3 boards, ~3–5 min each (two OTA cycles plus verification), parallel across board types → ~20–25 min for the upgrade matrix alone. + +**Combined per-PR cost.** G3 and G4 contend for the same physical boards, so they serialize per board rather than overlapping. The WPC board is the critical path both times: + +| | wpc board | sys11 board | data_east board | +|---|---|---|---| +| Config matrix (G3) | ~16–26 min | ~10–16 min | ~7–12 min | +| Upgrade matrix (G4) | ~15–25 min | ~15–25 min | ~15–25 min | +| **Serial total** | **~31–51 min** | ~25–41 min | ~22–37 min | + +So expect **roughly 30–50 minutes of bench occupancy per PR**, on a singleton bench, with PRs queueing behind each other. That is the accepted cost of treating updates as critical infrastructure. Levers, in the order worth reaching for: + +1. **Run G4 first.** The most valuable signal arrives earliest, and a broken update fails the run before spending 26 minutes on configs. +2. **A second WPC board.** Config matrix on board A, upgrade matrix on board B, genuinely in parallel — cuts the critical path from ~51 to ~26 min. This is the highest-leverage purchase on the whole bench and the manifest already supports pools. +3. **Path filters.** Docs-only and workflow-only PRs skip HIL entirely. +4. **Poll readiness over USB, not HTTP** — saves seconds on every one of ~180 boot cycles per run. +5. If it still hurts: move the *chained* upgrade walk to release tags and keep only direct jumps per PR. + +--- + +## 9. Runner hardening checklist + +- **Ephemeral runner** (`--ephemeral`), registered at repo scope, label `vector-hil` — *not yet + done; the bench currently runs a persistent runner. Ephemeral registration needs a PAT with + `administration: write` stored on the Pi to mint a token per job, which is a worse secret to + hold than the runner's own credentials. Acceptable only because `workflow_run` means the Pi + never executes PR-authored code; see RUNNER_SETUP.md for the full reasoning.* +- Unprivileged user, no sudo, no docker socket +- `ACTIONS_RUNNER_HOOK_JOB_STARTED` / `_COMPLETED`: wipe `_work/`, run a bench-health precheck (all boards enumerate and answer), fail the job immediately if the bench is unhealthy rather than producing a confusing test failure +- **No secrets on any self-hosted job.** In particular `WARPED_PINBALL_PRIVATE_KEY` must never be referenced by a job with a `self-hosted` label. Move signing into a dedicated environment restricted to `main` and tags so it is structurally unreachable from HIL. +- Repo setting: Actions → *Require approval for all outside collaborators* at minimum +- Minimal `permissions:` per job — HIL needs `checks: write`, `actions: read`, `contents: read`, nothing more +- Egress firewall on the Pi: github.com / api.github.com / objects.githubusercontent.com plus package mirrors; deny lateral movement into the LAN +- ~~**Pin actions by SHA.**~~ **Done in this PR.** All six workflows now pin every action to a commit SHA with the version in a trailing comment, and `dev/requirements.txt` pins every Python dependency. Previously `deploy_docs.yml`, `docs-on-pr.yml`, `specialfeatures-on-pr.yml`, `validate-json-configs.yml`, and `version-bump-guard.yml` used floating `@v4`/`@v5` tags — a compromised or repointed tag on any of them is a path to the same runner the bench will be attached to. Keep this invariant: **no floating tags, ever**, and consider a CI check that greps for `uses:.*@v[0-9]` to enforce it. +- `timeout-minutes` on every job, sized just above the measured worst case + +## 10. Risk register + +**Software-only reset means a wedged board stops the bench.** A PR — malicious or just buggy — can leave a board where `mpremote` can't reach the REPL: a `boot.py` that hard-faults, a tight loop blocking the USB REPL, or a corrupted filesystem. With no BOOTSEL control, recovery needs someone physically present. The config matrix touches every board on every PR, so the exposure is not hypothetical. + +Mitigations that cost nothing: + +- bench-health precheck that fails loudly, and optionally opens an issue, rather than timing out mysteriously +- a documented recovery runbook — `trench-coat/uf2/nuke.uf2` plus a UF2 reflash is the escape hatch +- tight `timeout-minutes` so a hung board doesn't burn an hour of queue + +Worth revisiting after a month: a `uhubctl`-capable powered hub plus BOOTSEL and RUN wired to Pi GPIO (2 pins per board) turns "walk over and press a button" into a test step, for roughly $20. If the bench wedges more than once or twice, buy the hardware. + +Note that `machine.reset()` is a full MCU reset, so `boot.py` and `main.py` do run — the boot-path coverage loss is small. What's genuinely not covered is cold-power-on and brown-out behavior, and FRAM state at power-up. + +**Other risks:** + +| Risk | Mitigation | +|---|---| +| `HDWR01` fires nondeterministically on bare boards | Characterize during bring-up; add a resistor pack if needed (§8, G1) | +| Bench is a singleton; PRs queue | Concurrency group, path filters, add boards as needed | +| Fork PR approved clean, then pushes dirty | Label auto-removed on `synchronize`; environment gate is per-run | +| Harness rots as API changes | Contract tests generated from `@api` docstrings fail when routes drift | +| Flaky HIL erodes trust in CI | Keep it non-blocking until measured flake rate is under ~1% over a week | +| `mpy-cross` bytecode version drifts away from shipped firmware | Now pinned; add a post-update HIL assertion that every `.mpy` imports (§8, G4) | +| ~30–50 min per-PR bench occupancy on a singleton bench | Run G4 first, path filters, second WPC board when it bites (§8) | +| 512 MB RAM on the Zero 2 W under 3-way parallelism | Keep test processes lean; watch for swap-induced timing flake | + +## 11. Rollout + +| Phase | Work | Exit criteria | +|---|---|---| +| 0 | Pi, VLAN, udev rules, 3 boards. Characterize bare-board fault behavior. Measure reset→ready. | 50 consecutive boots produce an identical fault set | +| 1 | `dev/hil` package, boot smoke test, `workflow_dispatch` only | Green run driven by hand | +| 2 | API contract tests over both transports. `workflow_run` trigger on, auto for maintainer branches, fork gate live. Non-blocking check. | Contract tests catch a deliberately broken route | +| 3 | **Upgrade matrix, every PR.** Promoted ahead of the config matrix — it's the highest-value signal and the suspected problem area. | Full 5-version matrix green across all three boards | +| 4 | Full config matrix | Combined run under ~50 min; flake rate under 1% over a week → make it a required check | +| 5 | Expand manifest to `em`, `whitestar`, `classic`; second WPC board to parallelize G3 against G4 | — | + +Phases 0 and 1 are where the real uncertainty lives. Everything after that is mostly typing. + +Note that phases 3 and 4 are deliberately ordered opposite to the goal numbering. The update path is the reason this bench exists, so it should be the first thing running on every PR — the config matrix is more coverage but less risk per unit of wall clock. + +## 12. Open questions + +1. **What should `mpy-cross` be pinned to?** It's now frozen at `1.28.0.post2` (what CI already resolved), but the shipped UF2s carry MicroPython v1.24.1 for System 11/9 and v1.26.0-preview for WPC and Data East. Ideally the build toolchain matches the target's firmware, which today would mean a per-target `mpy-cross` rather than one for all of them. Needs a hardware decision: standardize the firmware across targets, or make the build toolchain per-target. See §8. +2. **Bench WiFi credentials** — does the VLAN get its own SSID, or do boards join the existing one with VLAN assignment by MAC? +3. **Should HIL be a required check?** Recommendation: yes for G1–G4 once the flake rate is measured over a week. With G4 running per-PR by design, making it advisory-only would waste most of its value. +4. **How many versions back do we actually support?** The design says 5 stable releases; if the real support window is different, that's a one-line change to the matrix. +5. **Do we want a `latest.json` fixture served from the Pi** to make `/api/update/check` testable, or leave that endpoint untested? +6. **Is the existing USB hub self-powered?** Three Picos on a Zero 2 W's OTG port wants a powered hub. diff --git a/dev/hil/RUNNER_SETUP.md b/dev/hil/RUNNER_SETUP.md new file mode 100644 index 00000000..34cf6149 --- /dev/null +++ b/dev/hil/RUNNER_SETUP.md @@ -0,0 +1,179 @@ +# HIL Bench — Pi Runner Setup + +Sets up a Raspberry Pi with Vector boards attached as a GitHub Actions runner. + +Assumes the Pi is assembled, boards are on a powered USB hub, arm64 userland, and the boards +sit on the same VLAN as the Pi. Board detection and flashing use the repo's existing dev +pipeline (`dev/detect_boards.py`, `dev/sync.py`), so there is nothing bench-specific to +configure. + +> Raspberry Pi Connect's remote update only works on devices with A/B image support — Pi 4 and +> 5 — so the Zero 2 W can't be provisioned that way. This is a plain script you run on the Pi. + +## Setup + +**1. Get a registration token** from repo → Settings → Actions → Runners → New self-hosted +runner. Single-use, valid one hour. + +**2. On the Pi**, as your normal login user (not root): + +```bash +curl -fsSLO https://raw.githubusercontent.com/warped-pinball/vector/main/dev/hil/setup-runner.sh +chmod +x setup-runner.sh + +export VECTOR_HIL_WIFI_SSID="your-bench-ssid" +export VECTOR_HIL_WIFI_PASSWORD="your-bench-password" +export RUNNER_TOKEN="paste-from-github" + +./setup-runner.sh +``` + +Credentials go in environment variables so they stay out of the script and out of the repo. +Note they will land in your shell history — `unset RUNNER_TOKEN VECTOR_HIL_WIFI_PASSWORD` +afterwards, or prefix the exports with a space if your shell is set to ignore those. + +Takes about five minutes, most of it `pip install` building the dev pipeline. It prompts for +sudo once up front rather than midway through. + +If you already have the repo cloned at `~/vector`, run `dev/hil/setup-runner.sh` from there +instead of curling it — the script uses `~/vector` either way. + +## What it does + +1. Installs `git`, `python3-venv`, `curl`; adds you to `dialout` for serial access +2. Clones (or fetches) the repo into `~/vector` and builds a venv from `dev/requirements.txt` +3. Runs `dev/detect_boards.py` and prints what it found +4. Downloads the Actions runner into `~/actions-runner`, registers it with the `vector-hil` label +5. Writes bench credentials to the runner's `.env` +6. Installs and starts the systemd service, then verifies the unit actually stayed up + +It runs as your user and calls `sudo` only where needed — the Actions runner refuses to be +configured as root, and shouldn't run as root regardless. + +**Board detection is non-fatal.** A board unplugged or mid-reset prints a warning and setup +continues, rather than aborting over a transient USB state. + +**Safe to re-run.** An existing clone is fetched rather than re-cloned, an existing runner +download and registration are left alone, and `.env` is rewritten touching only the +`VECTOR_HIL_*` lines it owns. Re-running is the normal way to refresh WiFi credentials or pull +a newer harness — and after the first time you don't need `RUNNER_TOKEN`, since registration is +skipped once the runner exists. + +**The clone is pinned, not floating.** After fetching, the script checks out `origin/$REPO_BRANCH` +(default `main`) as a detached HEAD, so the bench always runs a known ref rather than whatever +was last left checked out. It refuses to run if the clone has uncommitted changes rather than +discarding them — if you've been debugging by hand there, commit, stash, or delete the clone. + +Overridable via environment if you need them: `REPO_URL`, `REPO_BRANCH`, `RUNNER_LABELS`, +`RUNNER_VERSION`, `RUNNER_ARCH`. + +### On WiFi credentials with unusual characters + +The runner reads `.env` line by line, splits on the first `=`, and takes the rest of the line +verbatim ([`Runner.Listener/Program.cs`](https://github.com/actions/runner/blob/v2.336.0/src/Runner.Listener/Program.cs#L179-L197)) — +there is no `EnvironmentFile=` in the generated systemd unit and `runsvc.sh` doesn't source it +either. So spaces, quotes and backslashes in an SSID or password are safe and must **not** be +escaped; quoting would store literal quote characters. A newline is the one value that cannot +be represented, and the script rejects it up front. + +### Why not an ephemeral runner + +DESIGN.md §9 calls for `--ephemeral`. This script registers a persistent runner instead, which +is a deliberate deviation with a reason: ephemeral runners deregister after every job, so +something has to mint a fresh registration token each time — which means storing a PAT with +`administration: write` on the Pi, readable by the same user that runs job code. That PAT is a +much more valuable secret than the runner's own credentials, which only let you receive jobs. + +The trade works because of the trust model: under `workflow_run`, the Pi only ever executes +harness code from a trusted ref (DESIGN.md §4), so the "poison the workspace for the next job" +attack that ephemeral runners defend against has no foothold. Revisit this if the Pi ever +starts executing PR-authored code — at that point ephemeral runners via JIT config stop being +optional. + +To re-register against a different repo or token, remove the registration first — the script +deliberately won't do this behind your back: + +```bash +cd ~/actions-runner && sudo ./svc.sh uninstall && ./config.sh remove --token +``` + +## Telling the boards apart + +Nothing on a board reports what hardware it is. `systemConfig.vectorSystem` is a build-time +constant baked into whatever was last flashed, so `dev/detect_boards.py` tells you what a board +is *running*, not what it is — which is exactly wrong after a mis-flash. `machine.unique_id()` +is the RP2040 chip id: stable per board and survives reflashing, but silent about the system. + +Since the bench boards are dedicated, pin them by chip id once and the question goes away. + +Blink each board in turn and watch the bench: + +```bash +cd ~/vector && PATH="$PWD/.venv/bin:$PATH" .venv/bin/python dev/hil/flash_and_check.py --identify +``` + +(`$VECTOR_HIL_VENV` is exported by the runner *service*, so it is not set in a login shell — +hence the explicit `.venv` path here.) + +Then record what you saw: + +```bash +echo 'VECTOR_HIL_BOARD_MAP==sys11,=wpc,=data_east' >> ~/actions-runner/.env +cd ~/actions-runner && sudo ./svc.sh stop && sudo ./svc.sh start +``` + +With the map set, `flash_and_check.py` uses it and ignores self-report entirely. Without it, +the harness falls back to self-report but **refuses to flash when two boards claim the same +system**, since that means at least one is running firmware for a system it isn't wired for. + +`--inventory-only` prints the chip ids without blinking or flashing anything. + +## Verify + +The runner should show **Idle** under Settings → Actions → Runners with the `vector-hil` label. + +End to end, [`.github/workflows/hil-smoke.yml`](../../.github/workflows/hil-smoke.yml) checks +that the runner picks up jobs, the bench environment reaches them, the serial devices are +present and accessible, and every detected board answers over `mpremote`. It fails if no +boards are found or if any detected board doesn't respond. + +`VECTOR_HIL_VENV` and `VECTOR_HIL_REPO` are exported into every job from `.env`, so workflows +don't hardcode paths. The smoke job deliberately does no `actions/checkout` — under the design's +trust model the bench runs harness code from the clone it already has, not from a PR. + +### Running it before this PR merges + +A `workflow_dispatch` workflow isn't dispatchable until it exists on the **default branch**, so +the "Run workflow" button won't appear while this is still a PR. The workflow therefore also +triggers on pushes to this branch — push anything to it and the job runs on the bench: + +```bash +git commit --allow-empty -m "trigger hil smoke" && git push +``` + +Watch it under the repo's Actions tab, or on the Pi itself: + +```bash +journalctl -u "$(cat ~/actions-runner/.service)" -f +``` + +Once this is on `main`, drop the `push:` trigger and use the Run workflow button. + +## If something breaks + +- **`registration failed`** — almost always an expired token. They last one hour. Get a fresh + one and re-run. +- **Jobs get OOM-killed** (`dmesg -T | grep -i oom`) — 512 MB is tight on a Zero 2 W. + `sudo apt install -y zram-tools` gets you compressed swap with no further config. +- **Runner won't start, globalization error** — libicu missing. Re-run + `~/actions-runner/bin/installdependencies.sh`, or add + `DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=1` to `~/actions-runner/.env`. +- **`.env` changes have no effect** — it is read at service start. + `cd ~/actions-runner && sudo ./svc.sh stop && sudo ./svc.sh start`. +- **`Permission denied` on `/dev/ttyACM*`** — the `dialout` group hasn't taken effect in the + running service. Reboot, or stop and start the service. +- **You need to tell two boards of the same type apart** — `detect_boards.py` identifies boards + by querying `systemConfig.vectorSystem`, so port order doesn't matter for distinct board + types. Only add udev rules if you have two of the same type. + +See [DESIGN.md](DESIGN.md) for the test architecture and the security model for fork PRs. diff --git a/dev/hil/flash_and_check.py b/dev/hil/flash_and_check.py new file mode 100644 index 00000000..52a7af9e --- /dev/null +++ b/dev/hil/flash_and_check.py @@ -0,0 +1,876 @@ +#!/usr/bin/env python3 +"""Flash every attached Vector board and health-check its API after boot. + +Run from the repo root on the bench Pi: + + cd ~/vector && PATH="$PWD/.venv/bin:$PATH" .venv/bin/python dev/hil/flash_and_check.py + +Inside an Actions job the runner's .env already provides VECTOR_HIL_VENV, so +plain `python dev/hil/flash_and_check.py` is enough there. In a login shell it +is not - .env is read by the runner service, not by your shell. + +Stages, in order: + + 1. inventory - probe every attached board for its RP2040 chip id and the + system its *current firmware* reports + 2. resolve - decide which target each board should be flashed with + 3. build - build each needed target once + 4. flash - wipe, copy, write bench config, reboot + 5. health - wait for boot, then exercise the API over USB and HTTP + +A note on identification, because it is the subtle part: nothing on the board +reports what *hardware* it is. ``systemConfig.vectorSystem`` is a build-time +constant baked into whatever was last flashed, and ``machine.unique_id()`` is +the RP2040 chip id - stable per board, but it says nothing about which system +the board is wired for. So "autodetection" can only tell you what a board is +currently *running*, which is exactly wrong after a mis-flash. See +resolve_targets() for how that is handled. +""" + +import argparse +import gzip +import json +import os +import re +import shutil +import subprocess +import sys +import time +import urllib.error +import urllib.request +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(REPO_ROOT / "dev")) + +import serial # noqa: E402 (ships with mpremote) +from usb_coms_demo import UsbApiClient # noqa: E402 + +# A bare bench board has nothing driving the game bus, so this one is correct +# and expected rather than a regression. +EXPECTED_FAULTS = {"HDWR02"} + +# Floating data lines can trip the >250-transition check in main.py and send +# the board down the safe_mode path, where the game config is never loaded. +# Warned about rather than failed, but it invalidates the config assertions - +# see DESIGN.md §8 (G1). +BENCH_WARN_FAULTS = {"HDWR01"} + +# Config key is the config filename without .json (dev/build.py:253). +DEFAULT_GAMENAME = { + "sys11": "GenericSystem11_", + "wpc": "Generic_WPC", + "data_east": "GenericDE_", + "em": "EM_machine_", +} + +# Boot is slow and variable, so we watch the console for the firmware saying +# it is ready rather than guessing at a delay. +# +# "Server: Loop Forever" is the correct marker and the only one: phew prints +# it immediately before loop.run_forever() (phew/server.py:381). The earlier +# "> starting web server on port 80" line is NOT a ready signal - it is +# printed before start_server is even scheduled, let alone bound, so matching +# it returns while the socket is still closed. backend.go() has already run +# connect_to_wifi() by this point, so the marker covers both transports. +READY_MARKER = "Server: Loop Forever" + +# The marker is printed just *before* run_forever(), so give the event loop a +# moment to actually accept the listening socket. http_get's retries cover any +# remainder. +SERVER_SETTLE_SECONDS = 2 + +BOOT_TIMEOUT = 150 +HTTP_TIMEOUT = 10 + +# Read-only routes exercised over HTTP. Kept side-effect free so the check can +# run against a board repeatedly without changing its state. +# Route -> expected body kind. Not everything is JSON: /api/game/name is +# documented as "Plain-text game name" (backend.py:451) and returns a bare +# string, even though route_wrapper still labels it application/json. +HTTP_ROUTES = ( + ("/api/version", "json"), + ("/api/fault", "json"), + ("/api/game/name", "text"), + ("/api/game/status", "json"), + ("/api/game/active_config", "json"), + ("/api/game/configs_list", "json"), + ("/api/leaders", "json"), + ("/api/players", "json"), + ("/api/machine_id", "json"), + ("/api/wifi/status", "json"), + ("/api/settings/get_tournament_mode", "json"), + ("/api/auth/challenge", "json"), +) + + +class CheckFailure(Exception): + pass + + +def log(msg): + print(msg, flush=True) + + +_TIMINGS = [] +_group = None + + +def group(title): + global _group + _group = (title, time.monotonic()) + print(f"::group::{title}", flush=True) + + +def endgroup(): + global _group + if _group: + title, started = _group + elapsed = time.monotonic() - started + _TIMINGS.append((title, elapsed)) + print(f" [{elapsed:.1f}s]", flush=True) + _group = None + print("::endgroup::", flush=True) + + +# -------------------------------------------------------------------------- +# 1. inventory +# -------------------------------------------------------------------------- + + +def ensure_tools_on_path(): + """Put the bench venv's bin dir on PATH and pick the interpreter to use. + + The harness gets run three ways - from an Actions job, from a login shell, + and by hand - and only the first has the runner's .env applied. dev/build.py + shells out to a bare `mpy-cross` and dev/flash.py to a bare `mpremote`, so + PATH has to be right for subprocesses too, not just for our own calls. + """ + candidates = [] + if os.environ.get("VECTOR_HIL_VENV"): + candidates.append(Path(os.environ["VECTOR_HIL_VENV"]) / "bin") + candidates.append(Path(sys.executable).parent) + candidates.append(REPO_ROOT / ".venv" / "bin") + + for bindir in candidates: + if (bindir / "mpremote").exists(): + os.environ["PATH"] = f"{bindir}{os.pathsep}{os.environ.get('PATH', '')}" + python = bindir / "python" + return str(python) if python.exists() else sys.executable + + if shutil.which("mpremote"): + return sys.executable + + raise CheckFailure( + "mpremote not found. Run with the bench venv, e.g.\n" + f" cd {REPO_ROOT} && PATH=\"$PWD/.venv/bin:$PATH\" .venv/bin/python dev/hil/flash_and_check.py ...\n" + "(VECTOR_HIL_VENV is exported by the runner service, so it is not set in a login shell.)" + ) + + +VENV_PYTHON = sys.executable + + +def mpremote(*args, timeout=60): + return subprocess.run(["mpremote", *args], capture_output=True, text=True, timeout=timeout) + + +def list_ports(): + result = mpremote("devs", timeout=30) + if result.returncode != 0: + raise CheckFailure(f"`mpremote devs` failed: {result.stderr.strip()}") + return [line.split()[0] for line in result.stdout.strip().splitlines() if line.strip()] + + +def probe(port): + """Return {port, chip_id, system, version} for one board. + + chip_id comes from the RP2040 itself so it survives any firmware state; + system/version come from the flashed firmware and may be missing if the + board is unflashed or broken. + """ + board = {"port": port, "chip_id": None, "system": None, "version": None} + + chip = mpremote( + "connect", port, "exec", + "from machine import unique_id;from binascii import hexlify;print(hexlify(unique_id()).decode())", + timeout=30, + ) + if chip.returncode == 0: + board["chip_id"] = chip.stdout.strip() + + info = mpremote( + "connect", port, "exec", + "import systemConfig;print(systemConfig.vectorSystem, systemConfig.SystemVersion)", + timeout=30, + ) + if info.returncode == 0 and info.stdout.strip(): + parts = info.stdout.split() + board["system"] = parts[0] + if len(parts) > 1: + board["version"] = parts[1] + + return board + + +def inventory(): + boards = [probe(port) for port in list_ports()] + if not boards: + raise CheckFailure("no boards found - check the USB hub and power") + + log(f"{'port':16} {'chip id':18} {'running':12} version") + for b in boards: + log(f"{b['port']:16} {b['chip_id'] or '?':18} {b['system'] or '(none)':12} {b['version'] or '-'}") + return boards + + +IDENTIFY_SNIPPET = """ +import machine, time +try: + import BoardLED as L + L.startUp() +except Exception: + L = None +led = machine.Pin("LED", machine.Pin.OUT) +for i in range({blinks}): + led.on() + if L: + L.ledColor(L.BLUE) + time.sleep(0.25) + led.off() + if L: + L.ledColor(L.BLACK) + time.sleep(0.25) +""" + + +def identify(boards, seconds=8): + """Blink each board in turn so a human can tell which is which. + + Uses the Pico W onboard LED, which works from the REPL no matter what + firmware is loaded, plus the Vector board's WS2812 in blue when the + flashed firmware happens to provide the driver. + """ + log(f"Blinking each board for ~{seconds}s. Watch the bench and note the order.") + log("") + for index, board in enumerate(boards, 1): + log(f" [{index}/{len(boards)}] BLINKING NOW: {board['port']} chip {board['chip_id']}") + result = mpremote( + "connect", board["port"], "exec", + IDENTIFY_SNIPPET.format(blinks=int(seconds / 0.5)), + timeout=seconds + 30, + ) + if result.returncode != 0: + log(f" could not blink this board: {result.stderr.strip()}") + else: + log(" done") + log("") + log("Now map what you saw to the chip ids, and put this in the runner's .env:") + log("") + log(" VECTOR_HIL_BOARD_MAP=" + ",".join(f"{b['chip_id']}=" for b in boards)) + + +# -------------------------------------------------------------------------- +# 2. resolve +# -------------------------------------------------------------------------- + + +def parse_board_map(raw): + """Parse VECTOR_HIL_BOARD_MAP: 'chipid=target,chipid=target'.""" + mapping = {} + for entry in (raw or "").split(","): + entry = entry.strip() + if not entry: + continue + if "=" not in entry: + raise CheckFailure(f"bad VECTOR_HIL_BOARD_MAP entry {entry!r}, expected chipid=target") + chip, target = entry.split("=", 1) + mapping[chip.strip()] = target.strip() + return mapping + + +def resolve_targets(boards, board_map): + """Decide the target for each board, refusing to guess when it matters. + + An explicit chip-id map is authoritative. Without one we fall back to what + each board's firmware reports, which is only trustworthy when every board + reports something different - if two boards claim the same system, that is + the signature of a previous mis-flash rather than of the hardware, and + flashing on that basis would silently perpetuate it. + """ + if board_map: + unmapped = [b for b in boards if b["chip_id"] not in board_map] + if unmapped: + raise CheckFailure( + "VECTOR_HIL_BOARD_MAP is set but does not cover: " + + ", ".join(f"{b['port']} ({b['chip_id']})" for b in unmapped) + ) + for b in boards: + b["target"] = board_map[b["chip_id"]] + log("targets from VECTOR_HIL_BOARD_MAP") + return boards + + missing = [b for b in boards if not b["system"]] + if missing: + raise CheckFailure( + "cannot identify " + + ", ".join(b["port"] for b in missing) + + " - firmware did not report a system. Set VECTOR_HIL_BOARD_MAP." + ) + + systems = [b["system"] for b in boards] + duplicates = {s for s in systems if systems.count(s) > 1} + if duplicates: + raise CheckFailure( + "refusing to flash from autodetection: " + + ", ".join(sorted(duplicates)) + + " is reported by more than one board.\n" + "Detection reads the *flashed firmware*, not the hardware, so duplicates mean\n" + "at least one board is running firmware for a system it is not wired for.\n" + "Pin them explicitly instead, using the chip ids above:\n" + " VECTOR_HIL_BOARD_MAP=" + + ",".join(f"{b['chip_id']}=" for b in boards) + ) + + for b in boards: + b["target"] = b["system"] + log("targets from firmware self-report (all distinct)") + return boards + + +# -------------------------------------------------------------------------- +# 3. build +# -------------------------------------------------------------------------- + + +def source_version(target): + config = REPO_ROOT / "src" / target / "systemConfig.py" + match = re.search(r'SystemVersion\s*=\s*"([^"]+)"', config.read_text()) + if not match: + raise CheckFailure(f"could not read SystemVersion from {config}") + return match.group(1) + + +def build(target): + build_dir = REPO_ROOT / "build" / target + result = subprocess.run( + [VENV_PYTHON, "dev/build.py", "--target_hardware", target, "--build-dir", str(build_dir)], + cwd=REPO_ROOT, capture_output=True, text=True, timeout=900, + ) + if result.returncode != 0: + log(result.stdout[-3000:]) + log(result.stderr[-3000:]) + raise CheckFailure(f"build failed for {target}") + return build_dir + + +# -------------------------------------------------------------------------- +# 4. flash +# -------------------------------------------------------------------------- + + +def write_bench_config(target, workdir): + ssid = os.environ.get("VECTOR_HIL_WIFI_SSID", "") + password = os.environ.get("VECTOR_HIL_WIFI_PASSWORD", "") + game_password = os.environ.get("VECTOR_HIL_GAME_PASSWORD", "hiltest") + + if not ssid or not password: + raise CheckFailure("VECTOR_HIL_WIFI_SSID / VECTOR_HIL_WIFI_PASSWORD are not set") + + # dev/flash.py builds a MicroPython snippet with single-quoted values, so a + # single quote anywhere here would produce a syntax error on the board + # rather than an obvious failure here. + for name, value in (("ssid", ssid), ("password", password), ("game password", game_password)): + if "'" in value or "\\" in value: + raise CheckFailure(f"bench {name} contains a quote or backslash, which dev/flash.py cannot write") + + config = { + "ssid": ssid, + "password": password, + "gamename": DEFAULT_GAMENAME[target], + "Gpassword": game_password, + } + path = workdir / f"hil-config-{target}.json" + path.write_text(json.dumps(config)) + return path + + +def flash(target, port, build_dir, config_path): + result = subprocess.run( + [VENV_PYTHON, "dev/flash.py", str(build_dir), "--port", port, "--write-config", str(config_path)], + cwd=REPO_ROOT, capture_output=True, text=True, timeout=900, + ) + if result.returncode != 0: + log(result.stdout[-3000:]) + log(result.stderr[-3000:]) + raise CheckFailure(f"flash failed for {target} on {port}") + + +# -------------------------------------------------------------------------- +# 5. health +# -------------------------------------------------------------------------- + + +def reset_board(port): + """Reset the board so we own the boot we are about to watch. + + The ready marker is printed exactly once per boot. dev/flash.py already + resets at the end of flashing, but flashing runs over every board before + any health check starts, so by the time we open a console the board booted + a minute ago and the marker is long gone. Resetting here makes the wait + deterministic and the reported boot time meaningful. + """ + result = mpremote( + "connect", port, "exec", "--no-follow", "import machine; machine.reset()", + timeout=30, + ) + if result.returncode != 0: + raise CheckFailure(f"could not reset {port} before the health check: {result.stderr.strip()}") + + +def wait_for_server(port, timeout=BOOT_TIMEOUT): + """Watch the boot console until the firmware reports its web server is up. + + Polling an API that is not listening yet tells you nothing about why, and + burns the whole timeout when a board fails to boot. Reading the console + instead gives an exact ready signal and, on failure, the boot log that + explains it. + + Returns the open serial connection so the USB API can reuse it - the + Pico exposes one CDC endpoint, so a second connection would fight this one. + """ + deadline = time.monotonic() + timeout + transcript = [] + connection = None + + while time.monotonic() < deadline: + if connection is None: + try: + # The port disappears and re-enumerates across the reset, so a + # failure to open here is expected for the first second or two. + connection = serial.Serial(port=port, baudrate=115200, timeout=1) + except Exception: + time.sleep(1) + continue + try: + raw = connection.readline() + except Exception: + try: + connection.close() + except Exception: + pass + connection = None + continue + + if not raw: + continue + text = raw.decode(errors="replace").rstrip("\r\n") + if not text: + continue + transcript.append(text) + + if READY_MARKER in text: + elapsed = timeout - (deadline - time.monotonic()) + log(f" server up after {elapsed:.1f}s ({text.strip()!r})") + time.sleep(SERVER_SETTLE_SECONDS) + return connection, transcript + + if connection is not None: + try: + connection.close() + except Exception: + pass + + tail = "\n ".join(transcript[-20:]) or "(nothing on the console)" + raise CheckFailure( + f"{port} never reported its web server within {timeout}s. Last console output:\n {tail}" + ) + + +def prime_usb(connection): + """Clear both ends of the serial line before the first API request. + + usb_comms accumulates stdin characters into a module-level `buffer` until + it sees a newline (usb_comms.py:132). Anything left there without a + terminator - a partial line, stray bytes from the raw-REPL session that + issued the reset - silently prefixes the next request, so the board parses + a route like "\x02/api/version", fails the `_routes` lookup and answers + 404. A lone newline flushes whatever is pending into a discarded request. + """ + try: + connection.reset_input_buffer() + connection.reset_output_buffer() + connection.write(b"\n") + connection.flush() + except Exception as exc: + log(f" warning: could not prime the USB link: {exc}") + return + # usb_request_handler is scheduled every 1000ms (phew/server.py:342), so + # give it a turn to consume the flush before the first real request. + time.sleep(1.5) + try: + connection.reset_input_buffer() + except Exception: + pass + + +def get(client, route, expect=200): + response = client.send_and_receive(route=route, payload=None, timeout=15) + status = response.get("status") + if status != expect: + # The board narrates its own routing failures ("USB REQ: route not + # found: ..."), but the client discards every line that is not a + # response. Drain whatever is pending so the reason is visible. + raise CheckFailure( + f"{route} returned {status}, expected {expect}" + f"{_drain_serial(client.ser)}" + ) + return response.get("body") + + +def _drain_serial(connection, limit=12): + """Return any pending board chatter, formatted for an error message.""" + try: + time.sleep(0.5) + pending = connection.read(connection.in_waiting or 0) + except Exception: + return "" + if not pending: + return "" + lines = [line for line in pending.decode(errors="replace").splitlines() if line.strip()] + if not lines: + return "" + return "\n board said: " + "\n board said: ".join(lines[:limit]) + + +def check_faults(board): + faults = get(board["client"], "/api/fault") or [] + if isinstance(faults, dict): + faults = faults.get("faults", []) + codes = {str(f)[:6] for f in faults} + + log(f" faults: {faults if faults else 'none'}") + + unexpected = codes - EXPECTED_FAULTS - BENCH_WARN_FAULTS + if unexpected: + raise CheckFailure(f"unexpected fault(s): {sorted(unexpected)}") + + warned = codes & BENCH_WARN_FAULTS + if warned: + log(f"::warning::{board['port']} raised {sorted(warned)} - bare-board bus noise, " + "the board is in safe mode and the game config was NOT loaded") + return warned + + +def health_check_usb(board): + client = board["client"] + target = board["target"] + expected_version = source_version(target) + + version = get(client, "/api/version") + reported = version.get("version") if isinstance(version, dict) else version + log(f" version: {reported}") + if expected_version not in str(reported): + raise CheckFailure(f"version {reported!r} does not match built {expected_version!r}") + + safe_mode = check_faults(board) + + status = get(client, "/api/game/status") + if not isinstance(status, dict): + raise CheckFailure(f"/api/game/status returned {type(status).__name__}, expected an object") + log(f" game status keys: {sorted(status)[:6]}") + + configs = get(client, "/api/game/configs_list") + if not isinstance(configs, dict) or not configs: + raise CheckFailure("/api/game/configs_list is empty - config bundle missing from the build") + log(f" configs available: {len(configs)}") + board["usb_config_count"] = len(configs) + + expected_config = DEFAULT_GAMENAME[target] + if safe_mode: + log(" skipping active-config check (board is in safe mode)") + else: + active = get(client, "/api/game/active_config") + log(f" active config: {active}") + if expected_config not in json.dumps(active): + raise CheckFailure(f"active config {active!r} is not the {expected_config!r} we flashed") + + leaders = get(client, "/api/leaders") + if leaders is None: + raise CheckFailure("/api/leaders returned no body") + + # Unauthenticated over USB by design (backend.py:280) - just prove it routes. + get(client, "/api/auth/challenge") + + wifi = get(client, "/api/wifi/status") + log(f" wifi: {wifi}") + + ip = None + try: + last_ip = get(client, "/api/last_ip") + ip = last_ip.get("ip") if isinstance(last_ip, dict) else None + except CheckFailure: + pass + return ip, wifi + + +def http_get(url, attempts=3): + """GET a URL, decompressing gzip and retrying transient failures. + + The board serves its web assets pre-gzipped with Content-Encoding: gzip + (backend.py:183) and urllib does not decompress automatically. Retries + exist because phew is a single-threaded server on a microcontroller that + is also fielding discovery broadcasts - an occasional dropped body is not + a regression worth failing a bench run over. + """ + last_error = None + for attempt in range(attempts): + try: + request = urllib.request.Request(url, headers={"User-Agent": "vector-hil"}) + with urllib.request.urlopen(request, timeout=HTTP_TIMEOUT) as response: + body = response.read() + if response.headers.get("Content-Encoding", "").lower() == "gzip": + body = gzip.decompress(body) + return response.status, body + except Exception as exc: + last_error = exc + if attempt + 1 < attempts: + time.sleep(2) + raise CheckFailure(f"GET {url} failed after {attempts} attempts: {last_error!r}") + + +def http_status(url): + """Return the status code, including for responses urllib treats as errors.""" + try: + request = urllib.request.Request(url, headers={"User-Agent": "vector-hil"}) + with urllib.request.urlopen(request, timeout=HTTP_TIMEOUT) as response: + return response.status + except urllib.error.HTTPError as exc: + return exc.code + + +def health_check_http(board): + ip = board.get("ip") + if not ip: + raise CheckFailure("board reported no IP address - it did not join the bench wifi") + base = f"http://{ip}" + + # The index page is served pre-gzipped; http_get transparently inflates it. + status, body = http_get(f"{base}/") + if status != 200: + raise CheckFailure(f"http / returned {status}") + head = body[:2000].lower() + if b"" for b in boards)) + log("") + log("Then: cd ~/actions-runner && sudo ./svc.sh stop && sudo ./svc.sh start") + return 0 + + workdir = REPO_ROOT / "build" + workdir.mkdir(exist_ok=True) + failures = [] + + group("Inventory") + boards = inventory() + endgroup() + + group("Resolve targets") + boards = resolve_targets(boards, parse_board_map(os.environ.get("VECTOR_HIL_BOARD_MAP"))) + for b in boards: + log(f" {b['port']} -> {b['target']}") + endgroup() + + if not args.skip_flash: + for target in sorted({b["target"] for b in boards}): + group(f"Build {target}") + build(target) + log(f"built {target} at version {source_version(target)}") + endgroup() + + for b in boards: + group(f"Flash {b['target']} on {b['port']}") + try: + config_path = write_bench_config(b["target"], workdir) + flash(b["target"], b["port"], REPO_ROOT / "build" / b["target"], config_path) + log("flashed") + except CheckFailure as exc: + log(f"::error::{exc}") + failures.append(f"{b['port']} ({b['target']}): {exc}") + b["skip"] = True + endgroup() + + for b in boards: + if b.get("skip"): + continue + group(f"Health check {b['target']} on {b['port']}") + try: + reset_board(b["port"]) + connection, boot_log = wait_for_server(b["port"]) + b["boot_log"] = boot_log + prime_usb(connection) + b["client"] = UsbApiClient(connection) + ip, _wifi = health_check_usb(b) + b["ip"] = ip + b["usb_version"] = source_version(b["target"]) + log(" USB API OK") + + if args.skip_http: + log(" HTTP checks skipped") + else: + health_check_http(b) + log(" HTTP API OK") + except CheckFailure as exc: + log(f"::error::{exc}") + _dump_boot_log(b) + failures.append(f"{b['port']} ({b['target']}): {exc}") + except Exception as exc: + log(f"::error::unexpected error: {exc}") + _dump_boot_log(b) + failures.append(f"{b['port']} ({b['target']}): {exc}") + finally: + client = b.get("client") + if client: + try: + client.close() + except Exception: + pass + endgroup() + + log("") + log("stage timings (build cost on the Zero 2 W is the number to watch):") + for title, elapsed in _TIMINGS: + log(f" {elapsed:7.1f}s {title}") + + log("") + log("=" * 60) + for b in boards: + state = "FAIL" if any(b["port"] in f for f in failures) else "ok" + log(f" {state:5} {b['port']:16} {b['target']:12} {b.get('ip') or ''}") + log("=" * 60) + + if failures: + log(f"\n{len(failures)} board(s) failed:") + for failure in failures: + log(f" - {failure}") + return 1 + + log(f"\nall {len(boards)} board(s) flashed and healthy") + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except CheckFailure as exc: + log(f"::error::{exc}") + sys.exit(1) diff --git a/dev/hil/setup-runner.sh b/dev/hil/setup-runner.sh new file mode 100755 index 00000000..28d25605 --- /dev/null +++ b/dev/hil/setup-runner.sh @@ -0,0 +1,198 @@ +#!/bin/sh +# +# Vector HIL bench — GitHub Actions runner setup. +# +# Run on the bench Pi as your normal login user (not root — the Actions +# runner refuses to be configured as root). Uses sudo for the parts that +# need it. +# +# export VECTOR_HIL_WIFI_SSID="bench-ssid" +# export VECTOR_HIL_WIFI_PASSWORD="bench-password" +# export RUNNER_TOKEN="from github, see below" +# ./setup-runner.sh +# +# RUNNER_TOKEN comes from the repo's +# Settings -> Actions -> Runners -> New self-hosted runner +# It is single-use and valid for one hour. It is only needed the first +# time; once the runner is registered you can re-run this without it. +# +# Safe to re-run: an existing clone is fetched rather than re-cloned, an +# existing runner download and registration are left alone, and only the +# VECTOR_HIL_* lines in the runner's .env are rewritten. + +set -eu + +REPO_URL="${REPO_URL:-https://github.com/warped-pinball/vector}" +REPO_BRANCH="${REPO_BRANCH:-main}" +RUNNER_LABELS="${RUNNER_LABELS:-vector-hil}" +RUNNER_VERSION="${RUNNER_VERSION:-2.336.0}" +RUNNER_ARCH="${RUNNER_ARCH:-arm64}" + +REPO_DIR="$HOME/vector" +RUNNER_DIR="$HOME/actions-runner" +VENV_DIR="$REPO_DIR/.venv" + +log() { echo "==> $*"; } +fail() { echo "ERROR: $*" >&2; exit 1; } + +# --- preflight ------------------------------------------------------------ + +[ "$(id -u)" -ne 0 ] || fail "run as your normal user, not root (the runner will not configure as root)" + +[ -n "${VECTOR_HIL_WIFI_SSID:-}" ] || fail "VECTOR_HIL_WIFI_SSID is not set" +[ -n "${VECTOR_HIL_WIFI_PASSWORD:-}" ] || fail "VECTOR_HIL_WIFI_PASSWORD is not set" + +# The runner parses .env line by line, splitting on the first '=' and taking +# the rest of the line verbatim (Runner.Listener/Program.cs). Spaces, quotes +# and backslashes are therefore safe and must NOT be escaped - quoting would +# put literal quote characters into the value. A newline is the one thing that +# cannot be represented, so reject it rather than write a corrupt file. +for cred in VECTOR_HIL_WIFI_SSID VECTOR_HIL_WIFI_PASSWORD; do + eval "cred_value=\$$cred" + if [ "$(printf '%s' "$cred_value" | wc -l)" -ne 0 ]; then + fail "$cred contains a newline, which cannot be stored in the runner's .env" + fi +done + +if [ ! -f "$RUNNER_DIR/.runner" ] && [ -z "${RUNNER_TOKEN:-}" ]; then + fail "RUNNER_TOKEN is not set and the runner is not registered yet" +fi + +# Prompt for sudo once here rather than halfway through. +sudo -v || fail "sudo is required" + +# --- system packages ------------------------------------------------------ + +log "installing system packages" +sudo apt-get update -qq || fail "apt-get update failed" +sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq git python3-venv curl \ + || fail "apt-get install failed" + +# Serial access to the Picos. systemd reads group membership when it starts +# the runner service below, so no reboot is needed. +if ! id -nG | grep -qw dialout; then + log "adding $(whoami) to dialout" + sudo usermod -aG dialout "$(whoami)" +fi + +# --- repo and dev pipeline ------------------------------------------------ + +if [ -d "$REPO_DIR/.git" ]; then + log "updating existing clone at $REPO_DIR" + git -C "$REPO_DIR" fetch --quiet origin \ + || echo "WARNING: git fetch failed, falling back to the last fetched state" +else + log "cloning $REPO_URL" + git clone --quiet "$REPO_URL" "$REPO_DIR" || fail "git clone failed" +fi + +# Fetching alone leaves the working tree wherever it was, so the bench could +# keep running stale or hand-modified harness code. Check out an explicit ref +# instead, detached, to make it obvious this is a pinned trusted checkout and +# not a branch anyone should be committing on. +if ! git -C "$REPO_DIR" diff --quiet HEAD 2>/dev/null; then + fail "$REPO_DIR has uncommitted changes - commit, stash, or delete the clone and re-run" +fi +git -C "$REPO_DIR" checkout --quiet --detach "origin/$REPO_BRANCH" \ + || fail "could not check out origin/$REPO_BRANCH" +log "harness at $(git -C "$REPO_DIR" rev-parse --short HEAD) (origin/$REPO_BRANCH)" + +if [ ! -x "$VENV_DIR/bin/python" ]; then + log "creating virtualenv" + python3 -m venv "$VENV_DIR" || fail "venv creation failed" +fi + +log "installing dev pipeline requirements (a few minutes)" +"$VENV_DIR/bin/pip" install --quiet --upgrade pip || fail "pip self-upgrade failed" +"$VENV_DIR/bin/pip" install --quiet -r "$REPO_DIR/dev/requirements.txt" \ + || fail "pip install of dev/requirements.txt failed" + +# --- board check ---------------------------------------------------------- +# Non-fatal: a board unplugged or mid-reset should not abort the setup. + +log "detecting boards" +if boards=$("$VENV_DIR/bin/python" "$REPO_DIR/dev/detect_boards.py" 2>&1); then + echo " $boards" + case "$boards" in + '{}'|'') echo " WARNING: no boards detected - check the USB hub and power" ;; + esac +else + echo " WARNING: board detection failed: $boards" +fi + +# --- actions runner ------------------------------------------------------- + +TARBALL="actions-runner-linux-${RUNNER_ARCH}-${RUNNER_VERSION}.tar.gz" + +mkdir -p "$RUNNER_DIR" + +if [ ! -x "$RUNNER_DIR/config.sh" ]; then + log "downloading actions runner $RUNNER_VERSION ($RUNNER_ARCH)" + curl -fSL --retry 3 -o "$RUNNER_DIR/$TARBALL" \ + "https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/${TARBALL}" \ + || fail "runner download failed" + tar -xzf "$RUNNER_DIR/$TARBALL" -C "$RUNNER_DIR" || fail "runner extraction failed" + rm -f "$RUNNER_DIR/$TARBALL" + + log "installing runner dependencies" + sudo "$RUNNER_DIR/bin/installdependencies.sh" >/dev/null \ + || fail "installdependencies.sh failed" +else + log "runner already downloaded, skipping" +fi + +if [ -f "$RUNNER_DIR/.runner" ]; then + log "runner already registered, leaving registration alone" +else + log "registering runner" + ( cd "$RUNNER_DIR" && ./config.sh \ + --url "$REPO_URL" \ + --token "$RUNNER_TOKEN" \ + --labels "$RUNNER_LABELS" \ + --unattended --replace >/dev/null ) \ + || fail "registration failed - the token may have expired (they last one hour)" +fi + +# --- bench environment ---------------------------------------------------- +# The runner reads .env at service start and exports it into every job. +# Only rewrite the lines we own so anything else set there survives. + +ENV_FILE="$RUNNER_DIR/.env" +log "writing bench environment" + +TMP_ENV="${ENV_FILE}.new" +: > "$TMP_ENV" +[ -f "$ENV_FILE" ] && { grep -v '^VECTOR_HIL_' "$ENV_FILE" >> "$TMP_ENV" || true; } +{ + echo "VECTOR_HIL_WIFI_SSID=$VECTOR_HIL_WIFI_SSID" + echo "VECTOR_HIL_WIFI_PASSWORD=$VECTOR_HIL_WIFI_PASSWORD" + echo "VECTOR_HIL_VENV=$VENV_DIR" + echo "VECTOR_HIL_REPO=$REPO_DIR" +} >> "$TMP_ENV" +mv "$TMP_ENV" "$ENV_FILE" +chmod 600 "$ENV_FILE" + +# --- service -------------------------------------------------------------- + +cd "$RUNNER_DIR" + +if [ -f "$RUNNER_DIR/.service" ]; then + log "restarting runner service" + sudo ./svc.sh stop >/dev/null 2>&1 || true +else + log "installing runner service" + sudo ./svc.sh install "$(whoami)" >/dev/null || fail "svc.sh install failed" +fi + +sudo ./svc.sh start >/dev/null || fail "svc.sh start failed" + +# Confirm the unit actually stayed up rather than reporting success for a +# service that immediately died. +sleep 5 +SERVICE_NAME=$(cat "$RUNNER_DIR/.service" 2>/dev/null || echo "") +if [ -n "$SERVICE_NAME" ] && ! systemctl is-active --quiet "$SERVICE_NAME"; then + systemctl status "$SERVICE_NAME" --no-pager --lines=20 || true + fail "runner service is not active after start" +fi + +log "done - runner '$SERVICE_NAME' is active with label '$RUNNER_LABELS'" diff --git a/dev/requirements.txt b/dev/requirements.txt index 45918634..b16e68e3 100644 --- a/dev/requirements.txt +++ b/dev/requirements.txt @@ -1,9 +1,14 @@ mpremote==1.23.0 -mpy-cross -csscompressor -jsmin -scour -htmlmin2 -beautifulsoup4 -cryptography -pre-commit +# mpy-cross emits .mpy bytecode that the board's MicroPython must be able to +# import. Leaving it unpinned means a future release can silently bump the +# bytecode version and produce builds no deployed board can load. Pinned to the +# version CI was already resolving; see dev/hil/DESIGN.md for the open question +# about aligning this with the MicroPython in each target's UF2. +mpy-cross==1.28.0.post2 +csscompressor==0.9.5 +jsmin==3.0.1 +scour==0.38.2 +htmlmin2==0.1.13 +beautifulsoup4==4.15.0 +cryptography==50.0.0 +pre-commit==4.6.2