From fd88449beb94d8f4dbeca8d57c381b894ef050a7 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 14 Aug 2026 22:59:58 +0000 Subject: [PATCH 01/22] docs: add hardware-in-the-loop testing design proposal Design for a self-hosted Actions runner driving real Vector boards from this public repo, covering boot smoke tests, API contract tests, the full game-config matrix, and the previous-5-versions upgrade matrix. The security model is the load-bearing part: PR code is built on GitHub-hosted runners and only artifacts cross onto the bench, with the harness itself always executing from the default branch via workflow_run. Design only -- no implementation. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01WPbnmZNF6DFnLQBtpQpcW4 --- dev/hil/DESIGN.md | 411 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 411 insertions(+) create mode 100644 dev/hil/DESIGN.md diff --git a/dev/hil/DESIGN.md b/dev/hil/DESIGN.md new file mode 100644 index 00000000..6c6185b4 --- /dev/null +++ b/dev/hil/DESIGN.md @@ -0,0 +1,411 @@ +# Hardware-in-the-Loop (HIL) Testing — Design + +**Status:** proposal, not yet implemented +**Scope:** a self-hosted GitHub Actions runner driving real Vector boards, safely, from a public repository. + +--- + +## 1. Goals + +The first four things we want covered: + +| # | Goal | Tier | +|---|---|---| +| G1 | The code boots and the boards run normally | every PR | +| G2 | We can connect to the boards and hit the API | every PR | +| G3 | Every available config can be parsed and boot | every PR | +| G4 | Updating from the last 5 versions to the proposed version works | nightly + release tags + on-demand | + +Non-goals for v1: gameplay/bus-level correctness (bare boards, no machine attached), AP-mode setup flow (needs a wire to GPIO22), long-duration soak testing. + +## 2. Decisions taken + +| Decision | Choice | +|---|---| +| Harness location | All in `warped-pinball/vector`. Bench topology and credentials live on the Pi, not in git. | +| Bench hardware | Bare boards, no machine or bus emulator attached. | +| Board networking | Dedicated VLAN on the existing LAN. | +| Trigger policy | Automatic for branches in `warped-pinball/vector`; forks require a maintainer gate. | +| Board recovery | Software reset only (`machine.reset()` over `mpremote`). | +| Update signing | `skip_signature_check: true` for the upgrade path, plus negative tests that the signature gate still rejects bad packages. | +| Initial inventory | `sys11`, `wpc`, `data_east`. More systems added later. | + +--- + +## 3. Threat model + +The repo is public and anyone can open a PR. A PR is, by construction, *untrusted code that we want to run on hardware*. The design has to make that safe rather than avoid it. + +### What we are protecting + +1. **The LAN.** Anything reachable from the runner or the boards. +2. **`WARPED_PINBALL_PRIVATE_KEY`.** This is the highest-value asset in the org. The matching public key is hardcoded in `src/common/update.py:107`, so a leak means an attacker can sign an update that every Vector in the field will accept. It must never be reachable from a job that runs on the self-hosted runner. +3. **The runner host** and its registration token. +4. **The boards** — recoverable, and the least of the four. + +### Attack surfaces, ranked + +**(a) Host-side code execution on the Pi.** The big one, and it is broader than it looks. Any of these executes PR-authored code on the runner: + +- an obvious `run: python dev/build.py` +- `pip install -r dev/requirements.txt` — a PR can repoint a requirement at a malicious package, or add a `setup.py` that runs at install time +- `pre-commit` hooks — the config names arbitrary repos and revisions +- **`pytest` collection** — `conftest.py` is imported automatically before any test runs. If the harness ever runs from a PR checkout, a PR that only adds a `conftest.py` gets code execution with zero test code executed +- workflow files themselves, if the workflow runs from the PR ref + +The mitigation is a single rule, and everything else follows from it: + +> **The Pi never checks out or executes PR-authored files. It executes harness code from a trusted ref, and consumes only build *artifacts* from the PR.** + +**(b) Board-side code execution.** Intentional — that's the test. `update.py:write_files()` honors `"execute": true` per file and `__import__`s it, so an update package is arbitrary code by design. Contained by assuming every board on the bench is fully compromised at all times, and isolating it accordingly. + +**(c) Runner persistence between jobs.** A compromised job leaves something behind for the next one. Mitigated with ephemeral runners and job hooks that wipe state. + +**(d) Secret exfiltration.** Mitigated by putting no secrets on the HIL workflow at all. + +### A note on the repo question + +Putting the harness in a separate repo would **not**, by itself, make any of this safer. A separate repo whose workflow still checked out the PR and ran `pytest` on the Pi is exactly as compromised as an in-repo one. The property that makes this safe is *trusted-ref execution*, not the repo boundary. + +So the repo choice is an ergonomics decision, and `vector` is the right home: the harness tests the API, and the API changes in this repo. Keeping them together means an endpoint change and its test ship in one PR and can't drift. The one carve-out is that the *instantiated* bench manifest — serial numbers, IPs, game passwords, WiFi credentials — stays on the Pi. The repo carries the schema and an example, not the instance. That is a "don't publish a map of the lab" measure, not a security boundary. + +--- + +## 4. Execution model + +Two stages, split by trust. + +### Stage A — build (untrusted, GitHub-hosted) + +This already exists: `.github/workflows/build_release.yml` runs on `pull_request`, builds all seven targets, and uploads a `update-files` artifact. Two additions needed: + +1. Also upload the full `build/` tree per target (tarball). The update packages are only good for OTA; flashing a board from scratch needs the file tree. +2. Write a `pr_meta.json` into the artifact: PR number, head SHA, head repo full name, and the per-target versions. Stage B needs these and cannot reliably get them from the event payload for fork PRs. + +No secrets, no self-hosted labels, unchanged trust posture. + +### Stage B — hardware (trusted, self-hosted) + +A new `.github/workflows/hil.yml`, triggered by **`workflow_run`** on completion of "Build and Deploy". + +`workflow_run` is the key primitive. GitHub always runs a `workflow_run` workflow *from the default branch, using the default branch's code*, regardless of what the triggering PR contains. A PR therefore cannot modify `hil.yml`, `dev/hil/**`, `conftest.py`, or the pinned dependency set that the Pi will execute. It gets to supply exactly one thing: the artifact bytes. + +``` +pull_request (fork or branch) + │ + ▼ + Build and Deploy ── GitHub-hosted, untrusted code, no secrets + │ artifact: update packages + build trees + pr_meta.json + ▼ + workflow_run: completed + │ + ├─ gate job (ubuntu-latest): decide auto vs. approval + │ head repo == warped-pinball/vector → proceed + │ fork → environment `hardware-lab`, + │ required reviewers, job blocks + ▼ + hil job (runs-on: [self-hosted, vector-hil]) + │ workflow + harness from default branch ← trusted + │ artifact downloaded from the build run ← untrusted payload only + ▼ + boards on isolated VLAN + │ + ▼ + check run posted against the PR head SHA +``` + +**Gating.** Auto for same-repo branches, approval for forks. Two mechanisms, both auditable: + +- a `hardware-lab` GitHub Environment with required reviewers — the job literally pauses until a maintainer clicks approve, and the approval is logged +- a `safe-to-hil` label as a secondary signal, **auto-removed on every new push** by a tiny `pull_request.synchronize` workflow, so a contributor can't get a clean diff approved and then push a dirty one + +Belt and braces, but the label alone is not enough — label state and head SHA can desync — and the environment gate alone gives no signal on the PR itself. + +**Reporting.** `workflow_run` jobs have write permissions, so the result is posted as a check run keyed to the PR's head SHA via the Checks API. That's what makes it eligible to be a required check later. + +--- + +## 5. Physical and network layout + +### Runner host + +**Check the Pi model before buying.** The GitHub Actions runner ships for Linux x64, ARM64, and ARM32 (ARMv7). The original Pi Zero / Zero W is ARMv6 (`ARM1176`) and is **not supported** — the runner will not start. Pi Zero 2 W (Cortex-A53) works under a 32-bit ARMv7 userland. Verify against GitHub's current supported-architecture list before ordering. + +Even on a Zero 2 W, be aware of: + +- **512 MB RAM** shared between the runner, Python, and 3+ concurrent `mpremote` sessions. Tight but workable; swap on the SD card will hurt. +- **One micro-USB OTG data port.** Three boards means a powered hub, and it must be self-powered. +- Wall-clock matters here — the config matrix is minutes of `mpremote` round-trips, and a Zero 2 W is slow at everything. A Pi 4 costs a little more and removes the whole category of problem. Worth it if the bench is going to be load-bearing. + +### Board network + +Boards live on a dedicated VLAN. Rules: + +- **Allow:** board subnet → Pi's HTTP file server (serves update packages for OTA tests). +- **Allow:** intra-VLAN broadcast, so `discovery.py` peer discovery is actually exercised. +- **Deny:** board subnet → rest of the LAN. +- **Deny:** board subnet → internet. +- The Pi needs an interface on the VLAN (tagged sub-interface on a USB ethernet adapter, or a second SSID) *plus* uplink to github.com. +- Firewall the runner from the rest of the LAN too. It is a CI box that pulls untrusted artifacts; it is not a trusted host. + +One wrinkle: `/api/update/check` fetches `http://software.warpedpinball.com/vector//latest.json` (`src/*/systemConfig.py:2`). Rather than punching an egress hole, **override that hostname in DNS on the VLAN to point at the Pi** and serve a canned `latest.json`. That keeps egress at zero and makes the update-check test deterministic — you can assert behavior against a pinned "latest" instead of whatever is live. + +### Board addressing + +`/dev/ttyACM*` ordering is not stable across reboots or re-enumeration. Use udev rules keyed on each Pico's unique USB serial number to get stable paths: + +``` +/dev/vector-sys11-a +/dev/vector-wpc-a +/dev/vector-data-east-a +``` + +Static DHCP leases on the VLAN, keyed by MAC, for the HTTP side. + +--- + +## 6. Bench manifest + +The inventory is going to grow (`em`, `whitestar`, `classic`, and probably a second WPC board). Nothing about board count or identity should be hardcoded. + +`dev/hil/bench.example.yaml` in the repo; the real one at `/etc/vector-hil/bench.yaml` on the Pi, located via `$VECTOR_HIL_BENCH`. + +```yaml +boards: + - id: sys11-a + target: sys11 # matches an id in dev/ci/targets.json + hardware: sys11 # build tree to flash + serial: /dev/vector-sys11-a + ip: 10.42.7.11 + - id: wpc-a + target: wpc + hardware: wpc + serial: /dev/vector-wpc-a + ip: 10.42.7.12 + - id: data-east-a + target: data_east + hardware: data_east + serial: /dev/vector-data-east-a + ip: 10.42.7.13 + +network: + update_server: http://10.42.7.1:8080 +secrets_file: /etc/vector-hil/secrets.yaml # game password, wifi creds +``` + +Tests parameterize over `manifest ∩ dev/ci/targets.json`. Adding `whitestar` later is a udev rule plus three lines of YAML — no test changes. Multiple boards with the same `target` are treated as a pool and sharded across, which is how you buy throughput later. + +`sys11_tiny` is the same hardware as `sys11`, so it runs as a second firmware pass on the `sys11-a` board rather than needing its own. + +--- + +## 7. Harness structure + +``` +dev/hil/ + bench.py # manifest load + validation + board.py # Board: transports, reset, wait_for_boot, flash, seed_config + transports/ + usb.py # JSON-over-serial client for usb_comms.py + http.py # HTTP client with challenge/HMAC auth + flashing.py # wipe + mpremote fs cp from a build tree; OTA via /api/update/apply + update_server.py # local HTTP server for OTA payloads + conftest.py # fixtures: board pool, per-test lease, artifact discovery + tests/ + test_boot.py + test_api_contract.py + test_configs.py + test_update_matrix.py + bench.example.yaml + README.md +``` + +### Two transports, and an important asymmetry + +Vector exposes the same routes over HTTP (phew) and over USB serial (`usb_comms.py` replays into `phew.server._routes`). Running tests over both catches regressions in either bridge. + +But note `backend.py:280-287`: **the USB transport bypasses authentication entirely.** Requests tagged `is_usb_transport` skip the HMAC check. Two consequences: + +- **Auth tests must run over HTTP.** Over USB every route is reachable unauthenticated, so USB can't tell you anything about the auth layer. +- **USB is a convenient privileged setup channel.** Seeding config, scores, and adjustment profiles for a test doesn't need credentials at all — just drive it over serial. That simplifies fixtures considerably. + +The HTTP client mirrors `src/common/web/js/utils.js:64-72`: `GET /api/auth/challenge`, then `hmac_sha256(password, challenge + path + query + body)` hex-encoded into `X-Auth-HMAC`, with the challenge in `X-Auth-challenge`. + +--- + +## 8. Test design + +### G1 — boots and runs normally + +Per board, per target firmware: wipe → flash build tree via `mpremote` → seed bench config → `machine.reset()` → capture serial console. + +Assertions: + +- Banner and `Version ` on the console within the timeout +- `/api/version` matches the version built from the PR head SHA +- `/api/fault` contains only bench-expected faults (see below) +- No `SFTW01` (drop-through) — that fault means `backend.go()` returned, which it never should +- **Soak, 60–120s:** poll `/api/memory-snapshot` and `/api/game/status`. Assert free memory doesn't trend toward zero and the scheduler still answers. This catches `SFTW02: async loop interrupted` and slow leaks, which are exactly the failures a single boot check misses. +- `/api/wifi/status` connected with the expected IP; `/api/network/peers` sees the other bench boards (exercises `discovery.py`) + +#### ⚠️ Bare-board boot behavior — characterize this first + +This is the most likely source of a flaky suite, and it needs measuring during bring-up before any test is written. + +`main.py:44-66` runs `bus_activity_fault_check()`: it samples GPIO14–21 (the data lines) for 800 ms and raises **`HDWR01: Early Bus Activity`** if it counts more than 250 transitions. On a bare board those eight pins are floating inputs. If they pick up enough noise to cross the threshold, the board takes a completely different boot path — `GameDefsLoad.go(safe_mode=True)`, `MemoryMain` skipped, error LED — and every downstream assertion changes meaning. + +Separately, `adr_activity_ok()` raises **`HDWR02: No Bus Activity`** when shadow RAM lamp columns don't change, which on a bare board is the correct and permanent state. + +So: + +- `HDWR02` is **expected** on the bench and belongs in the allowlist. +- `HDWR01` must be **deterministic**. If it appears intermittently, the fix is a resistor pack tying those eight lines to a known level on the test bench — a bench fixture, not a firmware change. Do not paper over it with a retry; a test suite that retries its way past nondeterministic boot paths will hide real regressions. + +Bring-up task: boot each bare board 50 times, record the fault set each time, and confirm it is identical every time. Until that holds, nothing else is worth automating. + +### G2 — connect and hit the API + +**Generated contract tests.** `tools/gen_api_docs.py` already parses the `@api` docstrings out of `backend.py`. Reuse that parser as the test inventory rather than hand-maintaining a list: + +- every documented unauthenticated route returns 200 and parseable JSON whose top-level shape matches the documented example +- every `auth: true` route returns 401 over HTTP without credentials, and succeeds with them +- **every route the server actually registers is documented** — walk `phew.server._routes` over the REPL and diff against the parsed docs + +That last one is the valuable one. It turns the API docs into a load-bearing artifact and makes "shipped an undocumented endpoint" a build failure instead of a discovery. + +**Auth behavior** (HTTP only): wrong password rejected; a replayed challenge rejected (`backend.py` deletes the challenge on use — worth a regression test); expired challenge rejected; more than 10 outstanding challenges returns 429. + +**Round-trip state:** set an adjustment profile name → read back; import scores → export → compare; set tournament mode → reset → still set (proves FRAM persistence, which is what update tests depend on later). + +**Static serving:** `/`, `/index.html`, gzip `Content-Encoding`, and the ETag/304 path. + +**Known gap:** AP-mode setup can't be tested. `check_ap_button()` reads GPIO22, and with software-only control we can't hold it. One wire per board from GPIO22 to a Pi GPIO would unlock it; worth doing only if AP-mode regressions start to bite. + +### G3 — every config parses and boots + +Full matrix on every PR, per the decision above. Per board, loop over that hardware's configs: + +1. Interrupt to REPL, write `gamename` into `SPI_DataStore` `configuration` record, `machine.reset()` +2. Wait for boot +3. Assert: + - no `CONF00` / `CONF01` fault + - `/api/game/active_config` is the config we set + - `/api/game/name` matches `GameInfo.GameName` **from the source JSON in the repo** — this cross-checks the on-board `config/all.jsonl.z` against the source and catches build-time config-packing bugs, not just parse errors + - `/api/leaders` and `/api/adjustments/status` both return 200 — proves the parsed definition is *usable*, not merely loadable + - free memory after load is above a floor + +That last assertion is the one that earns its keep. `sys11_tiny` exists because RAM is tight; a config that parses fine but leaves too little heap is the failure that actually reaches customers. + +**Throughput.** Boot cycle is roughly 15–25s (`main.py` alone has an 0.8s bus check plus ~4.5s of sleeps before WiFi comes up), times config count: + +| Board | Configs | Serial estimate | +|---|---|---| +| wpc | 63 | ~16–26 min | +| sys11 | 39 | ~10–16 min | +| data_east | 28 | ~7–12 min | +| em | 1 | trivial | + +Boards run in parallel, so wall clock ≈ the WPC leg, **~16–26 min per PR**. That's a real cost and the bench is a singleton. Recommended mitigations: + +- `concurrency: { group: hil-bench, cancel-in-progress: false }` — queue, don't interleave. Two jobs sharing one bench will corrupt each other's state. +- Path filters so docs-only and workflow-only PRs skip HIL entirely. +- Order the loop so configs touched by the diff run first — fail fast on the likely culprit. +- Poll readiness over **USB rather than HTTP**; the USB bridge is answering well before WiFi associates, which shaves seconds off every one of 130 iterations. +- When it gets annoying, add a second WPC board. The manifest already supports pools, and the runner shards across them — that alone roughly halves wall clock. + +### G4 — update from the last 5 versions + +Matrix: each board target × the last 5 **stable** releases. Note the release list is mostly prereleases (`1.11.28-beta1746` etc. are PR builds); filter to `prerelease == false`, which currently gives `1.11.28`, `1.11.27`, and so on. + +Getting a board *to* an old version is the interesting part. Rather than building historical source — which drags in period-correct `mpy-cross` and MicroPython versions and is genuinely painful: + +1. Flash the current build once. +2. Apply version *V*'s **signed** release asset via `/api/update/apply` with no skip flag. This gets us to V using the real OTA mechanism, and as a bonus exercises the *signed* code path on the way down. +3. Confirm `/api/version == V`. + +Caveat worth stating: that's a downgrade, which no user performs. But the resulting filesystem is exactly what V's update package produces, which is what the upgrade is going to be applied to — so the fidelity that matters is preserved. + +Then the actual test: + +4. Seed realistic state — scores, players, adjustment profiles, settings. `dev/test_data.json` already exists for this and `flash.py --test-data` already knows how to write it. +5. Serve the PR's `-update.json` from the Pi. +6. `POST /api/update/apply` with `skip_signature_check: true`. Consume the streamed progress JSON; assert percent is monotonic and no error lines appear. +7. After reboot, assert: + - version == the PR version + - **all seeded state survived** — scores, players, adjustments, tournament mode, claim methods, WiFi credentials, gamename. Data migration is the real risk in an upgrade, and it's invisible to a version check. + - no faults, API healthy, free memory sane + - **no stale files** — the update runs a `remove_extra_files.py` execute-step, so diff an `ls` over the REPL against the expected build tree. A file the cleanup misses is a file that shadows a module in a later version. + +#### Negative tests + +The update path is the most security-sensitive code we ship, and these are cheap: + +- unsigned PR package applied **without** `skip_signature_check` → must be rejected, **and the board must still boot afterward** +- one byte flipped in the body (hash mismatch) → rejected +- valid hash, garbage signature → rejected +- **interrupted mid-update** — reset the board at ~50% and see what happens. This is the "user pulled the plug" scenario. Either it recovers to something bootable, or we learn precisely how it fails and how to talk a customer through it. Right now nobody knows which. + +The negative tests are what buy back the coverage lost by using `skip_signature_check` for the happy path. + +**Runtime:** 5 versions × 3 boards, ~3–5 min each (two OTA cycles plus verification), parallel across boards → ~20–25 min. That is *on top of* the config matrix. + +**Recommendation:** run G4 nightly on `main`, on release tags, and on-demand via label — not on every PR. The risk that an upgrade from 1.11.24 breaks is a property of *the release*, not of each individual commit, and stacking 25 minutes onto an already 25-minute per-PR run will make people start skipping HIL. If a PR touches `src/common/update.py`, a path filter can opt it into the full matrix. + +--- + +## 9. Runner hardening checklist + +- **Ephemeral runner** (`--ephemeral`), registered at repo scope, label `vector-hil` +- Unprivileged user, no sudo, no docker socket +- `ACTIONS_RUNNER_HOOK_JOB_STARTED` / `_COMPLETED`: wipe `_work/`, run a bench-health precheck (all boards enumerate and answer), fail the job immediately if the bench is unhealthy rather than producing a confusing test failure +- **No secrets on any self-hosted job.** In particular `WARPED_PINBALL_PRIVATE_KEY` must never be referenced by a job with a `self-hosted` label. Move signing into a dedicated environment restricted to `main` and tags so it is structurally unreachable from HIL. +- Repo setting: Actions → *Require approval for all outside collaborators* at minimum +- Minimal `permissions:` per job — HIL needs `checks: write`, `actions: read`, `contents: read`, nothing more +- Egress firewall on the Pi: github.com / api.github.com / objects.githubusercontent.com plus package mirrors; deny lateral movement into the LAN +- **Pin actions by SHA.** `build_release.yml` already does this correctly; `deploy_docs.yml`, `docs-on-pr.yml`, `specialfeatures-on-pr.yml`, and `validate-json-configs.yml` use floating `@v4` tags. Tighten those as part of this work — a compromised tag on any of them is a path to the same runner. +- `timeout-minutes` on every job, sized just above the measured worst case + +## 10. Risk register + +**Software-only reset means a wedged board stops the bench.** A PR — malicious or just buggy — can leave a board where `mpremote` can't reach the REPL: a `boot.py` that hard-faults, a tight loop blocking the USB REPL, or a corrupted filesystem. With no BOOTSEL control, recovery needs someone physically present. The config matrix touches every board on every PR, so the exposure is not hypothetical. + +Mitigations that cost nothing: + +- bench-health precheck that fails loudly, and optionally opens an issue, rather than timing out mysteriously +- a documented recovery runbook — `trench-coat/uf2/nuke.uf2` plus a UF2 reflash is the escape hatch +- tight `timeout-minutes` so a hung board doesn't burn an hour of queue + +Worth revisiting after a month: a `uhubctl`-capable powered hub plus BOOTSEL and RUN wired to Pi GPIO (2 pins per board) turns "walk over and press a button" into a test step, for roughly $20. If the bench wedges more than once or twice, buy the hardware. + +Note that `machine.reset()` is a full MCU reset, so `boot.py` and `main.py` do run — the boot-path coverage loss is small. What's genuinely not covered is cold-power-on and brown-out behavior, and FRAM state at power-up. + +**Other risks:** + +| Risk | Mitigation | +|---|---| +| `HDWR01` fires nondeterministically on bare boards | Characterize during bring-up; add a resistor pack if needed (§8, G1) | +| Bench is a singleton; PRs queue | Concurrency group, path filters, add boards as needed | +| Fork PR approved clean, then pushes dirty | Label auto-removed on `synchronize`; environment gate is per-run | +| Harness rots as API changes | Contract tests generated from `@api` docstrings fail when routes drift | +| Flaky HIL erodes trust in CI | Keep it non-blocking until measured flake rate is under ~1% over a week | + +## 11. Rollout + +| Phase | Work | Exit criteria | +|---|---|---| +| 0 | Pi, VLAN, udev rules, 3 boards. Characterize bare-board fault behavior. Measure reset→ready. | 50 consecutive boots produce an identical fault set | +| 1 | `dev/hil` package, boot smoke test, `workflow_dispatch` only | Green run driven by hand | +| 2 | API contract tests over both transports. `workflow_run` trigger on, auto for maintainer branches, fork gate live. Non-blocking check. | Contract tests catch a deliberately broken route | +| 3 | Full config matrix | Under 30 min wall clock; flake rate under 1% over a week → make it a required check | +| 4 | Upgrade matrix, nightly + release tags + label | Full 5-version matrix green on `main` | +| 5 | Expand manifest to `em`, `whitestar`, `classic`; second WPC board if needed | — | + +Phases 0 and 1 are where the real uncertainty lives. Everything after that is mostly typing. + +## 12. Open questions + +1. **Which Pi?** Confirm ARMv7/ARM64 support before ordering. A Pi 4 removes several constraints for not much money. +2. **Bench WiFi credentials** — does the VLAN get its own SSID, or do boards join the existing one with VLAN assignment by MAC? +3. **Should HIL ever be a required check?** Recommendation: yes for G1–G3 once flake rate is measured, never for G4 (too slow, and it's a release-time concern). +4. **How many versions back do we actually support?** The design says 5 stable releases; if the real support window is different, that's a one-line change to the matrix. +5. **Do we want a `latest.json` fixture served from the Pi** to make `/api/update/check` testable, or leave that endpoint untested? From 41f53e3bd05ea145c64615e55326f5b65f8f9931 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 15 Aug 2026 02:04:47 +0000 Subject: [PATCH 02/22] ci: pin all actions and Python dependencies; update HIL design Pin every GitHub Action to a commit SHA with the version in a trailing comment. deploy_docs, docs-on-pr, specialfeatures-on-pr, validate-json-configs and version-bump-guard were on floating @v4/@v5 tags; build_release was pinned but to older revisions. All six now pin to the latest patch of the major version already in use, so this is a supply-chain change with no behaviour change. Pin every dependency in dev/requirements.txt. mpy-cross was the notable gap: it emits the .mpy bytecode the boards must import, and leaving it unpinned meant a future release could silently change the bytecode version and produce builds no deployed board can load. Pinned to 1.28.0.post2, the version CI was already resolving. Verified a full sys11 build succeeds and emits 41 .mpy files at bytecode version 6, matching what 1.23.0 produces. Design doc updates: the upgrade matrix now runs on every PR rather than nightly, with the combined per-PR wall-clock cost quantified and the levers to reduce it listed; runner host section reflects that the Pi is a Zero 2 W; new open question about which mpy-cross version the build should actually target, since the shipped UF2s carry MicroPython v1.24.1 for System 11/9 and v1.26.0-preview for WPC and Data East. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01WPbnmZNF6DFnLQBtpQpcW4 --- .github/workflows/build_release.yml | 12 ++-- .github/workflows/deploy_docs.yml | 4 +- .github/workflows/docs-on-pr.yml | 4 +- .github/workflows/specialfeatures-on-pr.yml | 4 +- .github/workflows/validate-json-configs.yml | 4 +- .github/workflows/version-bump-guard.yml | 6 +- dev/hil/DESIGN.md | 79 +++++++++++++++++---- dev/requirements.txt | 21 +++--- 8 files changed, 95 insertions(+), 39 deletions(-) diff --git a/.github/workflows/build_release.yml b/.github/workflows/build_release.yml index a81d2451..098c44d0 100644 --- a/.github/workflows/build_release.yml +++ b/.github/workflows/build_release.yml @@ -15,10 +15,10 @@ jobs: pull-requests: write steps: - name: Check out code - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Set up Python - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.10" @@ -71,7 +71,7 @@ jobs: - name: Upload update artifacts if: ${{ github.event_name == 'pull_request' }} - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: update-files path: pr-artifacts @@ -88,7 +88,7 @@ jobs: - name: Publish raw update files if: ${{ github.event_name == 'pull_request' }} id: publish_raw_updates - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 with: script: | const fs = require('fs/promises'); @@ -224,7 +224,7 @@ jobs: - name: Comment with update artifact links if: ${{ github.event_name == 'pull_request' }} - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 env: RAW_LINKS_JSON: ${{ steps.publish_raw_updates.outputs.links }} with: @@ -343,7 +343,7 @@ jobs: - name: Create or update GitHub Release if: ${{ env.SHOULD_RELEASE == 'true' }} id: create_release - uses: softprops/action-gh-release@a74c6b72af54cfa997e81df42d94703d6313a2d0 # v2.0.6 + uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2.6.2 with: tag_name: ${{ env.VERSION }} name: ${{ env.VERSION }} diff --git a/.github/workflows/deploy_docs.yml b/.github/workflows/deploy_docs.yml index d387bc7b..d262ef0c 100644 --- a/.github/workflows/deploy_docs.yml +++ b/.github/workflows/deploy_docs.yml @@ -13,10 +13,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.11" diff --git a/.github/workflows/docs-on-pr.yml b/.github/workflows/docs-on-pr.yml index dcf62faf..5c140016 100644 --- a/.github/workflows/docs-on-pr.yml +++ b/.github/workflows/docs-on-pr.yml @@ -15,14 +15,14 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: repository: ${{ github.event.pull_request.head.repo.full_name }} fetch-depth: 0 ref: ${{ github.head_ref }} - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: '3.11' diff --git a/.github/workflows/specialfeatures-on-pr.yml b/.github/workflows/specialfeatures-on-pr.yml index b660928c..c13866f6 100644 --- a/.github/workflows/specialfeatures-on-pr.yml +++ b/.github/workflows/specialfeatures-on-pr.yml @@ -16,14 +16,14 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: repository: ${{ github.event.pull_request.head.repo.full_name }} fetch-depth: 0 ref: ${{ github.head_ref }} - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: '3.11' diff --git a/.github/workflows/validate-json-configs.yml b/.github/workflows/validate-json-configs.yml index a2525152..ea9eb2b5 100644 --- a/.github/workflows/validate-json-configs.yml +++ b/.github/workflows/validate-json-configs.yml @@ -14,10 +14,10 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.11" diff --git a/.github/workflows/version-bump-guard.yml b/.github/workflows/version-bump-guard.yml index 5a6976b3..df1894ce 100644 --- a/.github/workflows/version-bump-guard.yml +++ b/.github/workflows/version-bump-guard.yml @@ -16,14 +16,14 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout PR branch - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: repository: ${{ github.event.pull_request.head.repo.full_name }} fetch-depth: 0 ref: ${{ github.head_ref }} - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.11" @@ -59,4 +59,4 @@ jobs: git config user.email "41898282+github-actions[bot]@users.noreply.github.com" git add src/common/SharedState.py src/em/systemConfig.py src/sys11/systemConfig.py src/wpc/systemConfig.py src/data_east/systemConfig.py src/whitestar/systemConfig.py src/classic/systemConfig.py git commit -m "ci: auto-bump required patch versions" - git push origin HEAD:${{ github.head_ref }} \ No newline at end of file + git push origin HEAD:${{ github.head_ref }} diff --git a/dev/hil/DESIGN.md b/dev/hil/DESIGN.md index 6c6185b4..774abcdb 100644 --- a/dev/hil/DESIGN.md +++ b/dev/hil/DESIGN.md @@ -14,7 +14,9 @@ The first four things we want covered: | G1 | The code boots and the boards run normally | every PR | | G2 | We can connect to the boards and hit the API | every PR | | G3 | Every available config can be parsed and boot | every PR | -| G4 | Updating from the last 5 versions to the proposed version works | nightly + release tags + on-demand | +| G4 | Updating from the last 5 versions to the proposed version works | every PR | + +G4 runs on every PR by decision: the update path is critical infrastructure, it is where we most suspect latent problems, and we would rather pay the wall-clock cost than find out at release time. §8 quantifies the cost and §11 lists the levers to dial it back if it becomes painful. Non-goals for v1: gameplay/bus-level correctness (bare boards, no machine attached), AP-mode setup flow (needs a wire to GPIO22), long-duration soak testing. @@ -29,6 +31,7 @@ Non-goals for v1: gameplay/bus-level correctness (bare boards, no machine attach | Board recovery | Software reset only (`machine.reset()` over `mpremote`). | | Update signing | `skip_signature_check: true` for the upgrade path, plus negative tests that the signature gate still rejects bad packages. | | Initial inventory | `sys11`, `wpc`, `data_east`. More systems added later. | +| Update matrix cadence | Every PR. Updates are the priority; dial back later only if wall clock becomes a problem. | --- @@ -129,13 +132,15 @@ Belt and braces, but the label alone is not enough — label state and head SHA ### Runner host -**Check the Pi model before buying.** The GitHub Actions runner ships for Linux x64, ARM64, and ARM32 (ARMv7). The original Pi Zero / Zero W is ARMv6 (`ARM1176`) and is **not supported** — the runner will not start. Pi Zero 2 W (Cortex-A53) works under a 32-bit ARMv7 userland. Verify against GitHub's current supported-architecture list before ordering. +The existing Pi already runs the Actions runner with a USB hub attached, so the hardware question is largely settled. Worth noting for the record: **that Pi is a Zero 2 W, not a Zero 1 W.** The GitHub Actions runner ships only for Linux x64, ARM64, and ARM32 (ARMv7); the original Zero/Zero W is ARMv6 (`ARM1176`) and the runner will not start on it at all. A runner that runs is a Zero 2 W (Cortex-A53). No need to verify further — it's proven by the fact that it works. + +Constraints to design around on that hardware: -Even on a Zero 2 W, be aware of: +- **512 MB RAM** shared between the runner, Python, and three concurrent `mpremote` sessions. Workable, but keep per-board test processes lean and avoid loading whole build trees into memory. Swapping to the SD card will hurt badly and will show up as timing flake. +- **Single-core-ish throughput.** Board work is I/O-bound on serial round-trips rather than CPU-bound, which is what makes 3-way parallelism viable at all — but orchestration overhead is not free at this scale. +- **USB hub already present.** Confirm it is self-powered; three Picos plus enumeration churn on a Zero 2 W's OTG port is more than the bus-powered case wants to supply. -- **512 MB RAM** shared between the runner, Python, and 3+ concurrent `mpremote` sessions. Tight but workable; swap on the SD card will hurt. -- **One micro-USB OTG data port.** Three boards means a powered hub, and it must be self-powered. -- Wall-clock matters here — the config matrix is minutes of `mpremote` round-trips, and a Zero 2 W is slow at everything. A Pi 4 costs a little more and removes the whole category of problem. Worth it if the bench is going to be load-bearing. +If the combined per-PR matrix (§8) turns out too slow, the first thing to try is more boards rather than a bigger Pi — the work is serial-I/O-bound, not compute-bound, so a faster host buys much less than a second WPC board does. ### Board network @@ -347,9 +352,49 @@ The update path is the most security-sensitive code we ship, and these are cheap The negative tests are what buy back the coverage lost by using `skip_signature_check` for the happy path. -**Runtime:** 5 versions × 3 boards, ~3–5 min each (two OTA cycles plus verification), parallel across boards → ~20–25 min. That is *on top of* the config matrix. +#### Additional cases worth having, given updates are the suspected problem area + +- **Repeated updates in one power cycle.** Apply an update, then apply another without a power cycle in between. `apply_update()` runs inside `LowMemoryMode`, which halts the phew scheduler and closes the discovery sockets on entry and rebuilds them on exit (`update.py:157-196`). If `__exit__` doesn't fully restore that state, the second update is where it shows. Field users do sometimes update twice in a row. +- **Chained vs. direct upgrades.** The common case is a direct jump from an old version to the newest, and that's what the main matrix covers. A chained walk (V-5 → V-4 → … → proposed) additionally catches migration-ordering bugs, where each individual hop works but the sequence doesn't. Worth running on release tags even if it's too slow for every PR. +- **Bytecode compatibility (see the toolchain note below).** After an update, assert every `.mpy` on the board actually imports. A bytecode-version mismatch produces a board that updates "successfully" and then fails to boot — which looks exactly like a mysterious update bug. + +#### ⚠️ Toolchain finding: `mpy-cross` was unpinned + +Worth surfacing here because it lands squarely in the "we suspect there might be issues with updates" category. + +`dev/requirements.txt` pinned `mpremote==1.23.0` but left **`mpy-cross` unpinned**, so CI resolved whatever was newest — currently `1.28.0.post2`. Meanwhile the MicroPython in the shipped UF2s is not uniform: + +| Firmware | MicroPython | +|---|---| +| `Vector_WPC_v5.uf2` | v1.26.0-preview.255 | +| `Vector_DataEast_v1.uf2` | v1.26.0-preview.255 | +| `vector_system_11_and_9_v4.uf2` | v1.24.1 | + +`.mpy` files carry a bytecode version in their header, and a board's MicroPython refuses to import a `.mpy` whose version it doesn't know. So the build was compiling bytecode with a 1.28 toolchain and shipping it to firmware three to four minor versions behind, with nothing asserting the pairing is valid. -**Recommendation:** run G4 nightly on `main`, on release tags, and on-demand via label — not on every PR. The risk that an upgrade from 1.11.24 breaks is a property of *the release*, not of each individual commit, and stacking 25 minutes onto an already 25-minute per-PR run will make people start skipping HIL. If a PR touches `src/common/update.py`, a path filter can opt it into the full matrix. +It happens to work today — verified empirically that `mpy-cross` 1.23.0 and 1.28.0.post2 both emit `mpy_version=6, flags=0x00`, and a full `sys11` build produces 41 `.mpy` files all at version 6. So this is not a live bug. But it is unpinned, undocumented, and load-bearing: the day `mpy-cross` bumps to bytecode version 7, every build silently produces modules that no deployed board can import, and every OTA update bricks on the next boot. That failure would be very hard to diagnose from the symptom. + +`mpy-cross` is now pinned to `1.28.0.post2` — deliberately the version CI was already resolving, so the pin freezes current behavior rather than changing it. **The open question is what it *should* be pinned to**, which is a hardware question we can't answer from the repo: it should match the MicroPython in each target's UF2, and today those differ per target while the build uses one toolchain for all of them. See §12. + +This is also the single best argument for the update matrix running on every PR: it's exactly the class of problem where the build is green, the unit tests pass, and only real hardware tells you. + +**Runtime:** 5 versions × 3 boards, ~3–5 min each (two OTA cycles plus verification), parallel across board types → ~20–25 min for the upgrade matrix alone. + +**Combined per-PR cost.** G3 and G4 contend for the same physical boards, so they serialize per board rather than overlapping. The WPC board is the critical path both times: + +| | wpc board | sys11 board | data_east board | +|---|---|---|---| +| Config matrix (G3) | ~16–26 min | ~10–16 min | ~7–12 min | +| Upgrade matrix (G4) | ~15–25 min | ~15–25 min | ~15–25 min | +| **Serial total** | **~31–51 min** | ~25–41 min | ~22–37 min | + +So expect **roughly 30–50 minutes of bench occupancy per PR**, on a singleton bench, with PRs queueing behind each other. That is the accepted cost of treating updates as critical infrastructure. Levers, in the order worth reaching for: + +1. **Run G4 first.** The most valuable signal arrives earliest, and a broken update fails the run before spending 26 minutes on configs. +2. **A second WPC board.** Config matrix on board A, upgrade matrix on board B, genuinely in parallel — cuts the critical path from ~51 to ~26 min. This is the highest-leverage purchase on the whole bench and the manifest already supports pools. +3. **Path filters.** Docs-only and workflow-only PRs skip HIL entirely. +4. **Poll readiness over USB, not HTTP** — saves seconds on every one of ~180 boot cycles per run. +5. If it still hurts: move the *chained* upgrade walk to release tags and keep only direct jumps per PR. --- @@ -362,7 +407,7 @@ The negative tests are what buy back the coverage lost by using `skip_signature_ - Repo setting: Actions → *Require approval for all outside collaborators* at minimum - Minimal `permissions:` per job — HIL needs `checks: write`, `actions: read`, `contents: read`, nothing more - Egress firewall on the Pi: github.com / api.github.com / objects.githubusercontent.com plus package mirrors; deny lateral movement into the LAN -- **Pin actions by SHA.** `build_release.yml` already does this correctly; `deploy_docs.yml`, `docs-on-pr.yml`, `specialfeatures-on-pr.yml`, and `validate-json-configs.yml` use floating `@v4` tags. Tighten those as part of this work — a compromised tag on any of them is a path to the same runner. +- ~~**Pin actions by SHA.**~~ **Done in this PR.** All six workflows now pin every action to a commit SHA with the version in a trailing comment, and `dev/requirements.txt` pins every Python dependency. Previously `deploy_docs.yml`, `docs-on-pr.yml`, `specialfeatures-on-pr.yml`, `validate-json-configs.yml`, and `version-bump-guard.yml` used floating `@v4`/`@v5` tags — a compromised or repointed tag on any of them is a path to the same runner the bench will be attached to. Keep this invariant: **no floating tags, ever**, and consider a CI check that greps for `uses:.*@v[0-9]` to enforce it. - `timeout-minutes` on every job, sized just above the measured worst case ## 10. Risk register @@ -388,6 +433,9 @@ Note that `machine.reset()` is a full MCU reset, so `boot.py` and `main.py` do r | Fork PR approved clean, then pushes dirty | Label auto-removed on `synchronize`; environment gate is per-run | | Harness rots as API changes | Contract tests generated from `@api` docstrings fail when routes drift | | Flaky HIL erodes trust in CI | Keep it non-blocking until measured flake rate is under ~1% over a week | +| `mpy-cross` bytecode version drifts away from shipped firmware | Now pinned; add a post-update HIL assertion that every `.mpy` imports (§8, G4) | +| ~30–50 min per-PR bench occupancy on a singleton bench | Run G4 first, path filters, second WPC board when it bites (§8) | +| 512 MB RAM on the Zero 2 W under 3-way parallelism | Keep test processes lean; watch for swap-induced timing flake | ## 11. Rollout @@ -396,16 +444,19 @@ Note that `machine.reset()` is a full MCU reset, so `boot.py` and `main.py` do r | 0 | Pi, VLAN, udev rules, 3 boards. Characterize bare-board fault behavior. Measure reset→ready. | 50 consecutive boots produce an identical fault set | | 1 | `dev/hil` package, boot smoke test, `workflow_dispatch` only | Green run driven by hand | | 2 | API contract tests over both transports. `workflow_run` trigger on, auto for maintainer branches, fork gate live. Non-blocking check. | Contract tests catch a deliberately broken route | -| 3 | Full config matrix | Under 30 min wall clock; flake rate under 1% over a week → make it a required check | -| 4 | Upgrade matrix, nightly + release tags + label | Full 5-version matrix green on `main` | -| 5 | Expand manifest to `em`, `whitestar`, `classic`; second WPC board if needed | — | +| 3 | **Upgrade matrix, every PR.** Promoted ahead of the config matrix — it's the highest-value signal and the suspected problem area. | Full 5-version matrix green across all three boards | +| 4 | Full config matrix | Combined run under ~50 min; flake rate under 1% over a week → make it a required check | +| 5 | Expand manifest to `em`, `whitestar`, `classic`; second WPC board to parallelize G3 against G4 | — | Phases 0 and 1 are where the real uncertainty lives. Everything after that is mostly typing. +Note that phases 3 and 4 are deliberately ordered opposite to the goal numbering. The update path is the reason this bench exists, so it should be the first thing running on every PR — the config matrix is more coverage but less risk per unit of wall clock. + ## 12. Open questions -1. **Which Pi?** Confirm ARMv7/ARM64 support before ordering. A Pi 4 removes several constraints for not much money. +1. **What should `mpy-cross` be pinned to?** It's now frozen at `1.28.0.post2` (what CI already resolved), but the shipped UF2s carry MicroPython v1.24.1 for System 11/9 and v1.26.0-preview for WPC and Data East. Ideally the build toolchain matches the target's firmware, which today would mean a per-target `mpy-cross` rather than one for all of them. Needs a hardware decision: standardize the firmware across targets, or make the build toolchain per-target. See §8. 2. **Bench WiFi credentials** — does the VLAN get its own SSID, or do boards join the existing one with VLAN assignment by MAC? -3. **Should HIL ever be a required check?** Recommendation: yes for G1–G3 once flake rate is measured, never for G4 (too slow, and it's a release-time concern). +3. **Should HIL be a required check?** Recommendation: yes for G1–G4 once the flake rate is measured over a week. With G4 running per-PR by design, making it advisory-only would waste most of its value. 4. **How many versions back do we actually support?** The design says 5 stable releases; if the real support window is different, that's a one-line change to the matrix. 5. **Do we want a `latest.json` fixture served from the Pi** to make `/api/update/check` testable, or leave that endpoint untested? +6. **Is the existing USB hub self-powered?** Three Picos on a Zero 2 W's OTG port wants a powered hub. diff --git a/dev/requirements.txt b/dev/requirements.txt index 45918634..b16e68e3 100644 --- a/dev/requirements.txt +++ b/dev/requirements.txt @@ -1,9 +1,14 @@ mpremote==1.23.0 -mpy-cross -csscompressor -jsmin -scour -htmlmin2 -beautifulsoup4 -cryptography -pre-commit +# mpy-cross emits .mpy bytecode that the board's MicroPython must be able to +# import. Leaving it unpinned means a future release can silently bump the +# bytecode version and produce builds no deployed board can load. Pinned to the +# version CI was already resolving; see dev/hil/DESIGN.md for the open question +# about aligning this with the MicroPython in each target's UF2. +mpy-cross==1.28.0.post2 +csscompressor==0.9.5 +jsmin==3.0.1 +scour==0.38.2 +htmlmin2==0.1.13 +beautifulsoup4==4.15.0 +cryptography==50.0.0 +pre-commit==4.6.2 From 9653c322a3d04bca1d0086a939cf19cc957ea23c Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 16 Aug 2026 16:41:40 +0000 Subject: [PATCH 03/22] docs: add HIL runner setup runbook for the Raspberry Pi Step-by-step setup for the Pi Zero 2 W bench runner: memory headroom, unprivileged runner user, pinned Python environment, stable board device names via udev, network isolation verification, bench manifest, Actions runner install with verified checksums, job hooks, and an end-to-end smoke workflow. Each phase ends with a verification step. Hook scripts and YAML blocks are syntax-checked and the pre-job health check is behaviour-tested for both the healthy and missing-board paths. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01WPbnmZNF6DFnLQBtpQpcW4 --- dev/hil/RUNNER_SETUP.md | 600 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 600 insertions(+) create mode 100644 dev/hil/RUNNER_SETUP.md diff --git a/dev/hil/RUNNER_SETUP.md b/dev/hil/RUNNER_SETUP.md new file mode 100644 index 00000000..0bd2b950 --- /dev/null +++ b/dev/hil/RUNNER_SETUP.md @@ -0,0 +1,600 @@ +# HIL Bench — Raspberry Pi Runner Setup + +How to turn a Raspberry Pi Zero 2 W with Vector boards attached into a GitHub Actions +self-hosted runner that can execute the hardware tests described in [DESIGN.md](DESIGN.md). + +**Assumes:** the Pi is assembled, the boards are wired to a powered USB hub, Raspberry Pi OS +is installed, and you can reach the Pi over Raspberry Pi Connect. Everything below is done +from a shell on the Pi. + +**Produces:** a runner labelled `vector-hil` attached to `warped-pinball/vector`, a pinned +Python environment, stable device names for each board, and a health check that fails a job +early instead of confusingly. + +Work through the phases in order. Each ends with a verification step — don't move on until +it passes. + +--- + +## Phase 0 — Confirm the starting point + +```bash +cat /etc/os-release | grep PRETTY_NAME +uname -m # kernel architecture +dpkg --print-architecture # userland architecture <- this is the one that matters +free -h +python3 -V +``` + +Two things to note before continuing. + +**Use `dpkg --print-architecture`, not `uname -m`, to pick the runner build.** Raspberry Pi OS +routinely runs a 64-bit kernel with a 32-bit userland, so `uname -m` reports `aarch64` while +the userland is `armhf` and only the 32-bit runner will work. `dpkg --print-architecture` +reports the userland, which is what the runner binary has to match. + +| `dpkg --print-architecture` | Runner build | +|---|---| +| `armhf` | `linux-arm` | +| `arm64` | `linux-arm64` | + +**Raspberry Pi OS Lite is strongly preferred on 512 MB.** If this is a desktop image, the +runner will be fighting the compositor for RAM. Also note Bookworm ships Python 3.11 with +PEP 668 enabled, so `pip install` outside a virtualenv is refused — Phase 3 uses a venv. + +--- + +## Phase 1 — Memory headroom + +512 MB is the real constraint on this box. The runner is a .NET process (~150–250 MB +resident), plus Python, plus three concurrent `mpremote` sessions. Without help it will +OOM-kill mid-job, and an OOM kill in the middle of a config sweep looks exactly like a +firmware hang — you'll waste an afternoon on it. + +Add compressed RAM swap first, since it costs no SD card writes: + +```bash +sudo apt update +sudo apt install -y zram-tools +sudo sed -i 's/^#\?ALGO=.*/ALGO=zstd/;s/^#\?PERCENT=.*/PERCENT=60/' /etc/default/zramswap +sudo systemctl restart zramswap +``` + +Then raise the on-disk swapfile as an overflow tier: + +```bash +sudo dphys-swapfile swapoff +sudo sed -i 's/^CONF_SWAPSIZE=.*/CONF_SWAPSIZE=1024/' /etc/dphys-swapfile +sudo sed -i 's/^#\?CONF_MAXSWAP=.*/CONF_MAXSWAP=2048/' /etc/dphys-swapfile +sudo dphys-swapfile setup +sudo dphys-swapfile swapon +``` + +Prefer zram over the SD card, and don't let the kernel swap eagerly: + +```bash +echo 'vm.swappiness=10' | sudo tee /etc/sysctl.d/99-hil.conf +sudo sysctl --system +``` + +**Verify:** + +```bash +swapon --show # expect /dev/zram0 (higher priority) and /var/swap +free -h +``` + +You want zram listed with a higher priority number than the file swap. If you later see jobs +slow to a crawl, check `vmstat 1` for sustained `si`/`so` — that's disk swap thrashing, and +the fix is fewer parallel boards, not a bigger swapfile. + +--- + +## Phase 2 — User, packages, layout + +Run the runner as a dedicated unprivileged user. It needs `dialout` for serial access to the +boards. + +```bash +sudo adduser --disabled-password --gecos "" hilrunner +sudo usermod -aG dialout hilrunner +``` + +Deliberately **no sudo rights** for `hilrunner`, and no docker group. + +```bash +sudo apt install -y git curl jq python3-venv python3-full usbutils +``` + +Directory layout — runner and harness support files kept separate: + +```bash +sudo mkdir -p /opt/vector-hil/{hooks,logs} +sudo mkdir -p /etc/vector-hil +sudo mkdir -p /opt/actions-runner +sudo chown -R hilrunner:hilrunner /opt/vector-hil /opt/actions-runner +sudo chown root:hilrunner /etc/vector-hil +sudo chmod 750 /etc/vector-hil +``` + +--- + +## Phase 3 — Python environment + +The workflow runs from a trusted ref and shouldn't be building a venv on every job — that's +minutes of wall clock on this hardware. Create it once, pinned: + +```bash +sudo -u hilrunner python3 -m venv /opt/vector-hil/venv +sudo -u hilrunner /opt/vector-hil/venv/bin/pip install --upgrade pip +sudo -u hilrunner /opt/vector-hil/venv/bin/pip install \ + mpremote==1.23.0 \ + pytest==9.1.1 \ + pytest-timeout==2.5.0 \ + pytest-xdist==3.8.0 \ + PyYAML==6.0.3 \ + requests==2.34.2 +``` + +`mpremote` is pinned to 1.23.0 to match `dev/requirements.txt`. `pytest-xdist` is what lets +the suite fan out across boards — the design's parallelism assumes it. + +`mpy-cross` is deliberately **not** installed here. Builds happen on GitHub-hosted runners; +the Pi only consumes artifacts. If you find yourself wanting to build on the Pi, stop and +re-read the threat model in DESIGN.md §3 — that's the boundary this whole design rests on. + +**Verify:** + +```bash +sudo -u hilrunner /opt/vector-hil/venv/bin/python -c "import pytest, yaml, requests; print('ok')" +sudo -u hilrunner /opt/vector-hil/venv/bin/mpremote --help >/dev/null && echo "mpremote ok" +``` + +--- + +## Phase 4 — Stable board device names + +`/dev/ttyACM0` ordering is not stable across reboots or re-enumeration. If the harness +addresses boards that way, a reboot silently swaps which board runs which test suite and you +get results attributed to the wrong hardware. Pin each board to a symlink by its USB serial. + +With all boards plugged in, list them: + +```bash +/opt/vector-hil/venv/bin/mpremote devs +``` + +For each port, get the identifying attributes: + +```bash +for d in /dev/ttyACM*; do + echo "=== $d" + udevadm info -a -n "$d" 2>/dev/null | grep -m3 -E 'ATTRS\{(idVendor|idProduct|serial)\}' +done +``` + +Raspberry Pi's vendor ID is `2e8a`; a Pico running MicroPython typically enumerates as +product `0005`. **Use whatever the command above actually prints** rather than trusting those +values — the firmware build determines them. + +Identify which physical board is which by asking each one: + +```bash +for d in /dev/ttyACM*; do + echo -n "$d -> " + /opt/vector-hil/venv/bin/mpremote connect "$d" exec \ + "import systemConfig; print(systemConfig.vectorSystem, systemConfig.SystemVersion)" 2>/dev/null \ + || echo "(no response)" +done +``` + +That's the same probe `dev/detect_boards.py` uses. Now write the rules, substituting the real +serial numbers: + +```bash +sudo tee /etc/udev/rules.d/99-vector-hil.rules >/dev/null <<'EOF' +# Vector HIL bench — stable names by USB serial. +# Get serials with: udevadm info -a -n /dev/ttyACM0 | grep ATTRS{serial} +SUBSYSTEM=="tty", ATTRS{idVendor}=="2e8a", ATTRS{serial}=="REPLACE_SYS11_SERIAL", SYMLINK+="vector-sys11-a", GROUP="dialout", MODE="0660" +SUBSYSTEM=="tty", ATTRS{idVendor}=="2e8a", ATTRS{serial}=="REPLACE_WPC_SERIAL", SYMLINK+="vector-wpc-a", GROUP="dialout", MODE="0660" +SUBSYSTEM=="tty", ATTRS{idVendor}=="2e8a", ATTRS{serial}=="REPLACE_DATA_EAST_SERIAL", SYMLINK+="vector-data-east-a", GROUP="dialout", MODE="0660" +EOF + +sudo udevadm control --reload-rules +sudo udevadm trigger --subsystem-match=tty +``` + +**Verify:** + +```bash +ls -l /dev/vector-* +``` + +You should see three symlinks. Unplug and replug the hub, confirm they come back pointing at +the right boards, and confirm `hilrunner` can open them: + +```bash +sudo -u hilrunner /opt/vector-hil/venv/bin/mpremote connect /dev/vector-wpc-a \ + exec "import systemConfig; print(systemConfig.vectorSystem)" +``` + +If that fails with a permissions error, `hilrunner`'s `dialout` membership hasn't taken +effect — log out and back in, or reboot. + +--- + +## Phase 5 — Network + +DESIGN.md §5 puts the boards on a VLAN isolated from the LAN. The Zero 2 W has a single +radio and no ethernet, which constrains how that gets built: **the Pi cannot be on the LAN +and the bench VLAN at the same time.** + +The workable arrangement with one radio: + +- The bench VLAN has its own SSID. The Pi joins **only** that SSID. +- On the router: allow the Pi's MAC out to the internet (it needs github.com); deny the board + MACs any WAN access; deny the whole VLAN any access to the LAN. +- Boards reach the Pi's update server directly — same subnet, no routing needed. +- Give each board a static DHCP lease so the manifest IPs stay put. + +If your router can't do per-MAC egress rules on a VLAN, the fallback is a USB ethernet +adapter on the hub for the LAN uplink, leaving `wlan0` for the bench. Note that this is +router configuration, not Pi configuration — the Pi side is just joining the right SSID. + +**Verify from the Pi:** + +```bash +curl -sS -o /dev/null -w "github: %{http_code}\n" https://api.github.com +ping -c2 10.42.7.11 # a board's static lease +``` + +**Verify isolation actually holds** — this is the part people skip and regret. From a board's +REPL, confirm it cannot reach the LAN: + +```bash +/opt/vector-hil/venv/bin/mpremote connect /dev/vector-wpc-a exec " +import socket +try: + s = socket.socket(); s.settimeout(3) + s.connect(socket.getaddrinfo('192.168.1.1', 80)[0][-1]) # your LAN gateway + print('REACHABLE - isolation is NOT working') +except Exception as e: + print('blocked (good):', e) +" +``` + +If that prints `REACHABLE`, stop and fix the router before attaching the runner. The entire +security argument for running untrusted firmware on this bench depends on that path being +closed. + +### Optional: DNS override for `/api/update/check` + +`/api/update/check` fetches `software.warpedpinball.com`. Rather than opening egress for it, +serve a canned response locally so the test is deterministic (DESIGN.md §5): + +```bash +sudo apt install -y dnsmasq +echo "address=/software.warpedpinball.com/10.42.7.1" | \ + sudo tee /etc/dnsmasq.d/vector-hil.conf +sudo systemctl restart dnsmasq +``` + +Only do this if the Pi is the VLAN's DNS server. If the router handles DHCP/DNS there, put +the override on the router instead — running a second DHCP server on the segment will cause +problems that are annoying to diagnose. + +--- + +## Phase 6 — Bench manifest and secrets + +The manifest describes the bench; it lives on the Pi, not in git (DESIGN.md §6). + +```bash +sudo tee /etc/vector-hil/bench.yaml >/dev/null <<'EOF' +boards: + - id: sys11-a + target: sys11 + hardware: sys11 + serial: /dev/vector-sys11-a + ip: 10.42.7.11 + - id: wpc-a + target: wpc + hardware: wpc + serial: /dev/vector-wpc-a + ip: 10.42.7.12 + - id: data-east-a + target: data_east + hardware: data_east + serial: /dev/vector-data-east-a + ip: 10.42.7.13 + +network: + update_server: http://10.42.7.1:8080 + +secrets_file: /etc/vector-hil/secrets.yaml +EOF + +sudo tee /etc/vector-hil/secrets.yaml >/dev/null <<'EOF' +wifi_ssid: "BENCH-SSID" +wifi_password: "..." +game_password: "..." +EOF + +sudo chown root:hilrunner /etc/vector-hil/bench.yaml /etc/vector-hil/secrets.yaml +sudo chmod 640 /etc/vector-hil/bench.yaml /etc/vector-hil/secrets.yaml +``` + +These are bench credentials for an isolated segment, not production secrets — but keep them +off GitHub regardless. Group-readable by `hilrunner` and nothing wider. + +Adding a board later is a udev rule plus an entry here. No test changes. + +--- + +## Phase 7 — Install the Actions runner + +Pick the build from Phase 0's `dpkg --print-architecture`. + +```bash +cd /opt/actions-runner +RUNNER_VERSION=2.336.0 + +# armhf userland: +RUNNER_ARCH=arm RUNNER_SHA=44a300f322a1b5bccfe0b146cf3ca74f27000eb8afed761d1ffd90be035969d4 +# arm64 userland — use these two lines instead: +# RUNNER_ARCH=arm64 RUNNER_SHA=58b758e420b87093fbd4bfddd368074960053e2f1388f01848c82624b90f27d1 + +sudo -u hilrunner curl -fSL -o runner.tar.gz \ + "https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/actions-runner-linux-${RUNNER_ARCH}-${RUNNER_VERSION}.tar.gz" + +echo "${RUNNER_SHA} runner.tar.gz" | sha256sum -c - || { echo "CHECKSUM MISMATCH"; rm -f runner.tar.gz; } +sudo -u hilrunner tar xzf runner.tar.gz && sudo -u hilrunner rm runner.tar.gz +``` + +Those checksums were computed from the published v2.336.0 artifacts. Verify before extracting +— on a box that will be handling artifacts from public PRs, "I'll check it later" is how you +end up not checking it. + +The runner is a .NET application and needs a few system libraries: + +```bash +sudo ./bin/installdependencies.sh +``` + +Get a registration token from +**Settings → Actions → Runners → New self-hosted runner** on `warped-pinball/vector`. It is +valid for one hour and is single-use. + +```bash +sudo -u hilrunner ./config.sh \ + --url https://github.com/warped-pinball/vector \ + --token \ + --name vector-hil-pi \ + --labels vector-hil \ + --work _work \ + --unattended --replace +``` + +The `vector-hil` label is what `hil.yml` will target via +`runs-on: [self-hosted, vector-hil]`. Don't rely on the bare `self-hosted` label — the moment +there's a second runner anywhere in the org, jobs start landing on the wrong machine. + +### A note on ephemeral runners + +DESIGN.md §9 calls for `--ephemeral`. Deliberately not doing that in this first setup, for a +specific reason: an ephemeral runner deregisters after every job, so something has to mint a +fresh registration token for each one. That means storing a PAT with `administration: write` +on the Pi — readable by the same user that runs job code. That PAT is a considerably more +valuable secret than the runner's own credentials, which only let you receive jobs. + +The trade is acceptable here because of the trust model: under `workflow_run`, the Pi only +ever executes harness code from the default branch (DESIGN.md §4). It never runs PR-authored +host code, so the "poison the workspace for the next job" attack that ephemeral runners +defend against doesn't have a foothold. The job hooks in Phase 8 cover the rest. + +Revisit this if the Pi ever starts executing PR-authored code — at that point ephemeral +runners via JIT config stop being optional. + +--- + +## Phase 8 — Job hooks + +Two hooks: a pre-job health check that fails fast when the bench isn't sane, and a post-job +cleanup. + +```bash +sudo -u hilrunner tee /opt/vector-hil/hooks/pre-job.sh >/dev/null <<'EOF' +#!/usr/bin/env bash +# Fail the job immediately if the bench isn't healthy, rather than letting the +# tests fail in a way that looks like a firmware regression. +set -euo pipefail + +VENV=/opt/vector-hil/venv/bin +MANIFEST=${VECTOR_HIL_BENCH:-/etc/vector-hil/bench.yaml} + +echo "::group::Bench health check" +rc=0 +for dev in $("$VENV/python" -c " +import yaml +for b in yaml.safe_load(open('$MANIFEST'))['boards']: + print(b['serial']) +"); do + if [[ ! -e "$dev" ]]; then + echo "::error::$dev is missing - board not enumerated" + rc=1 + continue + fi + if out=$(timeout 20 "$VENV/mpremote" connect "$dev" exec \ + "import systemConfig; print(systemConfig.vectorSystem)" 2>&1); then + echo " $dev -> ${out//[$'\r\n']/}" + else + echo "::error::$dev did not respond to mpremote: $out" + rc=1 + fi +done + +avail=$(awk '/MemAvailable/ {print int($2/1024)}' /proc/meminfo) +echo " available memory: ${avail} MB" +if [[ $avail -lt 80 ]]; then + echo "::warning::low memory before job start (${avail} MB)" +fi + +echo "::endgroup::" +exit $rc +EOF + +sudo -u hilrunner tee /opt/vector-hil/hooks/post-job.sh >/dev/null <<'EOF' +#!/usr/bin/env bash +# Best-effort cleanup. Never fail the job from here - the tests already ran. +set -uo pipefail + +VENV=/opt/vector-hil/venv/bin +MANIFEST=${VECTOR_HIL_BENCH:-/etc/vector-hil/bench.yaml} + +for dev in $("$VENV/python" -c " +import yaml +for b in yaml.safe_load(open('$MANIFEST'))['boards']: + print(b['serial']) +" 2>/dev/null); do + [[ -e "$dev" ]] || continue + timeout 15 "$VENV/mpremote" connect "$dev" exec \ + "import machine; machine.reset()" >/dev/null 2>&1 || true +done + +find /opt/actions-runner/_work -mindepth 1 -maxdepth 1 \ + ! -name '_tool' ! -name '_temp' -exec rm -rf {} + 2>/dev/null || true + +exit 0 +EOF + +sudo -u hilrunner chmod +x /opt/vector-hil/hooks/*.sh +``` + +Register them in the runner's `.env`, which the service reads at start: + +```bash +sudo -u hilrunner tee -a /opt/actions-runner/.env >/dev/null <<'EOF' +ACTIONS_RUNNER_HOOK_JOB_STARTED=/opt/vector-hil/hooks/pre-job.sh +ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/opt/vector-hil/hooks/post-job.sh +VECTOR_HIL_BENCH=/etc/vector-hil/bench.yaml +VECTOR_HIL_VENV=/opt/vector-hil/venv +EOF +``` + +`.env` is only read when the service starts, so restart it after any change here. + +**Verify:** + +```bash +sudo -u hilrunner /opt/vector-hil/hooks/pre-job.sh && echo "PRE-JOB OK" +``` + +--- + +## Phase 9 — Run as a service + +```bash +cd /opt/actions-runner +sudo ./svc.sh install hilrunner +sudo ./svc.sh start +sudo ./svc.sh status +``` + +**Verify** the runner shows **Idle** under Settings → Actions → Runners, with the `vector-hil` +label attached. + +```bash +journalctl -u "actions.runner.warped-pinball-vector.vector-hil-pi.service" -f +``` + +The runner auto-updates itself by default. Leave that on — GitHub stops dispatching jobs to +runners that fall too far behind. Just be aware an update pulls ~77 MB, so an occasional job +will start slowly. + +--- + +## Phase 10 — End-to-end check + +Before there's any harness code, confirm the whole path works with a throwaway workflow on a +branch: + +```yaml +name: HIL smoke +on: workflow_dispatch + +jobs: + smoke: + runs-on: [self-hosted, vector-hil] + timeout-minutes: 10 + steps: + - name: Report bench state + run: | + source "$VECTOR_HIL_VENV/bin/activate" + python - <<'PY' + import os, subprocess, yaml + bench = yaml.safe_load(open(os.environ["VECTOR_HIL_BENCH"])) + for b in bench["boards"]: + out = subprocess.run( + ["mpremote", "connect", b["serial"], "exec", + "import systemConfig; print(systemConfig.vectorSystem, systemConfig.SystemVersion)"], + capture_output=True, text=True, timeout=30) + print(f'{b["id"]:14} {out.stdout.strip() or out.stderr.strip()}') + PY +``` + +Dispatch it. A green run that prints all three boards and their versions means the runner, +the venv, the udev names, the manifest, and the hooks are all wired correctly — which is +everything Phase 0–9 was for. + +--- + +## Maintenance and troubleshooting + +**SD card wear.** This bench writes a lot: artifact downloads, `_work` churn, swap. Use a +decent A2 card, keep the file swap modest, and treat the card as consumable. Take an image +once the setup is verified so a rebuild is a restore rather than a repeat of this document. + +**Log growth.** Runner diagnostic logs accumulate in `_diag`: + +```bash +sudo tee /etc/logrotate.d/vector-hil >/dev/null <<'EOF' +/opt/actions-runner/_diag/*.log { + weekly + rotate 4 + compress + missingok + notifempty +} +EOF +``` + +**A board stops responding.** Expected occasionally — recovery is software-only by decision +(DESIGN.md §10), so `machine.reset()` over `mpremote` is the first move. If MicroPython +itself won't come up, the board needs a physical BOOTSEL press and a reflash with +`trench-coat/uf2/nuke.uf2`. The pre-job hook will keep failing jobs loudly until that's done, +which is the intended behaviour — a silently absent board would attribute its tests to +nothing. + +**Jobs get OOM-killed.** Check `dmesg -T | grep -i oom`. Reduce `pytest-xdist` parallelism +before touching swap; three concurrent `mpremote` sessions plus the runner is close to the +ceiling on 512 MB. + +**Runner shows Offline after a reboot.** `sudo ./svc.sh status`; the service is enabled at +install but confirm with `systemctl is-enabled`. + +**Permission denied on `/dev/vector-*`.** `hilrunner` isn't in `dialout` yet, or the group +change hasn't been picked up by the running service. Restart the service. + +**Symlinks point at the wrong board after a hub replug.** A serial number in the udev rules +is wrong or duplicated. Re-run the Phase 4 identification loop. + +--- + +## What this does not cover + +- The `hil.yml` workflow and the `dev/hil` harness — not written yet; see DESIGN.md §4 and §7. +- The `hardware-lab` GitHub Environment and fork gating — repo settings, not Pi settings + (DESIGN.md §4). +- The update file server the OTA tests need on `10.42.7.1:8080` — the harness starts that + itself per-run (DESIGN.md §7, `update_server.py`). +- Router and VLAN configuration, which is where the isolation guarantee actually lives. From 984c4a9c4eb02391d3c23c10e156551dd394f9e5 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 16 Aug 2026 16:57:32 +0000 Subject: [PATCH 04/22] docs: cut runner setup down to the minimum Board detection and flashing already work through the repo's dev pipeline (detect_boards.py identifies boards by querying systemConfig.vectorSystem, so port ordering does not matter), which removes the need for udev rules, a bench manifest, and job hooks at setup time. Drops the swap tuning, dedicated user, udev rules, dnsmasq, bench manifest, secrets file, job hooks, and logrotate config down to a short list of commands. The most likely failure modes are kept as one-line fixes at the bottom rather than pre-emptive configuration. The removed detail remains in git history if any of it turns out to be needed. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01WPbnmZNF6DFnLQBtpQpcW4 --- dev/hil/RUNNER_SETUP.md | 610 ++++------------------------------------ 1 file changed, 55 insertions(+), 555 deletions(-) diff --git a/dev/hil/RUNNER_SETUP.md b/dev/hil/RUNNER_SETUP.md index 0bd2b950..28978051 100644 --- a/dev/hil/RUNNER_SETUP.md +++ b/dev/hil/RUNNER_SETUP.md @@ -1,522 +1,73 @@ -# HIL Bench — Raspberry Pi Runner Setup +# HIL Bench — Pi Runner Setup -How to turn a Raspberry Pi Zero 2 W with Vector boards attached into a GitHub Actions -self-hosted runner that can execute the hardware tests described in [DESIGN.md](DESIGN.md). +Minimal setup to make a Raspberry Pi with Vector boards attached run GitHub Actions jobs. +Assumes the Pi is assembled, boards are on a powered USB hub, and you have a shell on it. -**Assumes:** the Pi is assembled, the boards are wired to a powered USB hub, Raspberry Pi OS -is installed, and you can reach the Pi over Raspberry Pi Connect. Everything below is done -from a shell on the Pi. - -**Produces:** a runner labelled `vector-hil` attached to `warped-pinball/vector`, a pinned -Python environment, stable device names for each board, and a health check that fails a job -early instead of confusingly. - -Work through the phases in order. Each ends with a verification step — don't move on until -it passes. - ---- - -## Phase 0 — Confirm the starting point - -```bash -cat /etc/os-release | grep PRETTY_NAME -uname -m # kernel architecture -dpkg --print-architecture # userland architecture <- this is the one that matters -free -h -python3 -V -``` - -Two things to note before continuing. - -**Use `dpkg --print-architecture`, not `uname -m`, to pick the runner build.** Raspberry Pi OS -routinely runs a 64-bit kernel with a 32-bit userland, so `uname -m` reports `aarch64` while -the userland is `armhf` and only the 32-bit runner will work. `dpkg --print-architecture` -reports the userland, which is what the runner binary has to match. - -| `dpkg --print-architecture` | Runner build | -|---|---| -| `armhf` | `linux-arm` | -| `arm64` | `linux-arm64` | - -**Raspberry Pi OS Lite is strongly preferred on 512 MB.** If this is a desktop image, the -runner will be fighting the compositor for RAM. Also note Bookworm ships Python 3.11 with -PEP 668 enabled, so `pip install` outside a virtualenv is refused — Phase 3 uses a venv. - ---- - -## Phase 1 — Memory headroom - -512 MB is the real constraint on this box. The runner is a .NET process (~150–250 MB -resident), plus Python, plus three concurrent `mpremote` sessions. Without help it will -OOM-kill mid-job, and an OOM kill in the middle of a config sweep looks exactly like a -firmware hang — you'll waste an afternoon on it. - -Add compressed RAM swap first, since it costs no SD card writes: +Board detection and flashing use the repo's existing dev pipeline (`dev/detect_boards.py`, +`dev/sync.py`), so there is nothing bench-specific to configure. ```bash +# --- system packages ------------------------------------------------------- sudo apt update -sudo apt install -y zram-tools -sudo sed -i 's/^#\?ALGO=.*/ALGO=zstd/;s/^#\?PERCENT=.*/PERCENT=60/' /etc/default/zramswap -sudo systemctl restart zramswap -``` - -Then raise the on-disk swapfile as an overflow tier: - -```bash -sudo dphys-swapfile swapoff -sudo sed -i 's/^CONF_SWAPSIZE=.*/CONF_SWAPSIZE=1024/' /etc/dphys-swapfile -sudo sed -i 's/^#\?CONF_MAXSWAP=.*/CONF_MAXSWAP=2048/' /etc/dphys-swapfile -sudo dphys-swapfile setup -sudo dphys-swapfile swapon -``` - -Prefer zram over the SD card, and don't let the kernel swap eagerly: - -```bash -echo 'vm.swappiness=10' | sudo tee /etc/sysctl.d/99-hil.conf -sudo sysctl --system -``` - -**Verify:** - -```bash -swapon --show # expect /dev/zram0 (higher priority) and /var/swap -free -h -``` - -You want zram listed with a higher priority number than the file swap. If you later see jobs -slow to a crawl, check `vmstat 1` for sustained `si`/`so` — that's disk swap thrashing, and -the fix is fewer parallel boards, not a bigger swapfile. - ---- - -## Phase 2 — User, packages, layout - -Run the runner as a dedicated unprivileged user. It needs `dialout` for serial access to the -boards. - -```bash -sudo adduser --disabled-password --gecos "" hilrunner -sudo usermod -aG dialout hilrunner -``` - -Deliberately **no sudo rights** for `hilrunner`, and no docker group. - -```bash -sudo apt install -y git curl jq python3-venv python3-full usbutils -``` - -Directory layout — runner and harness support files kept separate: - -```bash -sudo mkdir -p /opt/vector-hil/{hooks,logs} -sudo mkdir -p /etc/vector-hil -sudo mkdir -p /opt/actions-runner -sudo chown -R hilrunner:hilrunner /opt/vector-hil /opt/actions-runner -sudo chown root:hilrunner /etc/vector-hil -sudo chmod 750 /etc/vector-hil -``` - ---- - -## Phase 3 — Python environment - -The workflow runs from a trusted ref and shouldn't be building a venv on every job — that's -minutes of wall clock on this hardware. Create it once, pinned: - -```bash -sudo -u hilrunner python3 -m venv /opt/vector-hil/venv -sudo -u hilrunner /opt/vector-hil/venv/bin/pip install --upgrade pip -sudo -u hilrunner /opt/vector-hil/venv/bin/pip install \ - mpremote==1.23.0 \ - pytest==9.1.1 \ - pytest-timeout==2.5.0 \ - pytest-xdist==3.8.0 \ - PyYAML==6.0.3 \ - requests==2.34.2 -``` - -`mpremote` is pinned to 1.23.0 to match `dev/requirements.txt`. `pytest-xdist` is what lets -the suite fan out across boards — the design's parallelism assumes it. - -`mpy-cross` is deliberately **not** installed here. Builds happen on GitHub-hosted runners; -the Pi only consumes artifacts. If you find yourself wanting to build on the Pi, stop and -re-read the threat model in DESIGN.md §3 — that's the boundary this whole design rests on. - -**Verify:** - -```bash -sudo -u hilrunner /opt/vector-hil/venv/bin/python -c "import pytest, yaml, requests; print('ok')" -sudo -u hilrunner /opt/vector-hil/venv/bin/mpremote --help >/dev/null && echo "mpremote ok" -``` - ---- - -## Phase 4 — Stable board device names - -`/dev/ttyACM0` ordering is not stable across reboots or re-enumeration. If the harness -addresses boards that way, a reboot silently swaps which board runs which test suite and you -get results attributed to the wrong hardware. Pin each board to a symlink by its USB serial. - -With all boards plugged in, list them: - -```bash -/opt/vector-hil/venv/bin/mpremote devs -``` - -For each port, get the identifying attributes: - -```bash -for d in /dev/ttyACM*; do - echo "=== $d" - udevadm info -a -n "$d" 2>/dev/null | grep -m3 -E 'ATTRS\{(idVendor|idProduct|serial)\}' -done -``` - -Raspberry Pi's vendor ID is `2e8a`; a Pico running MicroPython typically enumerates as -product `0005`. **Use whatever the command above actually prints** rather than trusting those -values — the firmware build determines them. - -Identify which physical board is which by asking each one: - -```bash -for d in /dev/ttyACM*; do - echo -n "$d -> " - /opt/vector-hil/venv/bin/mpremote connect "$d" exec \ - "import systemConfig; print(systemConfig.vectorSystem, systemConfig.SystemVersion)" 2>/dev/null \ - || echo "(no response)" -done -``` - -That's the same probe `dev/detect_boards.py` uses. Now write the rules, substituting the real -serial numbers: - -```bash -sudo tee /etc/udev/rules.d/99-vector-hil.rules >/dev/null <<'EOF' -# Vector HIL bench — stable names by USB serial. -# Get serials with: udevadm info -a -n /dev/ttyACM0 | grep ATTRS{serial} -SUBSYSTEM=="tty", ATTRS{idVendor}=="2e8a", ATTRS{serial}=="REPLACE_SYS11_SERIAL", SYMLINK+="vector-sys11-a", GROUP="dialout", MODE="0660" -SUBSYSTEM=="tty", ATTRS{idVendor}=="2e8a", ATTRS{serial}=="REPLACE_WPC_SERIAL", SYMLINK+="vector-wpc-a", GROUP="dialout", MODE="0660" -SUBSYSTEM=="tty", ATTRS{idVendor}=="2e8a", ATTRS{serial}=="REPLACE_DATA_EAST_SERIAL", SYMLINK+="vector-data-east-a", GROUP="dialout", MODE="0660" -EOF - -sudo udevadm control --reload-rules -sudo udevadm trigger --subsystem-match=tty -``` - -**Verify:** - -```bash -ls -l /dev/vector-* -``` - -You should see three symlinks. Unplug and replug the hub, confirm they come back pointing at -the right boards, and confirm `hilrunner` can open them: - -```bash -sudo -u hilrunner /opt/vector-hil/venv/bin/mpremote connect /dev/vector-wpc-a \ - exec "import systemConfig; print(systemConfig.vectorSystem)" -``` - -If that fails with a permissions error, `hilrunner`'s `dialout` membership hasn't taken -effect — log out and back in, or reboot. - ---- +sudo apt install -y git python3-venv curl -## Phase 5 — Network - -DESIGN.md §5 puts the boards on a VLAN isolated from the LAN. The Zero 2 W has a single -radio and no ethernet, which constrains how that gets built: **the Pi cannot be on the LAN -and the bench VLAN at the same time.** - -The workable arrangement with one radio: - -- The bench VLAN has its own SSID. The Pi joins **only** that SSID. -- On the router: allow the Pi's MAC out to the internet (it needs github.com); deny the board - MACs any WAN access; deny the whole VLAN any access to the LAN. -- Boards reach the Pi's update server directly — same subnet, no routing needed. -- Give each board a static DHCP lease so the manifest IPs stay put. - -If your router can't do per-MAC egress rules on a VLAN, the fallback is a USB ethernet -adapter on the hub for the LAN uplink, leaving `wlan0` for the bench. Note that this is -router configuration, not Pi configuration — the Pi side is just joining the right SSID. - -**Verify from the Pi:** - -```bash -curl -sS -o /dev/null -w "github: %{http_code}\n" https://api.github.com -ping -c2 10.42.7.11 # a board's static lease +# serial access to the Picos; log out and back in for this to take effect +sudo usermod -aG dialout $USER ``` -**Verify isolation actually holds** — this is the part people skip and regret. From a board's -REPL, confirm it cannot reach the LAN: - ```bash -/opt/vector-hil/venv/bin/mpremote connect /dev/vector-wpc-a exec " -import socket -try: - s = socket.socket(); s.settimeout(3) - s.connect(socket.getaddrinfo('192.168.1.1', 80)[0][-1]) # your LAN gateway - print('REACHABLE - isolation is NOT working') -except Exception as e: - print('blocked (good):', e) -" -``` - -If that prints `REACHABLE`, stop and fix the router before attaching the runner. The entire -security argument for running untrusted firmware on this bench depends on that path being -closed. - -### Optional: DNS override for `/api/update/check` +# --- repo + dev pipeline --------------------------------------------------- +git clone https://github.com/warped-pinball/vector.git ~/vector +python3 -m venv ~/vector/.venv +~/vector/.venv/bin/pip install -r ~/vector/dev/requirements.txt -`/api/update/check` fetches `software.warpedpinball.com`. Rather than opening egress for it, -serve a canned response locally so the test is deterministic (DESIGN.md §5): - -```bash -sudo apt install -y dnsmasq -echo "address=/software.warpedpinball.com/10.42.7.1" | \ - sudo tee /etc/dnsmasq.d/vector-hil.conf -sudo systemctl restart dnsmasq +# confirm the boards enumerate and identify themselves +cd ~/vector && .venv/bin/python dev/detect_boards.py ``` -Only do this if the Pi is the VLAN's DNS server. If the router handles DHCP/DNS there, put -the override on the router instead — running a second DHCP server on the segment will cause -problems that are annoying to diagnose. - ---- - -## Phase 6 — Bench manifest and secrets - -The manifest describes the bench; it lives on the Pi, not in git (DESIGN.md §6). +That last command should print something like +`{"sys11": ["/dev/ttyACM0"], "wpc": ["/dev/ttyACM1"], "data_east": ["/dev/ttyACM2"]}`. +If it does, the hardware side is done. ```bash -sudo tee /etc/vector-hil/bench.yaml >/dev/null <<'EOF' -boards: - - id: sys11-a - target: sys11 - hardware: sys11 - serial: /dev/vector-sys11-a - ip: 10.42.7.11 - - id: wpc-a - target: wpc - hardware: wpc - serial: /dev/vector-wpc-a - ip: 10.42.7.12 - - id: data-east-a - target: data_east - hardware: data_east - serial: /dev/vector-data-east-a - ip: 10.42.7.13 - -network: - update_server: http://10.42.7.1:8080 - -secrets_file: /etc/vector-hil/secrets.yaml -EOF - -sudo tee /etc/vector-hil/secrets.yaml >/dev/null <<'EOF' -wifi_ssid: "BENCH-SSID" -wifi_password: "..." -game_password: "..." -EOF - -sudo chown root:hilrunner /etc/vector-hil/bench.yaml /etc/vector-hil/secrets.yaml -sudo chmod 640 /etc/vector-hil/bench.yaml /etc/vector-hil/secrets.yaml -``` - -These are bench credentials for an isolated segment, not production secrets — but keep them -off GitHub regardless. Group-readable by `hilrunner` and nothing wider. +# --- actions runner -------------------------------------------------------- +mkdir -p ~/actions-runner && cd ~/actions-runner -Adding a board later is a udev rule plus an entry here. No test changes. +# pick the build matching the *userland* arch (not `uname -m`, which can differ) +ARCH=$([ "$(dpkg --print-architecture)" = arm64 ] && echo arm64 || echo arm) +curl -fSLo runner.tar.gz \ + "https://github.com/actions/runner/releases/download/v2.336.0/actions-runner-linux-${ARCH}-2.336.0.tar.gz" ---- +# optional: verify the download +sha256sum runner.tar.gz +# arm -> 44a300f322a1b5bccfe0b146cf3ca74f27000eb8afed761d1ffd90be035969d4 +# arm64 -> 58b758e420b87093fbd4bfddd368074960053e2f1388f01848c82624b90f27d1 -## Phase 7 — Install the Actions runner +tar xzf runner.tar.gz && rm runner.tar.gz -Pick the build from Phase 0's `dpkg --print-architecture`. - -```bash -cd /opt/actions-runner -RUNNER_VERSION=2.336.0 - -# armhf userland: -RUNNER_ARCH=arm RUNNER_SHA=44a300f322a1b5bccfe0b146cf3ca74f27000eb8afed761d1ffd90be035969d4 -# arm64 userland — use these two lines instead: -# RUNNER_ARCH=arm64 RUNNER_SHA=58b758e420b87093fbd4bfddd368074960053e2f1388f01848c82624b90f27d1 - -sudo -u hilrunner curl -fSL -o runner.tar.gz \ - "https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/actions-runner-linux-${RUNNER_ARCH}-${RUNNER_VERSION}.tar.gz" - -echo "${RUNNER_SHA} runner.tar.gz" | sha256sum -c - || { echo "CHECKSUM MISMATCH"; rm -f runner.tar.gz; } -sudo -u hilrunner tar xzf runner.tar.gz && sudo -u hilrunner rm runner.tar.gz -``` - -Those checksums were computed from the published v2.336.0 artifacts. Verify before extracting -— on a box that will be handling artifacts from public PRs, "I'll check it later" is how you -end up not checking it. - -The runner is a .NET application and needs a few system libraries: - -```bash +# .NET runtime libs the runner needs sudo ./bin/installdependencies.sh -``` -Get a registration token from -**Settings → Actions → Runners → New self-hosted runner** on `warped-pinball/vector`. It is -valid for one hour and is single-use. - -```bash -sudo -u hilrunner ./config.sh \ +# token from: repo Settings -> Actions -> Runners -> New self-hosted runner +# (valid one hour, single use) +./config.sh \ --url https://github.com/warped-pinball/vector \ --token \ - --name vector-hil-pi \ --labels vector-hil \ - --work _work \ - --unattended --replace -``` - -The `vector-hil` label is what `hil.yml` will target via -`runs-on: [self-hosted, vector-hil]`. Don't rely on the bare `self-hosted` label — the moment -there's a second runner anywhere in the org, jobs start landing on the wrong machine. - -### A note on ephemeral runners - -DESIGN.md §9 calls for `--ephemeral`. Deliberately not doing that in this first setup, for a -specific reason: an ephemeral runner deregisters after every job, so something has to mint a -fresh registration token for each one. That means storing a PAT with `administration: write` -on the Pi — readable by the same user that runs job code. That PAT is a considerably more -valuable secret than the runner's own credentials, which only let you receive jobs. - -The trade is acceptable here because of the trust model: under `workflow_run`, the Pi only -ever executes harness code from the default branch (DESIGN.md §4). It never runs PR-authored -host code, so the "poison the workspace for the next job" attack that ephemeral runners -defend against doesn't have a foothold. The job hooks in Phase 8 cover the rest. - -Revisit this if the Pi ever starts executing PR-authored code — at that point ephemeral -runners via JIT config stop being optional. - ---- - -## Phase 8 — Job hooks + --unattended -Two hooks: a pre-job health check that fails fast when the bench isn't sane, and a post-job -cleanup. - -```bash -sudo -u hilrunner tee /opt/vector-hil/hooks/pre-job.sh >/dev/null <<'EOF' -#!/usr/bin/env bash -# Fail the job immediately if the bench isn't healthy, rather than letting the -# tests fail in a way that looks like a firmware regression. -set -euo pipefail - -VENV=/opt/vector-hil/venv/bin -MANIFEST=${VECTOR_HIL_BENCH:-/etc/vector-hil/bench.yaml} - -echo "::group::Bench health check" -rc=0 -for dev in $("$VENV/python" -c " -import yaml -for b in yaml.safe_load(open('$MANIFEST'))['boards']: - print(b['serial']) -"); do - if [[ ! -e "$dev" ]]; then - echo "::error::$dev is missing - board not enumerated" - rc=1 - continue - fi - if out=$(timeout 20 "$VENV/mpremote" connect "$dev" exec \ - "import systemConfig; print(systemConfig.vectorSystem)" 2>&1); then - echo " $dev -> ${out//[$'\r\n']/}" - else - echo "::error::$dev did not respond to mpremote: $out" - rc=1 - fi -done - -avail=$(awk '/MemAvailable/ {print int($2/1024)}' /proc/meminfo) -echo " available memory: ${avail} MB" -if [[ $avail -lt 80 ]]; then - echo "::warning::low memory before job start (${avail} MB)" -fi - -echo "::endgroup::" -exit $rc -EOF - -sudo -u hilrunner tee /opt/vector-hil/hooks/post-job.sh >/dev/null <<'EOF' -#!/usr/bin/env bash -# Best-effort cleanup. Never fail the job from here - the tests already ran. -set -uo pipefail - -VENV=/opt/vector-hil/venv/bin -MANIFEST=${VECTOR_HIL_BENCH:-/etc/vector-hil/bench.yaml} - -for dev in $("$VENV/python" -c " -import yaml -for b in yaml.safe_load(open('$MANIFEST'))['boards']: - print(b['serial']) -" 2>/dev/null); do - [[ -e "$dev" ]] || continue - timeout 15 "$VENV/mpremote" connect "$dev" exec \ - "import machine; machine.reset()" >/dev/null 2>&1 || true -done - -find /opt/actions-runner/_work -mindepth 1 -maxdepth 1 \ - ! -name '_tool' ! -name '_temp' -exec rm -rf {} + 2>/dev/null || true - -exit 0 -EOF - -sudo -u hilrunner chmod +x /opt/vector-hil/hooks/*.sh -``` - -Register them in the runner's `.env`, which the service reads at start: - -```bash -sudo -u hilrunner tee -a /opt/actions-runner/.env >/dev/null <<'EOF' -ACTIONS_RUNNER_HOOK_JOB_STARTED=/opt/vector-hil/hooks/pre-job.sh -ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/opt/vector-hil/hooks/post-job.sh -VECTOR_HIL_BENCH=/etc/vector-hil/bench.yaml -VECTOR_HIL_VENV=/opt/vector-hil/venv -EOF -``` - -`.env` is only read when the service starts, so restart it after any change here. - -**Verify:** - -```bash -sudo -u hilrunner /opt/vector-hil/hooks/pre-job.sh && echo "PRE-JOB OK" -``` - ---- - -## Phase 9 — Run as a service - -```bash -cd /opt/actions-runner -sudo ./svc.sh install hilrunner +# run as a service so it survives reboots +sudo ./svc.sh install sudo ./svc.sh start sudo ./svc.sh status ``` -**Verify** the runner shows **Idle** under Settings → Actions → Runners, with the `vector-hil` -label attached. +The runner should now show **Idle** under Settings → Actions → Runners with the `vector-hil` +label. Workflows target it with `runs-on: [self-hosted, vector-hil]`. -```bash -journalctl -u "actions.runner.warped-pinball-vector.vector-hil-pi.service" -f -``` +## Smoke test -The runner auto-updates itself by default. Leave that on — GitHub stops dispatching jobs to -runners that fall too far behind. Just be aware an update pulls ~77 MB, so an occasional job -will start slowly. - ---- - -## Phase 10 — End-to-end check - -Before there's any harness code, confirm the whole path works with a throwaway workflow on a -branch: +Put this on a branch and dispatch it to confirm the full path works: ```yaml name: HIL smoke @@ -527,74 +78,23 @@ jobs: runs-on: [self-hosted, vector-hil] timeout-minutes: 10 steps: - - name: Report bench state - run: | - source "$VECTOR_HIL_VENV/bin/activate" - python - <<'PY' - import os, subprocess, yaml - bench = yaml.safe_load(open(os.environ["VECTOR_HIL_BENCH"])) - for b in bench["boards"]: - out = subprocess.run( - ["mpremote", "connect", b["serial"], "exec", - "import systemConfig; print(systemConfig.vectorSystem, systemConfig.SystemVersion)"], - capture_output=True, text=True, timeout=30) - print(f'{b["id"]:14} {out.stdout.strip() or out.stderr.strip()}') - PY + - run: ~/vector/.venv/bin/python ~/vector/dev/detect_boards.py ``` -Dispatch it. A green run that prints all three boards and their versions means the runner, -the venv, the udev names, the manifest, and the hooks are all wired correctly — which is -everything Phase 0–9 was for. - ---- - -## Maintenance and troubleshooting - -**SD card wear.** This bench writes a lot: artifact downloads, `_work` churn, swap. Use a -decent A2 card, keep the file swap modest, and treat the card as consumable. Take an image -once the setup is verified so a rebuild is a restore rather than a repeat of this document. - -**Log growth.** Runner diagnostic logs accumulate in `_diag`: - -```bash -sudo tee /etc/logrotate.d/vector-hil >/dev/null <<'EOF' -/opt/actions-runner/_diag/*.log { - weekly - rotate 4 - compress - missingok - notifempty -} -EOF -``` - -**A board stops responding.** Expected occasionally — recovery is software-only by decision -(DESIGN.md §10), so `machine.reset()` over `mpremote` is the first move. If MicroPython -itself won't come up, the board needs a physical BOOTSEL press and a reflash with -`trench-coat/uf2/nuke.uf2`. The pre-job hook will keep failing jobs loudly until that's done, -which is the intended behaviour — a silently absent board would attribute its tests to -nothing. - -**Jobs get OOM-killed.** Check `dmesg -T | grep -i oom`. Reduce `pytest-xdist` parallelism -before touching swap; three concurrent `mpremote` sessions plus the runner is close to the -ceiling on 512 MB. - -**Runner shows Offline after a reboot.** `sudo ./svc.sh status`; the service is enabled at -install but confirm with `systemctl is-enabled`. - -**Permission denied on `/dev/vector-*`.** `hilrunner` isn't in `dialout` yet, or the group -change hasn't been picked up by the running service. Restart the service. - -**Symlinks point at the wrong board after a hub replug.** A serial number in the udev rules -is wrong or duplicated. Re-run the Phase 4 identification loop. +## If something breaks ---- +Add these back only as needed: -## What this does not cover +- **Jobs get OOM-killed** (`dmesg -T | grep -i oom`) — 512 MB is tight on a Zero 2 W. + `sudo apt install -y zram-tools` gets you compressed swap with no further config. +- **`Permission denied` on `/dev/ttyACM*`** — the `dialout` group hasn't taken effect. + Reboot, then `sudo ./svc.sh stop && sudo ./svc.sh start`. +- **Board ordering shifts between runs and you start caring which is which** — `detect_boards.py` + identifies boards by querying `systemConfig.vectorSystem`, so it doesn't care about port + order. Only add udev rules if you need to pin a *specific physical board* rather than a + board type. +- **Runner won't start, globalization error** — `installdependencies.sh` didn't get libicu. + Re-run it, or set `DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=1` in `~/actions-runner/.env`. -- The `hil.yml` workflow and the `dev/hil` harness — not written yet; see DESIGN.md §4 and §7. -- The `hardware-lab` GitHub Environment and fork gating — repo settings, not Pi settings - (DESIGN.md §4). -- The update file server the OTA tests need on `10.42.7.1:8080` — the harness starts that - itself per-run (DESIGN.md §7, `update_server.py`). -- Router and VLAN configuration, which is where the isolation guarantee actually lives. +See [DESIGN.md](DESIGN.md) for the test architecture, the security model for fork PRs, and +the network isolation the bench needs before untrusted firmware runs on it. From c06c794cc82ea0674e2bee87d76e36fc5ddf47af Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 16 Aug 2026 17:02:20 +0000 Subject: [PATCH 05/22] docs: simplify runner setup for the actual bench Drop the download checksum step and the arch detection (arm64 confirmed in both kernel and userland). Remove the dialout group step from the main flow since the default login user already has it; keep it in the troubleshooting list. Boards share the VLAN with the Pi, so there is no network configuration in this doc. Bench wifi credentials go in the runner's .env, which is read at service start and exported to every job. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01WPbnmZNF6DFnLQBtpQpcW4 --- dev/hil/RUNNER_SETUP.md | 53 ++++++++++++++++++++++------------------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/dev/hil/RUNNER_SETUP.md b/dev/hil/RUNNER_SETUP.md index 28978051..ec097df5 100644 --- a/dev/hil/RUNNER_SETUP.md +++ b/dev/hil/RUNNER_SETUP.md @@ -1,7 +1,8 @@ # HIL Bench — Pi Runner Setup Minimal setup to make a Raspberry Pi with Vector boards attached run GitHub Actions jobs. -Assumes the Pi is assembled, boards are on a powered USB hub, and you have a shell on it. +Assumes the Pi is assembled, boards are on a powered USB hub, arm64 userland, and the boards +sit on the same VLAN as the Pi (so there is no routing or firewall setup to do here). Board detection and flashing use the repo's existing dev pipeline (`dev/detect_boards.py`, `dev/sync.py`), so there is nothing bench-specific to configure. @@ -10,9 +11,6 @@ Board detection and flashing use the repo's existing dev pipeline (`dev/detect_b # --- system packages ------------------------------------------------------- sudo apt update sudo apt install -y git python3-venv curl - -# serial access to the Picos; log out and back in for this to take effect -sudo usermod -aG dialout $USER ``` ```bash @@ -33,16 +31,8 @@ If it does, the hardware side is done. # --- actions runner -------------------------------------------------------- mkdir -p ~/actions-runner && cd ~/actions-runner -# pick the build matching the *userland* arch (not `uname -m`, which can differ) -ARCH=$([ "$(dpkg --print-architecture)" = arm64 ] && echo arm64 || echo arm) curl -fSLo runner.tar.gz \ - "https://github.com/actions/runner/releases/download/v2.336.0/actions-runner-linux-${ARCH}-2.336.0.tar.gz" - -# optional: verify the download -sha256sum runner.tar.gz -# arm -> 44a300f322a1b5bccfe0b146cf3ca74f27000eb8afed761d1ffd90be035969d4 -# arm64 -> 58b758e420b87093fbd4bfddd368074960053e2f1388f01848c82624b90f27d1 - + https://github.com/actions/runner/releases/download/v2.336.0/actions-runner-linux-arm64-2.336.0.tar.gz tar xzf runner.tar.gz && rm runner.tar.gz # .NET runtime libs the runner needs @@ -55,8 +45,21 @@ sudo ./bin/installdependencies.sh --token \ --labels vector-hil \ --unattended +``` + +```bash +# --- bench wifi credentials ------------------------------------------------ +# the runner reads .env at service start and passes these to every job +cat >> ~/actions-runner/.env <<'EOF' +VECTOR_HIL_WIFI_SSID=your-bench-ssid +VECTOR_HIL_WIFI_PASSWORD=your-bench-password +EOF +chmod 600 ~/actions-runner/.env +``` -# run as a service so it survives reboots +```bash +# --- run as a service ------------------------------------------------------ +cd ~/actions-runner sudo ./svc.sh install sudo ./svc.sh start sudo ./svc.sh status @@ -65,6 +68,9 @@ sudo ./svc.sh status The runner should now show **Idle** under Settings → Actions → Runners with the `vector-hil` label. Workflows target it with `runs-on: [self-hosted, vector-hil]`. +`.env` is only read when the service starts, so `sudo ./svc.sh stop && sudo ./svc.sh start` +after changing the credentials. + ## Smoke test Put this on a branch and dispatch it to confirm the full path works: @@ -79,6 +85,7 @@ jobs: timeout-minutes: 10 steps: - run: ~/vector/.venv/bin/python ~/vector/dev/detect_boards.py + - run: test -n "$VECTOR_HIL_WIFI_SSID" && echo "wifi env present" ``` ## If something breaks @@ -87,14 +94,12 @@ Add these back only as needed: - **Jobs get OOM-killed** (`dmesg -T | grep -i oom`) — 512 MB is tight on a Zero 2 W. `sudo apt install -y zram-tools` gets you compressed swap with no further config. -- **`Permission denied` on `/dev/ttyACM*`** — the `dialout` group hasn't taken effect. - Reboot, then `sudo ./svc.sh stop && sudo ./svc.sh start`. -- **Board ordering shifts between runs and you start caring which is which** — `detect_boards.py` - identifies boards by querying `systemConfig.vectorSystem`, so it doesn't care about port - order. Only add udev rules if you need to pin a *specific physical board* rather than a - board type. - **Runner won't start, globalization error** — `installdependencies.sh` didn't get libicu. - Re-run it, or set `DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=1` in `~/actions-runner/.env`. - -See [DESIGN.md](DESIGN.md) for the test architecture, the security model for fork PRs, and -the network isolation the bench needs before untrusted firmware runs on it. + Re-run it, or add `DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=1` to `~/actions-runner/.env`. +- **`Permission denied` on `/dev/ttyACM*`** — the login user isn't in `dialout`. + `sudo usermod -aG dialout $USER`, then reboot. +- **You need to pin a specific physical board** rather than a board type — `detect_boards.py` + identifies boards by querying `systemConfig.vectorSystem`, so port order doesn't matter. + Only add udev rules if you need to tell two boards of the same type apart. + +See [DESIGN.md](DESIGN.md) for the test architecture and the security model for fork PRs. From f6c433bcd21c67bd3d06f320e18365fcfbf7f4eb Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 16 Aug 2026 17:24:32 +0000 Subject: [PATCH 06/22] docs: package runner setup as a Raspberry Pi Connect artefact Converts the manual command list into a POSIX sh script artefact plus an otamaker manifest, so the bench Pi can be provisioned remotely instead of by hand. The artefact contract shapes the design: the script runs as root, so it drops to the runner user via runuser for everything that should not be root (the Actions runner refuses to configure as root); output goes to the journal, so each phase logs a line; and exit codes follow the 0/1/2 success/failure/reboot convention. Made re-runnable, since a redeploy is the natural way to refresh credentials or pull a newer harness: an existing clone is fetched rather than re-cloned, an existing runner download and registration are left alone, and .env is rewritten in place touching only the VECTOR_HIL_ lines it owns. Board detection is non-fatal so a transient USB state does not fail the whole deployment. Placeholder credentials are rejected up front. Verified against a stubbed root environment: fresh install, re-run idempotency, .env non-duplication and preservation of unmanaged vars, and failure propagation for apt, registration, and a service that starts but does not stay active. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01WPbnmZNF6DFnLQBtpQpcW4 --- dev/hil/RUNNER_SETUP.md | 155 +++++++++++++---------- dev/hil/ota/hil-runner-setup.sh | 202 ++++++++++++++++++++++++++++++ dev/hil/ota/hil-runner-setup.yaml | 8 ++ 3 files changed, 299 insertions(+), 66 deletions(-) create mode 100755 dev/hil/ota/hil-runner-setup.sh create mode 100644 dev/hil/ota/hil-runner-setup.yaml diff --git a/dev/hil/RUNNER_SETUP.md b/dev/hil/RUNNER_SETUP.md index ec097df5..53d8647f 100644 --- a/dev/hil/RUNNER_SETUP.md +++ b/dev/hil/RUNNER_SETUP.md @@ -1,79 +1,96 @@ # HIL Bench — Pi Runner Setup -Minimal setup to make a Raspberry Pi with Vector boards attached run GitHub Actions jobs. +Sets up a Raspberry Pi with Vector boards attached as a GitHub Actions runner, deployed as a +[Raspberry Pi Connect script artefact](https://www.raspberrypi.com/documentation/services/connect.html#remoteupdate-intro). + Assumes the Pi is assembled, boards are on a powered USB hub, arm64 userland, and the boards -sit on the same VLAN as the Pi (so there is no routing or firewall setup to do here). +sit on the same VLAN as the Pi. Board detection and flashing use the repo's existing dev +pipeline (`dev/detect_boards.py`, `dev/sync.py`), so there is nothing bench-specific to +configure. -Board detection and flashing use the repo's existing dev pipeline (`dev/detect_boards.py`, -`dev/sync.py`), so there is nothing bench-specific to configure. +Files live in [`ota/`](ota/): -```bash -# --- system packages ------------------------------------------------------- -sudo apt update -sudo apt install -y git python3-venv curl -``` +| File | Purpose | +|---|---| +| `hil-runner-setup.sh` | the script artefact — runs as root on the Pi | +| `hil-runner-setup.yaml` | the otamaker manifest | -```bash -# --- repo + dev pipeline --------------------------------------------------- -git clone https://github.com/warped-pinball/vector.git ~/vector -python3 -m venv ~/vector/.venv -~/vector/.venv/bin/pip install -r ~/vector/dev/requirements.txt +## Deploy -# confirm the boards enumerate and identify themselves -cd ~/vector && .venv/bin/python dev/detect_boards.py -``` +**1. Get a registration token.** Repo → Settings → Actions → Runners → New self-hosted +runner. It is single-use and **valid for one hour**, so do this immediately before packaging. -That last command should print something like -`{"sys11": ["/dev/ttyACM0"], "wpc": ["/dev/ttyACM1"], "data_east": ["/dev/ttyACM2"]}`. -If it does, the hardware side is done. +**2. Fill in the config block** at the top of `ota/hil-runner-setup.sh`: -```bash -# --- actions runner -------------------------------------------------------- -mkdir -p ~/actions-runner && cd ~/actions-runner - -curl -fSLo runner.tar.gz \ - https://github.com/actions/runner/releases/download/v2.336.0/actions-runner-linux-arm64-2.336.0.tar.gz -tar xzf runner.tar.gz && rm runner.tar.gz - -# .NET runtime libs the runner needs -sudo ./bin/installdependencies.sh - -# token from: repo Settings -> Actions -> Runners -> New self-hosted runner -# (valid one hour, single use) -./config.sh \ - --url https://github.com/warped-pinball/vector \ - --token \ - --labels vector-hil \ - --unattended +```sh +RUNNER_USER="pi" +REGISTRATION_TOKEN="PASTE_REGISTRATION_TOKEN" +WIFI_SSID="PASTE_BENCH_SSID" +WIFI_PASSWORD="PASTE_BENCH_PASSWORD" ``` +The script refuses to run if the placeholders are still in place, so a half-filled artefact +fails immediately and visibly rather than registering a runner with a broken environment. + +**3. Build the artefact:** + ```bash -# --- bench wifi credentials ------------------------------------------------ -# the runner reads .env at service start and passes these to every job -cat >> ~/actions-runner/.env <<'EOF' -VECTOR_HIL_WIFI_SSID=your-bench-ssid -VECTOR_HIL_WIFI_PASSWORD=your-bench-password -EOF -chmod 600 ~/actions-runner/.env +cd dev/hil/ota +otamaker hil-runner-setup.yaml ``` +This produces a `.tar.zst` and prints its SHA-256. + +**4. Deploy** through Raspberry Pi Connect to the bench Pi. + +**5. Watch it run** — the script logs each phase to the device journal: + ```bash -# --- run as a service ------------------------------------------------------ -cd ~/actions-runner -sudo ./svc.sh install -sudo ./svc.sh start -sudo ./svc.sh status +journalctl -t rpi-ota-connector -f ``` -The runner should now show **Idle** under Settings → Actions → Runners with the `vector-hil` -label. Workflows target it with `runs-on: [self-hosted, vector-hil]`. +Expect roughly five minutes, most of it `pip install` building the dev pipeline. + +> **Do not commit a filled-in script.** The token is single-use and short-lived, but the WiFi +> password is not. Fill in a working copy, package it, and discard it. -`.env` is only read when the service starts, so `sudo ./svc.sh stop && sudo ./svc.sh start` -after changing the credentials. +## What the script does -## Smoke test +Runs as root, per the Connect artefact contract, and drops to `$RUNNER_USER` via `runuser` +for everything that shouldn't be root — the Actions runner refuses to be configured as root, +and shouldn't run as root regardless. -Put this on a branch and dispatch it to confirm the full path works: +1. Installs `git`, `python3-venv`, `curl`; adds the runner user to `dialout` for serial access +2. Clones (or fetches) the repo and builds a venv from `dev/requirements.txt` +3. Runs `dev/detect_boards.py` and logs what it found +4. Downloads the Actions runner, registers it with the `vector-hil` label +5. Writes bench credentials to the runner's `.env` +6. Installs and starts the systemd service, then verifies the unit is actually active + +Exit codes follow the Connect contract: `0` success, `1` failure, `2` success-plus-reboot. +The script never returns `2` — nothing it does needs a reboot, because systemd reads the new +`dialout` membership when it starts the service. + +**Board detection is non-fatal.** A board that is unplugged or mid-reset logs a warning and +the deployment continues, rather than failing the whole setup over a transient USB state. +Bench health is the pre-job check's job, not the installer's. + +**It is safe to re-run.** Existing clone gets fetched instead of re-cloned, existing runner +download and registration are left alone, and `.env` is rewritten in place — only the +`VECTOR_HIL_*` lines it owns, so anything else you set there survives. Re-running it is the +normal way to refresh WiFi credentials or pull a newer harness. + +To re-register against a different repo or token, remove the registration first — the script +deliberately won't do this behind your back: + +```bash +cd ~/actions-runner && sudo ./svc.sh uninstall && ./config.sh remove --token +``` + +## Verify + +The runner should show **Idle** under Settings → Actions → Runners with the `vector-hil` +label. Confirm end to end with a workflow on a branch: ```yaml name: HIL smoke @@ -84,22 +101,28 @@ jobs: runs-on: [self-hosted, vector-hil] timeout-minutes: 10 steps: - - run: ~/vector/.venv/bin/python ~/vector/dev/detect_boards.py + - run: $VECTOR_HIL_VENV/bin/python $VECTOR_HIL_REPO/dev/detect_boards.py - run: test -n "$VECTOR_HIL_WIFI_SSID" && echo "wifi env present" ``` -## If something breaks +`VECTOR_HIL_VENV` and `VECTOR_HIL_REPO` are exported into every job from `.env`, so workflows +don't hardcode paths. -Add these back only as needed: +## If something breaks +- **Where did it fail?** `journalctl -t rpi-ota-connector` — every phase is logged, and each + failure exits with a specific message. +- **`runner registration failed`** — almost always an expired token. They last one hour. + Get a fresh one, refill, rebuild, redeploy. - **Jobs get OOM-killed** (`dmesg -T | grep -i oom`) — 512 MB is tight on a Zero 2 W. `sudo apt install -y zram-tools` gets you compressed swap with no further config. -- **Runner won't start, globalization error** — `installdependencies.sh` didn't get libicu. - Re-run it, or add `DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=1` to `~/actions-runner/.env`. -- **`Permission denied` on `/dev/ttyACM*`** — the login user isn't in `dialout`. - `sudo usermod -aG dialout $USER`, then reboot. -- **You need to pin a specific physical board** rather than a board type — `detect_boards.py` - identifies boards by querying `systemConfig.vectorSystem`, so port order doesn't matter. - Only add udev rules if you need to tell two boards of the same type apart. +- **Runner won't start, globalization error** — libicu missing. Re-run + `~/actions-runner/bin/installdependencies.sh`, or add + `DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=1` to `~/actions-runner/.env`. +- **`.env` changes have no effect** — it is read at service start. + `cd ~/actions-runner && sudo ./svc.sh stop && sudo ./svc.sh start`. +- **You need to tell two boards of the same type apart** — `detect_boards.py` identifies + boards by querying `systemConfig.vectorSystem`, so port order doesn't matter for distinct + board types. Only add udev rules if you have two of the same type. See [DESIGN.md](DESIGN.md) for the test architecture and the security model for fork PRs. diff --git a/dev/hil/ota/hil-runner-setup.sh b/dev/hil/ota/hil-runner-setup.sh new file mode 100755 index 00000000..653e45f6 --- /dev/null +++ b/dev/hil/ota/hil-runner-setup.sh @@ -0,0 +1,202 @@ +#!/bin/sh +# +# Vector HIL bench — GitHub Actions runner setup. +# +# Deployed as a Raspberry Pi Connect script artefact. Runs as root on the +# target Pi; everything that should not be root is run via runuser. +# +# Exit codes are the Connect contract: +# 0 success +# 1 failure +# 2 success, device needs a reboot +# +# Output goes to the device journal: journalctl -t rpi-ota-connector +# +# Safe to re-run. If the runner is already registered it is left alone and +# only the environment file and service state are refreshed. + +set -eu + +# -------------------------------------------------------------------------- +# Configuration — fill these in before running otamaker. +# +# REGISTRATION_TOKEN comes from the repo's +# Settings -> Actions -> Runners -> New self-hosted runner +# and is single-use and valid for one hour, so build and deploy promptly. +# -------------------------------------------------------------------------- +RUNNER_USER="pi" +REPO_URL="https://github.com/warped-pinball/vector" +REGISTRATION_TOKEN="PASTE_REGISTRATION_TOKEN" +RUNNER_LABELS="vector-hil" +RUNNER_VERSION="2.336.0" +RUNNER_ARCH="arm64" +WIFI_SSID="PASTE_BENCH_SSID" +WIFI_PASSWORD="PASTE_BENCH_PASSWORD" +# -------------------------------------------------------------------------- + +EXIT_FAILURE=1 + +log() { echo "[hil-setup] $*"; } +fail() { echo "[hil-setup] ERROR: $*" >&2; exit "$EXIT_FAILURE"; } + +as_user() { runuser -u "$RUNNER_USER" -- "$@"; } + +# --- preflight ------------------------------------------------------------ + +[ "$(id -u)" -eq 0 ] || fail "must run as root" + +for placeholder in "$REGISTRATION_TOKEN" "$WIFI_SSID" "$WIFI_PASSWORD"; do + case "$placeholder" in + PASTE_*) fail "configuration placeholders not filled in before packaging" ;; + esac +done + +id "$RUNNER_USER" >/dev/null 2>&1 || fail "user '$RUNNER_USER' does not exist" + +USER_HOME=$(getent passwd "$RUNNER_USER" | cut -d: -f6) +[ -n "$USER_HOME" ] && [ -d "$USER_HOME" ] || fail "no home directory for '$RUNNER_USER'" + +REPO_DIR="$USER_HOME/vector" +RUNNER_DIR="$USER_HOME/actions-runner" +VENV_DIR="$REPO_DIR/.venv" + +log "installing for user '$RUNNER_USER' (home: $USER_HOME)" + +# --- system packages ------------------------------------------------------ + +log "installing system packages" +export DEBIAN_FRONTEND=noninteractive +apt-get update >/dev/null 2>&1 || fail "apt-get update failed" +apt-get install -y git python3-venv curl >/dev/null 2>&1 \ + || fail "apt-get install failed" + +# Serial access to the Picos. Group membership is read by systemd when the +# runner service starts, and we start it below, so no reboot is needed. +if ! id -nG "$RUNNER_USER" | grep -qw dialout; then + log "adding $RUNNER_USER to dialout" + usermod -aG dialout "$RUNNER_USER" +fi + +# --- repo and dev pipeline ------------------------------------------------ + +if [ -d "$REPO_DIR/.git" ]; then + log "updating existing clone at $REPO_DIR" + as_user git -C "$REPO_DIR" fetch --quiet origin \ + || log "WARNING: git fetch failed, continuing with the existing checkout" +else + log "cloning $REPO_URL" + as_user git clone --quiet "$REPO_URL" "$REPO_DIR" || fail "git clone failed" +fi + +if [ ! -x "$VENV_DIR/bin/python" ]; then + log "creating virtualenv" + as_user python3 -m venv "$VENV_DIR" || fail "venv creation failed" +fi + +log "installing dev pipeline requirements (this takes a few minutes)" +as_user "$VENV_DIR/bin/pip" install --quiet --upgrade pip \ + || fail "pip self-upgrade failed" +as_user "$VENV_DIR/bin/pip" install --quiet -r "$REPO_DIR/dev/requirements.txt" \ + || fail "pip install of dev/requirements.txt failed" + +# --- board check ---------------------------------------------------------- +# Non-fatal: a board that is unplugged or mid-reset should not fail the whole +# deployment. It is logged loudly and the pre-job bench check owns it later. + +log "detecting boards" +if boards=$(as_user "$VENV_DIR/bin/python" "$REPO_DIR/dev/detect_boards.py" 2>&1); then + log "detected: $boards" + case "$boards" in + '{}'|'') log "WARNING: no boards detected - check the USB hub and power" ;; + esac +else + log "WARNING: board detection failed: $boards" +fi + +# --- actions runner ------------------------------------------------------- + +TARBALL="actions-runner-linux-${RUNNER_ARCH}-${RUNNER_VERSION}.tar.gz" +RUNNER_TARBALL_URL="https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/${TARBALL}" + +as_user mkdir -p "$RUNNER_DIR" + +if [ ! -x "$RUNNER_DIR/config.sh" ]; then + log "downloading actions runner $RUNNER_VERSION ($RUNNER_ARCH)" + as_user curl -fSL --retry 3 -o "$RUNNER_DIR/$TARBALL" "$RUNNER_TARBALL_URL" \ + || fail "runner download failed" + as_user tar -xzf "$RUNNER_DIR/$TARBALL" -C "$RUNNER_DIR" \ + || fail "runner extraction failed" + as_user rm -f "$RUNNER_DIR/$TARBALL" + + log "installing runner dependencies" + "$RUNNER_DIR/bin/installdependencies.sh" >/dev/null 2>&1 \ + || fail "installdependencies.sh failed" +else + log "runner already present, skipping download" +fi + +# config.sh refuses to run as root and errors if already configured, so +# re-registration is a deliberate manual step rather than something this +# script does behind your back. +if [ -f "$RUNNER_DIR/.runner" ]; then + log "runner already registered, leaving registration untouched" +else + log "registering runner with $REPO_URL" + as_user sh -c "cd '$RUNNER_DIR' && ./config.sh \ + --url '$REPO_URL' \ + --token '$REGISTRATION_TOKEN' \ + --labels '$RUNNER_LABELS' \ + --unattended --replace" >/dev/null 2>&1 \ + || fail "runner registration failed - the token may have expired (they last one hour)" +fi + +# --- bench environment ---------------------------------------------------- +# The runner reads .env at service start and exports it into every job. +# Rewrite only the lines this script owns so anything else set there survives. + +ENV_FILE="$RUNNER_DIR/.env" +log "writing bench environment to $ENV_FILE" + +TMP_ENV="${ENV_FILE}.new" +: > "$TMP_ENV" +if [ -f "$ENV_FILE" ]; then + grep -v '^VECTOR_HIL_' "$ENV_FILE" >> "$TMP_ENV" || true +fi +{ + echo "VECTOR_HIL_WIFI_SSID=$WIFI_SSID" + echo "VECTOR_HIL_WIFI_PASSWORD=$WIFI_PASSWORD" + echo "VECTOR_HIL_VENV=$VENV_DIR" + echo "VECTOR_HIL_REPO=$REPO_DIR" +} >> "$TMP_ENV" +mv "$TMP_ENV" "$ENV_FILE" +chown "$RUNNER_USER": "$ENV_FILE" +chmod 600 "$ENV_FILE" + +# --- service -------------------------------------------------------------- + +cd "$RUNNER_DIR" + +if [ -f "$RUNNER_DIR/.service" ]; then + log "restarting existing runner service" + ./svc.sh stop >/dev/null 2>&1 || true +else + log "installing runner service" + ./svc.sh install "$RUNNER_USER" >/dev/null 2>&1 || fail "svc.sh install failed" +fi + +./svc.sh start >/dev/null 2>&1 || fail "svc.sh start failed" + +# Give systemd a moment, then confirm the unit actually came up rather than +# reporting success for a service that immediately died. +sleep 5 +SERVICE_NAME=$(cat "$RUNNER_DIR/.service" 2>/dev/null || echo "") +if [ -n "$SERVICE_NAME" ] && ! systemctl is-active --quiet "$SERVICE_NAME"; then + log "service status:" + systemctl status "$SERVICE_NAME" --no-pager --lines=20 || true + fail "runner service is not active after start" +fi + +log "runner service '$SERVICE_NAME' is active" +log "setup complete - the runner should show Idle with label '$RUNNER_LABELS'" + +exit 0 diff --git a/dev/hil/ota/hil-runner-setup.yaml b/dev/hil/ota/hil-runner-setup.yaml new file mode 100644 index 00000000..34753e09 --- /dev/null +++ b/dev/hil/ota/hil-runner-setup.yaml @@ -0,0 +1,8 @@ +artefact: + name: vector-hil-runner-setup + version: 1.0 + device_type: rpi + +payloads: + - name: hil-runner-setup.sh + type: script From d7e15a5403db11f6535c6d0a880aa507c8779e5e Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 16 Aug 2026 21:01:21 +0000 Subject: [PATCH 07/22] build: commit prebuilt Connect update artefact and checksum Adds the built .tar.zst plus its SHA-256 so Raspberry Pi Connect can be pointed at a URL in this repo rather than at a locally built file. Committing the artefact required moving credentials out of it first. Baking the registration token and wifi password into the script before packaging would have published the bench wifi password to a public repo. The script now sources /etc/vector-hil.conf on the device instead, which never enters the repo, and refuses to run if that file is group- or world-writable or not owned by root. That also removes the rebuild-per-deploy step: the artefact carries no per-deployment state, so redeploying is pointing Connect at the same URL again. The registration token is only needed for the first deploy, since the script skips registration once the runner is registered. build.sh produces the archive reproducibly - fixed mtimes and ownership, sorted entries - so an unchanged source rebuilds byte-identically and a changed checksum in a diff means the contents really changed. It also syntax-checks the script and refuses to package one containing literal credentials. Marks *.tar.zst as binary in .gitattributes; the repo sets `* text=auto`, and any normalization of the artefact would invalidate the checksum the device verifies against. Verified: reproducible rebuild, credential guard, and the config-file paths (missing, bad permissions, missing token when unregistered, and tokenless redeploy when already registered). Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01WPbnmZNF6DFnLQBtpQpcW4 --- .gitattributes | 7 + dev/hil/RUNNER_SETUP.md | 150 +++++++++++------- dev/hil/ota/build.sh | 58 +++++++ dev/hil/ota/hil-runner-setup.sh | 56 +++++-- dev/hil/ota/vector-hil-runner-setup.tar.zst | Bin 0 -> 3233 bytes .../vector-hil-runner-setup.tar.zst.sha256 | 1 + dev/hil/ota/vector-hil.conf.example | 30 ++++ 7 files changed, 232 insertions(+), 70 deletions(-) create mode 100755 dev/hil/ota/build.sh create mode 100644 dev/hil/ota/vector-hil-runner-setup.tar.zst create mode 100644 dev/hil/ota/vector-hil-runner-setup.tar.zst.sha256 create mode 100644 dev/hil/ota/vector-hil.conf.example diff --git a/.gitattributes b/.gitattributes index dfe07704..70369b76 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,9 @@ # Auto detect text files and perform LF normalization * text=auto + +# Prebuilt Raspberry Pi Connect update artefacts. Never normalize these - +# any rewriting invalidates the committed SHA-256 the device verifies against. +*.tar.zst binary + +# The artefact script runs as POSIX sh on the Pi; CRLF would break the shebang. +dev/hil/ota/*.sh text eol=lf diff --git a/dev/hil/RUNNER_SETUP.md b/dev/hil/RUNNER_SETUP.md index 53d8647f..f9b363d5 100644 --- a/dev/hil/RUNNER_SETUP.md +++ b/dev/hil/RUNNER_SETUP.md @@ -8,42 +8,59 @@ sit on the same VLAN as the Pi. Board detection and flashing use the repo's exis pipeline (`dev/detect_boards.py`, `dev/sync.py`), so there is nothing bench-specific to configure. -Files live in [`ota/`](ota/): +Everything lives in [`ota/`](ota/): | File | Purpose | |---|---| -| `hil-runner-setup.sh` | the script artefact — runs as root on the Pi | -| `hil-runner-setup.yaml` | the otamaker manifest | +| `vector-hil-runner-setup.tar.zst` | **the built artefact** — point Connect at this | +| `vector-hil-runner-setup.tar.zst.sha256` | its checksum, for the Connect UI | +| `hil-runner-setup.sh` | the script inside the artefact; runs as root on the Pi | +| `hil-runner-setup.yaml` | otamaker manifest | +| `vector-hil.conf.example` | template for the device-side config | +| `build.sh` | rebuilds the artefact and checksum reproducibly | -## Deploy +## Why the artefact is committed -**1. Get a registration token.** Repo → Settings → Actions → Runners → New self-hosted -runner. It is single-use and **valid for one hour**, so do this immediately before packaging. +The script carries **no credentials**. Secrets live in `/etc/vector-hil.conf` on the Pi, which +never enters the repo. That is what makes committing the artefact safe — and it also means +you never rebuild it to redeploy. Point Connect at the same URL every time. -**2. Fill in the config block** at the top of `ota/hil-runner-setup.sh`: +The alternative — baking the token and WiFi password into the script before packaging — would +have published the bench WiFi password to a public repo the moment the artefact was committed. -```sh -RUNNER_USER="pi" -REGISTRATION_TOKEN="PASTE_REGISTRATION_TOKEN" -WIFI_SSID="PASTE_BENCH_SSID" -WIFI_PASSWORD="PASTE_BENCH_PASSWORD" +## First deploy + +**1. Put the config on the Pi.** Once per device, over Raspberry Pi Connect's shell or SSH: + +```bash +sudo tee /etc/vector-hil.conf >/dev/null <<'EOF' +WIFI_SSID="your-bench-ssid" +WIFI_PASSWORD="your-bench-password" +REGISTRATION_TOKEN="paste-from-github" +EOF +sudo chmod 600 /etc/vector-hil.conf +sudo chown root:root /etc/vector-hil.conf ``` -The script refuses to run if the placeholders are still in place, so a half-filled artefact -fails immediately and visibly rather than registering a runner with a broken environment. +The registration token comes from repo → Settings → Actions → Runners → New self-hosted +runner. It is single-use and valid for one hour, so generate it just before deploying. -**3. Build the artefact:** +The script sources this file as root and refuses to run if it is group- or world-writable, or +not owned by root. -```bash -cd dev/hil/ota -otamaker hil-runner-setup.yaml +**2. Deploy** via Connect's remote update, pointing at the committed artefact: + +``` +https://raw.githubusercontent.com/warped-pinball/vector/main/dev/hil/ota/vector-hil-runner-setup.tar.zst ``` -This produces a `.tar.zst` and prints its SHA-256. +Paste the checksum from `vector-hil-runner-setup.tar.zst.sha256` when Connect asks. Connect +passes it to the device, which verifies the download before running anything. -**4. Deploy** through Raspberry Pi Connect to the bench Pi. +Any HTTP, HTTPS, FTP, SFTP or `file://` URL the Pi can reach works — the location doesn't need +to be reachable by Connect's servers, only by the Pi. -**5. Watch it run** — the script logs each phase to the device journal: +**3. Watch it run:** ```bash journalctl -t rpi-ota-connector -f @@ -51,34 +68,17 @@ journalctl -t rpi-ota-connector -f Expect roughly five minutes, most of it `pip install` building the dev pipeline. -> **Do not commit a filled-in script.** The token is single-use and short-lived, but the WiFi -> password is not. Fill in a working copy, package it, and discard it. +**4. Blank the token** in `/etc/vector-hil.conf` once the runner is registered. It is spent, +and later redeploys don't need one. -## What the script does +## Redeploying -Runs as root, per the Connect artefact contract, and drops to `$RUNNER_USER` via `runuser` -for everything that shouldn't be root — the Actions runner refuses to be configured as root, -and shouldn't run as root regardless. +No rebuild, no new token, no config changes — just point Connect at the same URL again. This +is the normal way to refresh WiFi credentials (edit the conf first) or pull a newer harness. -1. Installs `git`, `python3-venv`, `curl`; adds the runner user to `dialout` for serial access -2. Clones (or fetches) the repo and builds a venv from `dev/requirements.txt` -3. Runs `dev/detect_boards.py` and logs what it found -4. Downloads the Actions runner, registers it with the `vector-hil` label -5. Writes bench credentials to the runner's `.env` -6. Installs and starts the systemd service, then verifies the unit is actually active - -Exit codes follow the Connect contract: `0` success, `1` failure, `2` success-plus-reboot. -The script never returns `2` — nothing it does needs a reboot, because systemd reads the new -`dialout` membership when it starts the service. - -**Board detection is non-fatal.** A board that is unplugged or mid-reset logs a warning and -the deployment continues, rather than failing the whole setup over a transient USB state. -Bench health is the pre-job check's job, not the installer's. - -**It is safe to re-run.** Existing clone gets fetched instead of re-cloned, existing runner -download and registration are left alone, and `.env` is rewritten in place — only the -`VECTOR_HIL_*` lines it owns, so anything else you set there survives. Re-running it is the -normal way to refresh WiFi credentials or pull a newer harness. +The script is idempotent: an existing clone is fetched rather than re-cloned, an existing +runner download and registration are left alone, and `.env` is rewritten touching only the +`VECTOR_HIL_*` lines it owns, so anything else you set there survives. To re-register against a different repo or token, remove the registration first — the script deliberately won't do this behind your back: @@ -87,10 +87,51 @@ deliberately won't do this behind your back: cd ~/actions-runner && sudo ./svc.sh uninstall && ./config.sh remove --token ``` +## Rebuilding the artefact + +After changing `hil-runner-setup.sh` or the manifest: + +```bash +cd dev/hil/ota && ./build.sh +``` + +Commit both the `.tar.zst` and the regenerated `.sha256`. The build is reproducible — fixed +mtimes, fixed ownership, sorted entries — so an unchanged source rebuilds byte-identically and +a changed checksum in a diff means the contents actually changed. `build.sh` also refuses to +package a script with literal credentials in it, and syntax-checks the script first. + +> **Archive layout caveat.** The Connect docs say the artefact is a zstd-compressed tar +> "containing the manifest and the script" but don't document the internal layout, so this +> build puts both at the archive root. If the device rejects it, run `otamaker +> hil-runner-setup.yaml` once and compare `tar -tf` output against +> `zstd -dc vector-hil-runner-setup.tar.zst | tar -tv`, then adjust `build.sh` to match. + +## What the script does + +Runs as root, per the Connect artefact contract, and drops to `$RUNNER_USER` via `runuser` for +everything that shouldn't be root — the Actions runner refuses to be configured as root, and +shouldn't run as root regardless. + +1. Reads and validates `/etc/vector-hil.conf` +2. Installs `git`, `python3-venv`, `curl`; adds the runner user to `dialout` for serial access +3. Clones (or fetches) the repo and builds a venv from `dev/requirements.txt` +4. Runs `dev/detect_boards.py` and logs what it found +5. Downloads the Actions runner, registers it with the `vector-hil` label +6. Writes bench credentials to the runner's `.env` +7. Installs and starts the systemd service, then verifies the unit is actually active + +Exit codes follow the Connect contract: `0` success, `1` failure, `2` success-plus-reboot. The +script never returns `2` — nothing it does needs a reboot, because systemd reads the new +`dialout` membership when it starts the service. + +**Board detection is non-fatal.** A board that is unplugged or mid-reset logs a warning and the +deployment continues, rather than failing the whole setup over a transient USB state. Bench +health is the pre-job check's job, not the installer's. + ## Verify -The runner should show **Idle** under Settings → Actions → Runners with the `vector-hil` -label. Confirm end to end with a workflow on a branch: +The runner should show **Idle** under Settings → Actions → Runners with the `vector-hil` label. +Confirm end to end with a workflow on a branch: ```yaml name: HIL smoke @@ -110,10 +151,11 @@ don't hardcode paths. ## If something breaks -- **Where did it fail?** `journalctl -t rpi-ota-connector` — every phase is logged, and each +- **Where did it fail?** `journalctl -t rpi-ota-connector` — every phase logs a line, and each failure exits with a specific message. -- **`runner registration failed`** — almost always an expired token. They last one hour. - Get a fresh one, refill, rebuild, redeploy. +- **`/etc/vector-hil.conf not found`** — step 1 hasn't been done on this Pi. +- **`runner registration failed`** — almost always an expired token. They last one hour. Put a + fresh one in the conf and redeploy; no rebuild needed. - **Jobs get OOM-killed** (`dmesg -T | grep -i oom`) — 512 MB is tight on a Zero 2 W. `sudo apt install -y zram-tools` gets you compressed swap with no further config. - **Runner won't start, globalization error** — libicu missing. Re-run @@ -121,8 +163,8 @@ don't hardcode paths. `DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=1` to `~/actions-runner/.env`. - **`.env` changes have no effect** — it is read at service start. `cd ~/actions-runner && sudo ./svc.sh stop && sudo ./svc.sh start`. -- **You need to tell two boards of the same type apart** — `detect_boards.py` identifies - boards by querying `systemConfig.vectorSystem`, so port order doesn't matter for distinct - board types. Only add udev rules if you have two of the same type. +- **You need to tell two boards of the same type apart** — `detect_boards.py` identifies boards + by querying `systemConfig.vectorSystem`, so port order doesn't matter for distinct board + types. Only add udev rules if you have two of the same type. See [DESIGN.md](DESIGN.md) for the test architecture and the security model for fork PRs. diff --git a/dev/hil/ota/build.sh b/dev/hil/ota/build.sh new file mode 100755 index 00000000..c45a5ba4 --- /dev/null +++ b/dev/hil/ota/build.sh @@ -0,0 +1,58 @@ +#!/bin/sh +# +# Build the Raspberry Pi Connect update artefact and write its checksum. +# +# ./build.sh +# +# Produces, in this directory: +# vector-hil-runner-setup.tar.zst the artefact +# vector-hil-runner-setup.tar.zst.sha256 its SHA-256, for the Connect UI +# +# The archive is built reproducibly - fixed mtimes, fixed ownership, sorted +# entries - so rebuilding from an unchanged source produces a byte-identical +# file and the committed checksum stays meaningful in diffs. +# +# The artefact contains no credentials; those live in /etc/vector-hil.conf on +# the device. That is what makes it safe to commit and redeploy unchanged. + +set -eu + +cd "$(dirname "$0")" + +ARTEFACT="vector-hil-runner-setup.tar.zst" +MANIFEST="hil-runner-setup.yaml" +SCRIPT="hil-runner-setup.sh" + +for f in "$MANIFEST" "$SCRIPT"; do + [ -f "$f" ] || { echo "missing $f" >&2; exit 1; } +done + +# Refuse to package a script someone has pasted credentials into. +if grep -qE '^(WIFI_PASSWORD|REGISTRATION_TOKEN)="..*"' "$SCRIPT"; then + echo "ERROR: $SCRIPT appears to contain literal credentials." >&2 + echo "Secrets belong in /etc/vector-hil.conf on the device, not the artefact." >&2 + exit 1 +fi + +sh -n "$SCRIPT" || { echo "ERROR: $SCRIPT is not valid POSIX sh" >&2; exit 1; } + +tar --create \ + --file - \ + --sort=name \ + --owner=root:0 \ + --group=root:0 \ + --numeric-owner \ + --mtime='UTC 2020-01-01' \ + --mode='u=rwX,go=rX' \ + --format=gnu \ + "$MANIFEST" "$SCRIPT" \ + | zstd --quiet --force -19 -o "$ARTEFACT" + +sha256sum "$ARTEFACT" > "${ARTEFACT}.sha256" + +echo "built $ARTEFACT" +echo " size: $(wc -c < "$ARTEFACT") bytes" +echo " sha256: $(cut -d' ' -f1 < "${ARTEFACT}.sha256")" +echo +echo "contents:" +zstd --decompress --stdout "$ARTEFACT" | tar --list --verbose diff --git a/dev/hil/ota/hil-runner-setup.sh b/dev/hil/ota/hil-runner-setup.sh index 653e45f6..08b4daed 100755 --- a/dev/hil/ota/hil-runner-setup.sh +++ b/dev/hil/ota/hil-runner-setup.sh @@ -5,6 +5,13 @@ # Deployed as a Raspberry Pi Connect script artefact. Runs as root on the # target Pi; everything that should not be root is run via runuser. # +# This script contains NO credentials, so the built artefact is safe to commit +# and redeploy unchanged. Secrets are read from a config file on the device: +# +# /etc/vector-hil.conf (root-owned, chmod 600) +# +# See vector-hil.conf.example. Create it once per Pi before the first deploy. +# # Exit codes are the Connect contract: # 0 success # 1 failure @@ -17,22 +24,19 @@ set -eu -# -------------------------------------------------------------------------- -# Configuration — fill these in before running otamaker. -# -# REGISTRATION_TOKEN comes from the repo's -# Settings -> Actions -> Runners -> New self-hosted runner -# and is single-use and valid for one hour, so build and deploy promptly. -# -------------------------------------------------------------------------- +CONFIG_FILE="/etc/vector-hil.conf" + +# Defaults. Anything here can be overridden in the config file. RUNNER_USER="pi" REPO_URL="https://github.com/warped-pinball/vector" -REGISTRATION_TOKEN="PASTE_REGISTRATION_TOKEN" RUNNER_LABELS="vector-hil" RUNNER_VERSION="2.336.0" RUNNER_ARCH="arm64" -WIFI_SSID="PASTE_BENCH_SSID" -WIFI_PASSWORD="PASTE_BENCH_PASSWORD" -# -------------------------------------------------------------------------- + +# Supplied by the config file. +REGISTRATION_TOKEN="" +WIFI_SSID="" +WIFI_PASSWORD="" EXIT_FAILURE=1 @@ -45,11 +49,21 @@ as_user() { runuser -u "$RUNNER_USER" -- "$@"; } [ "$(id -u)" -eq 0 ] || fail "must run as root" -for placeholder in "$REGISTRATION_TOKEN" "$WIFI_SSID" "$WIFI_PASSWORD"; do - case "$placeholder" in - PASTE_*) fail "configuration placeholders not filled in before packaging" ;; - esac -done +[ -f "$CONFIG_FILE" ] || fail "$CONFIG_FILE not found - create it before deploying (see vector-hil.conf.example)" + +# We are about to source this as root, so refuse it if anyone but root can +# write to it. +config_perms=$(stat -c '%a %U' "$CONFIG_FILE") +case "$config_perms" in + *[2367]" "*|*[2367][0-7]" "*) fail "$CONFIG_FILE is group- or world-writable ($config_perms) - chmod 600 it" ;; +esac +case "$config_perms" in + *" root") : ;; + *) fail "$CONFIG_FILE must be owned by root (currently ${config_perms#* })" ;; +esac + +# shellcheck source=/dev/null +. "$CONFIG_FILE" id "$RUNNER_USER" >/dev/null 2>&1 || fail "user '$RUNNER_USER' does not exist" @@ -60,6 +74,16 @@ REPO_DIR="$USER_HOME/vector" RUNNER_DIR="$USER_HOME/actions-runner" VENV_DIR="$REPO_DIR/.venv" +[ -n "$WIFI_SSID" ] || fail "WIFI_SSID not set in $CONFIG_FILE" +[ -n "$WIFI_PASSWORD" ] || fail "WIFI_PASSWORD not set in $CONFIG_FILE" + +# The registration token is only needed the first time. Once the runner is +# registered it can be removed from the config file - it is single-use and +# expires an hour after you generate it anyway. +if [ ! -f "$RUNNER_DIR/.runner" ] && [ -z "$REGISTRATION_TOKEN" ]; then + fail "runner is not registered and REGISTRATION_TOKEN is not set in $CONFIG_FILE" +fi + log "installing for user '$RUNNER_USER' (home: $USER_HOME)" # --- system packages ------------------------------------------------------ diff --git a/dev/hil/ota/vector-hil-runner-setup.tar.zst b/dev/hil/ota/vector-hil-runner-setup.tar.zst new file mode 100644 index 0000000000000000000000000000000000000000..ee86626e8ab1aad989c0a69af73b063c797e4e73 GIT binary patch literal 3233 zcmV;S3|{jnwJ-eyXr*KT%6I@4G(dCO!!QiPFbuv!m+baWj86h{GN0ZIY6p44ybHS5pz^T^~ky3AG!)vi+Z5ioSf>?Wy?HuHkSTI@V2tjS!upsx!DY@O{(q zDY4EZ_Ij%q>-hY(-`DuPqp#>0eYrc*_j21R?E3B4!&7EcSKJ7AKKaz6U)Up2V=CX% z3bE8YuH*dVsim9xEA2RQ)m}!W_NnV9Z;1EIwQpMcYrU9eQeRDJrD(~~se9dI@gwzX z{5pE7PU{ujm{$4l)258^@}gm0Vp@aLO|22aXpk1D1S$=0s5ET?1Qr?+I1dgF2@MSo z0M%Q0A@1(3H0*GEb*7%JDrv7xeaWFbdE{IP-Jw?>H%VA;RQm{OtQK*9i0jW+iW`6{%e*;gY;1vaezpbC9UNlfjT})B#=n-L8>m< z)MJ$Q`l-?f+plZ%(-;1kA68kG2c!|QC|!gsL?eo-%I^9dSJf3=!+tUU_xL(~>KmK4nM0{#a&yJ2fR1>dWt~(7mtjUC)P|`REOeLV0h{*sot&dvkrFK41PT zPky0_<#@XG5|yb$=v;_GBGM;>%1qwc?=q)2*BX4s+M;m%uyQJ=TWPyqJZHuDVxv-A zTff-_?SM*Ab)pdPU)N9_pIfQn>!!`OzUT{AC^=rwCojotw?M1*DcdizPPw+C`@d*& zzi004tNe8Rsic*9DnuX;(THSG5`97wpo;!mE32_*4+TEkn?FwB?ikGZ9#d(x#2{8M z*8x4!n5?eULtUqyb*`zt{@ClAtJd^2_A9N5dM%@Rjn(HT~I(Z4w&msbp0823ku_B-!st02&06t z*Vvg%LP3N=M?`s5b7s0ZdE9tVplfhW*RaN=4+K~SIyImAy%`!%RHro-eAp`ph_+m7 z0vw+))F{{fnG8q4lRfJy?cnUN4A58#|KLYd0#SrOs)n}i6U*rTWb|!&)ou^M*Dqr;6(j!)yX_Tx7@<(tvEzG>tCquKOkCX;dQ%49l1tD)0iu21vH z)Hk}xH+o%b>yLZBOy6db(?!1-lZ|h^*R7Q()VhS!BX#SQ>+RlGacspJsIC_}b>FXZ ztpHRnmN;(}IH8Kng1HJ);(oJg$$g3hW00>VD;HQs>dw*^|0d>wp>H;ERe!ALj#53MzA(B zxMrmaRpZt#O%>noZ}5oqykYI(sJr0Buz)!BQWK3|mh=+ktPIeW38JtUfGyhV3%%AI zpZ@+QgAm;x(V<=$%J$3+QXa%S^LVa$`ZGhfM(gEb5@P;L45~zSMMkx-lBN&tt_s&6 zJ-4P;pZkoaSTag)O3T&QxKV1pb8Qn2Bs#Qt!7m*)R6GjTANV14$d%B?Qgvb#?hz*t zX_Q-IGZmwZ%rbqEd(pmtu;Ki60d8pFxtH&xblXBNHa81Ds>ZRpZtJ=oF{aZx%g|3y zenfAizBhYpL>p?9FEX_^zC+ohv#Vat-uFdW2&kJ!r52#7-`RHwD=)KmvmM++&^T zieJkAMYnri1H)eqoA!E>jpz2n2vF8#KD39Qs8%9T-`6Gs0_j%Eea-L-0Z;3HMEJgj z1?wn8rWDYTzNy9ljCRw9vf^-uq``E9gax#f`v@RQhqChT*C`b5qdBM(jf9SLH*e)P z^K~;aPS7mqy#jGm{rXkRaE2^1YZb)s%~=L1j^xDhd!5i`QoA9c0F zDg_g5g5xtd@U@Sk7q0P;As zIQz7e&Hd$KaOugqpIr@MGKdZB;AZ+MM|745B#)c+5cfu&B zSBLFi2*?~vxQkGIfIeAOVV9kDREKVY(qXmFXFgp(tfxa|U^k*vgiF8C4=-?+U~;fHugONx28e?USK< zAvur&7|Mh9wxGjGAu~SyiA?L0N`Gx6%Ydx$D=XMJK^?nY0J9Dhvp9|^EMDg56dh9W zEQ8PVGznF|R#<)Rx$3@F`2+b?sVH?To63h!5 zR*rw-_-73f38dnT7~&u_>#^&V6xpa`Itk{c1Ay5~WBj)s-WJ{7o#;N@M>^uU1lzff zvId0*wE9qWqBI5gO!tI7&6MQJD%l}2QRx|z8>EEUgUpsi;*r~e5=tm23!Re2tf~}c z?E*`eD6@eDwaB58Lwrt?DTSiu_T==$5 z(>V!i8i#+irCNI<&Sfz!ct;$5n^mvsvN;nv2qCV@tTxDu5;QGMhxb@Ml|>E80@X?( z)oQfV1At8t*{LFi#Yr^Hj&JysH8oBoULdK^qw$T_juKGc-*< zo47Cm`k;_{OT%Z>UdmAbj$J~Db6PHhptN<6dKMc~dFqnopwM7~v651hgLc;EhT0B3 z%60ZBLr9KOb`#xYq%%TJMO7tlX#lYxRt80ly^lFg#?`pa?2=WG*$NPkLkE`zYBv$s zdZ#`$Lqz|j_eSYsWr|Ig`39tQ987wG6@WC8w8FyN`7?95V{A6#WQr<|;`)&Ch0Ovz zE*N7z{70-JGuks{7^*$M9I{7L;{|qHV@+o=f~1rIso$)G Settings -> Actions -> Runners -> New self-hosted runner +# +# Single-use and valid for one hour. Only needed for the FIRST deploy - once +# the runner is registered the setup script skips registration, and you can +# blank this out. Later redeploys need no token. +REGISTRATION_TOKEN="" + +# Optional overrides. Defaults shown; uncomment to change. +#RUNNER_USER="pi" +#REPO_URL="https://github.com/warped-pinball/vector" +#RUNNER_LABELS="vector-hil" +#RUNNER_VERSION="2.336.0" +#RUNNER_ARCH="arm64" From 02da1883e89b59f4df7744dfa8b75bfcc402f3b4 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 16 Aug 2026 21:05:07 +0000 Subject: [PATCH 08/22] docs: add pre-merge artefact URL for testing The documented raw.githubusercontent URL points at main, which does not resolve until this merges. Adds the branch URL so the artefact can be deployed and tested first, plus a note that GitHub does not render binary blobs in the Files changed view, so the committed artefact is easy to mistake for missing. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01WPbnmZNF6DFnLQBtpQpcW4 --- dev/hil/RUNNER_SETUP.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/dev/hil/RUNNER_SETUP.md b/dev/hil/RUNNER_SETUP.md index f9b363d5..e5fdb129 100644 --- a/dev/hil/RUNNER_SETUP.md +++ b/dev/hil/RUNNER_SETUP.md @@ -57,6 +57,18 @@ https://raw.githubusercontent.com/warped-pinball/vector/main/dev/hil/ota/vector- Paste the checksum from `vector-hil-runner-setup.tar.zst.sha256` when Connect asks. Connect passes it to the device, which verifies the download before running anything. +> **Testing before this merges?** The `main` URL above doesn't exist until then. Use the +> branch instead — `raw.githubusercontent.com` serves any ref: +> +> ``` +> https://raw.githubusercontent.com/warped-pinball/vector/claude/hil-testing-design-s564ln/dev/hil/ota/vector-hil-runner-setup.tar.zst +> ``` +> +> Switch to the `main` URL after merging; branch URLs stop resolving once the branch is +> deleted. Note that GitHub's "Files changed" view doesn't render binary blobs, so the +> artefact won't appear as a readable diff in the PR even though it is committed — confirm it +> with `git cat-file blob :dev/hil/ota/vector-hil-runner-setup.tar.zst | sha256sum`. + Any HTTP, HTTPS, FTP, SFTP or `file://` URL the Pi can reach works — the location doesn't need to be reachable by Connect's servers, only by the Pi. From 7567cdbcd023962733ed17e5461597ad4e14ae7f Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 17 Aug 2026 14:42:42 +0000 Subject: [PATCH 09/22] docs: replace OTA artefact with a plain setup script Raspberry Pi Connect's remote update requires A/B image support, which only Pi 4 and 5 have, so the Zero 2 W cannot be provisioned that way. Removes the artefact, its manifest, the reproducible build script, the committed tarball and checksum, and the device-side config file. Replaces them with dev/hil/setup-runner.sh, run directly on the Pi as the login user with sudo where needed. Credentials come from environment variables rather than a file, so nothing sensitive lands in the repo or on disk. Keeps the properties that were worth having: idempotent re-runs, non-fatal board detection, and a check that the service actually stayed up rather than reporting success for one that immediately died. Verified against stubs: missing credentials, missing token while unregistered, full install, and a tokenless re-run preserving unmanaged .env entries. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01WPbnmZNF6DFnLQBtpQpcW4 --- .gitattributes | 8 +- dev/hil/RUNNER_SETUP.md | 156 ++++-------- dev/hil/ota/build.sh | 58 ----- dev/hil/ota/hil-runner-setup.sh | 226 ------------------ dev/hil/ota/hil-runner-setup.yaml | 8 - dev/hil/ota/vector-hil-runner-setup.tar.zst | Bin 3233 -> 0 bytes .../vector-hil-runner-setup.tar.zst.sha256 | 1 - dev/hil/ota/vector-hil.conf.example | 30 --- dev/hil/setup-runner.sh | 173 ++++++++++++++ 9 files changed, 217 insertions(+), 443 deletions(-) delete mode 100755 dev/hil/ota/build.sh delete mode 100755 dev/hil/ota/hil-runner-setup.sh delete mode 100644 dev/hil/ota/hil-runner-setup.yaml delete mode 100644 dev/hil/ota/vector-hil-runner-setup.tar.zst delete mode 100644 dev/hil/ota/vector-hil-runner-setup.tar.zst.sha256 delete mode 100644 dev/hil/ota/vector-hil.conf.example create mode 100755 dev/hil/setup-runner.sh diff --git a/.gitattributes b/.gitattributes index 70369b76..82303ef6 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,9 +1,5 @@ # Auto detect text files and perform LF normalization * text=auto -# Prebuilt Raspberry Pi Connect update artefacts. Never normalize these - -# any rewriting invalidates the committed SHA-256 the device verifies against. -*.tar.zst binary - -# The artefact script runs as POSIX sh on the Pi; CRLF would break the shebang. -dev/hil/ota/*.sh text eol=lf +# Bench setup scripts run as POSIX sh on the Pi; CRLF would break the shebang. +dev/hil/*.sh text eol=lf diff --git a/dev/hil/RUNNER_SETUP.md b/dev/hil/RUNNER_SETUP.md index e5fdb129..3bd30288 100644 --- a/dev/hil/RUNNER_SETUP.md +++ b/dev/hil/RUNNER_SETUP.md @@ -1,96 +1,66 @@ # HIL Bench — Pi Runner Setup -Sets up a Raspberry Pi with Vector boards attached as a GitHub Actions runner, deployed as a -[Raspberry Pi Connect script artefact](https://www.raspberrypi.com/documentation/services/connect.html#remoteupdate-intro). +Sets up a Raspberry Pi with Vector boards attached as a GitHub Actions runner. Assumes the Pi is assembled, boards are on a powered USB hub, arm64 userland, and the boards sit on the same VLAN as the Pi. Board detection and flashing use the repo's existing dev pipeline (`dev/detect_boards.py`, `dev/sync.py`), so there is nothing bench-specific to configure. -Everything lives in [`ota/`](ota/): +> Raspberry Pi Connect's remote update only works on devices with A/B image support — Pi 4 and +> 5 — so the Zero 2 W can't be provisioned that way. This is a plain script you run on the Pi. -| File | Purpose | -|---|---| -| `vector-hil-runner-setup.tar.zst` | **the built artefact** — point Connect at this | -| `vector-hil-runner-setup.tar.zst.sha256` | its checksum, for the Connect UI | -| `hil-runner-setup.sh` | the script inside the artefact; runs as root on the Pi | -| `hil-runner-setup.yaml` | otamaker manifest | -| `vector-hil.conf.example` | template for the device-side config | -| `build.sh` | rebuilds the artefact and checksum reproducibly | +## Setup -## Why the artefact is committed +**1. Get a registration token** from repo → Settings → Actions → Runners → New self-hosted +runner. Single-use, valid one hour. -The script carries **no credentials**. Secrets live in `/etc/vector-hil.conf` on the Pi, which -never enters the repo. That is what makes committing the artefact safe — and it also means -you never rebuild it to redeploy. Point Connect at the same URL every time. - -The alternative — baking the token and WiFi password into the script before packaging — would -have published the bench WiFi password to a public repo the moment the artefact was committed. - -## First deploy - -**1. Put the config on the Pi.** Once per device, over Raspberry Pi Connect's shell or SSH: +**2. On the Pi**, as your normal login user (not root): ```bash -sudo tee /etc/vector-hil.conf >/dev/null <<'EOF' -WIFI_SSID="your-bench-ssid" -WIFI_PASSWORD="your-bench-password" -REGISTRATION_TOKEN="paste-from-github" -EOF -sudo chmod 600 /etc/vector-hil.conf -sudo chown root:root /etc/vector-hil.conf -``` - -The registration token comes from repo → Settings → Actions → Runners → New self-hosted -runner. It is single-use and valid for one hour, so generate it just before deploying. - -The script sources this file as root and refuses to run if it is group- or world-writable, or -not owned by root. +curl -fsSLO https://raw.githubusercontent.com/warped-pinball/vector/main/dev/hil/setup-runner.sh +chmod +x setup-runner.sh -**2. Deploy** via Connect's remote update, pointing at the committed artefact: +export VECTOR_HIL_WIFI_SSID="your-bench-ssid" +export VECTOR_HIL_WIFI_PASSWORD="your-bench-password" +export RUNNER_TOKEN="paste-from-github" -``` -https://raw.githubusercontent.com/warped-pinball/vector/main/dev/hil/ota/vector-hil-runner-setup.tar.zst +./setup-runner.sh ``` -Paste the checksum from `vector-hil-runner-setup.tar.zst.sha256` when Connect asks. Connect -passes it to the device, which verifies the download before running anything. +Credentials go in environment variables so they stay out of the script and out of the repo. +Note they will land in your shell history — `unset RUNNER_TOKEN VECTOR_HIL_WIFI_PASSWORD` +afterwards, or prefix the exports with a space if your shell is set to ignore those. -> **Testing before this merges?** The `main` URL above doesn't exist until then. Use the -> branch instead — `raw.githubusercontent.com` serves any ref: -> -> ``` -> https://raw.githubusercontent.com/warped-pinball/vector/claude/hil-testing-design-s564ln/dev/hil/ota/vector-hil-runner-setup.tar.zst -> ``` -> -> Switch to the `main` URL after merging; branch URLs stop resolving once the branch is -> deleted. Note that GitHub's "Files changed" view doesn't render binary blobs, so the -> artefact won't appear as a readable diff in the PR even though it is committed — confirm it -> with `git cat-file blob :dev/hil/ota/vector-hil-runner-setup.tar.zst | sha256sum`. +Takes about five minutes, most of it `pip install` building the dev pipeline. It prompts for +sudo once up front rather than midway through. -Any HTTP, HTTPS, FTP, SFTP or `file://` URL the Pi can reach works — the location doesn't need -to be reachable by Connect's servers, only by the Pi. +If you already have the repo cloned at `~/vector`, run `dev/hil/setup-runner.sh` from there +instead of curling it — the script uses `~/vector` either way. -**3. Watch it run:** +## What it does -```bash -journalctl -t rpi-ota-connector -f -``` +1. Installs `git`, `python3-venv`, `curl`; adds you to `dialout` for serial access +2. Clones (or fetches) the repo into `~/vector` and builds a venv from `dev/requirements.txt` +3. Runs `dev/detect_boards.py` and prints what it found +4. Downloads the Actions runner into `~/actions-runner`, registers it with the `vector-hil` label +5. Writes bench credentials to the runner's `.env` +6. Installs and starts the systemd service, then verifies the unit actually stayed up -Expect roughly five minutes, most of it `pip install` building the dev pipeline. +It runs as your user and calls `sudo` only where needed — the Actions runner refuses to be +configured as root, and shouldn't run as root regardless. -**4. Blank the token** in `/etc/vector-hil.conf` once the runner is registered. It is spent, -and later redeploys don't need one. +**Board detection is non-fatal.** A board unplugged or mid-reset prints a warning and setup +continues, rather than aborting over a transient USB state. -## Redeploying +**Safe to re-run.** An existing clone is fetched rather than re-cloned, an existing runner +download and registration are left alone, and `.env` is rewritten touching only the +`VECTOR_HIL_*` lines it owns. Re-running is the normal way to refresh WiFi credentials or pull +a newer harness — and after the first time you don't need `RUNNER_TOKEN`, since registration is +skipped once the runner exists. -No rebuild, no new token, no config changes — just point Connect at the same URL again. This -is the normal way to refresh WiFi credentials (edit the conf first) or pull a newer harness. - -The script is idempotent: an existing clone is fetched rather than re-cloned, an existing -runner download and registration are left alone, and `.env` is rewritten touching only the -`VECTOR_HIL_*` lines it owns, so anything else you set there survives. +Overridable via environment if you need them: `REPO_URL`, `RUNNER_LABELS`, `RUNNER_VERSION`, +`RUNNER_ARCH`. To re-register against a different repo or token, remove the registration first — the script deliberately won't do this behind your back: @@ -99,47 +69,6 @@ deliberately won't do this behind your back: cd ~/actions-runner && sudo ./svc.sh uninstall && ./config.sh remove --token ``` -## Rebuilding the artefact - -After changing `hil-runner-setup.sh` or the manifest: - -```bash -cd dev/hil/ota && ./build.sh -``` - -Commit both the `.tar.zst` and the regenerated `.sha256`. The build is reproducible — fixed -mtimes, fixed ownership, sorted entries — so an unchanged source rebuilds byte-identically and -a changed checksum in a diff means the contents actually changed. `build.sh` also refuses to -package a script with literal credentials in it, and syntax-checks the script first. - -> **Archive layout caveat.** The Connect docs say the artefact is a zstd-compressed tar -> "containing the manifest and the script" but don't document the internal layout, so this -> build puts both at the archive root. If the device rejects it, run `otamaker -> hil-runner-setup.yaml` once and compare `tar -tf` output against -> `zstd -dc vector-hil-runner-setup.tar.zst | tar -tv`, then adjust `build.sh` to match. - -## What the script does - -Runs as root, per the Connect artefact contract, and drops to `$RUNNER_USER` via `runuser` for -everything that shouldn't be root — the Actions runner refuses to be configured as root, and -shouldn't run as root regardless. - -1. Reads and validates `/etc/vector-hil.conf` -2. Installs `git`, `python3-venv`, `curl`; adds the runner user to `dialout` for serial access -3. Clones (or fetches) the repo and builds a venv from `dev/requirements.txt` -4. Runs `dev/detect_boards.py` and logs what it found -5. Downloads the Actions runner, registers it with the `vector-hil` label -6. Writes bench credentials to the runner's `.env` -7. Installs and starts the systemd service, then verifies the unit is actually active - -Exit codes follow the Connect contract: `0` success, `1` failure, `2` success-plus-reboot. The -script never returns `2` — nothing it does needs a reboot, because systemd reads the new -`dialout` membership when it starts the service. - -**Board detection is non-fatal.** A board that is unplugged or mid-reset logs a warning and the -deployment continues, rather than failing the whole setup over a transient USB state. Bench -health is the pre-job check's job, not the installer's. - ## Verify The runner should show **Idle** under Settings → Actions → Runners with the `vector-hil` label. @@ -163,11 +92,8 @@ don't hardcode paths. ## If something breaks -- **Where did it fail?** `journalctl -t rpi-ota-connector` — every phase logs a line, and each - failure exits with a specific message. -- **`/etc/vector-hil.conf not found`** — step 1 hasn't been done on this Pi. -- **`runner registration failed`** — almost always an expired token. They last one hour. Put a - fresh one in the conf and redeploy; no rebuild needed. +- **`registration failed`** — almost always an expired token. They last one hour. Get a fresh + one and re-run. - **Jobs get OOM-killed** (`dmesg -T | grep -i oom`) — 512 MB is tight on a Zero 2 W. `sudo apt install -y zram-tools` gets you compressed swap with no further config. - **Runner won't start, globalization error** — libicu missing. Re-run @@ -175,6 +101,8 @@ don't hardcode paths. `DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=1` to `~/actions-runner/.env`. - **`.env` changes have no effect** — it is read at service start. `cd ~/actions-runner && sudo ./svc.sh stop && sudo ./svc.sh start`. +- **`Permission denied` on `/dev/ttyACM*`** — the `dialout` group hasn't taken effect in the + running service. Reboot, or stop and start the service. - **You need to tell two boards of the same type apart** — `detect_boards.py` identifies boards by querying `systemConfig.vectorSystem`, so port order doesn't matter for distinct board types. Only add udev rules if you have two of the same type. diff --git a/dev/hil/ota/build.sh b/dev/hil/ota/build.sh deleted file mode 100755 index c45a5ba4..00000000 --- a/dev/hil/ota/build.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/sh -# -# Build the Raspberry Pi Connect update artefact and write its checksum. -# -# ./build.sh -# -# Produces, in this directory: -# vector-hil-runner-setup.tar.zst the artefact -# vector-hil-runner-setup.tar.zst.sha256 its SHA-256, for the Connect UI -# -# The archive is built reproducibly - fixed mtimes, fixed ownership, sorted -# entries - so rebuilding from an unchanged source produces a byte-identical -# file and the committed checksum stays meaningful in diffs. -# -# The artefact contains no credentials; those live in /etc/vector-hil.conf on -# the device. That is what makes it safe to commit and redeploy unchanged. - -set -eu - -cd "$(dirname "$0")" - -ARTEFACT="vector-hil-runner-setup.tar.zst" -MANIFEST="hil-runner-setup.yaml" -SCRIPT="hil-runner-setup.sh" - -for f in "$MANIFEST" "$SCRIPT"; do - [ -f "$f" ] || { echo "missing $f" >&2; exit 1; } -done - -# Refuse to package a script someone has pasted credentials into. -if grep -qE '^(WIFI_PASSWORD|REGISTRATION_TOKEN)="..*"' "$SCRIPT"; then - echo "ERROR: $SCRIPT appears to contain literal credentials." >&2 - echo "Secrets belong in /etc/vector-hil.conf on the device, not the artefact." >&2 - exit 1 -fi - -sh -n "$SCRIPT" || { echo "ERROR: $SCRIPT is not valid POSIX sh" >&2; exit 1; } - -tar --create \ - --file - \ - --sort=name \ - --owner=root:0 \ - --group=root:0 \ - --numeric-owner \ - --mtime='UTC 2020-01-01' \ - --mode='u=rwX,go=rX' \ - --format=gnu \ - "$MANIFEST" "$SCRIPT" \ - | zstd --quiet --force -19 -o "$ARTEFACT" - -sha256sum "$ARTEFACT" > "${ARTEFACT}.sha256" - -echo "built $ARTEFACT" -echo " size: $(wc -c < "$ARTEFACT") bytes" -echo " sha256: $(cut -d' ' -f1 < "${ARTEFACT}.sha256")" -echo -echo "contents:" -zstd --decompress --stdout "$ARTEFACT" | tar --list --verbose diff --git a/dev/hil/ota/hil-runner-setup.sh b/dev/hil/ota/hil-runner-setup.sh deleted file mode 100755 index 08b4daed..00000000 --- a/dev/hil/ota/hil-runner-setup.sh +++ /dev/null @@ -1,226 +0,0 @@ -#!/bin/sh -# -# Vector HIL bench — GitHub Actions runner setup. -# -# Deployed as a Raspberry Pi Connect script artefact. Runs as root on the -# target Pi; everything that should not be root is run via runuser. -# -# This script contains NO credentials, so the built artefact is safe to commit -# and redeploy unchanged. Secrets are read from a config file on the device: -# -# /etc/vector-hil.conf (root-owned, chmod 600) -# -# See vector-hil.conf.example. Create it once per Pi before the first deploy. -# -# Exit codes are the Connect contract: -# 0 success -# 1 failure -# 2 success, device needs a reboot -# -# Output goes to the device journal: journalctl -t rpi-ota-connector -# -# Safe to re-run. If the runner is already registered it is left alone and -# only the environment file and service state are refreshed. - -set -eu - -CONFIG_FILE="/etc/vector-hil.conf" - -# Defaults. Anything here can be overridden in the config file. -RUNNER_USER="pi" -REPO_URL="https://github.com/warped-pinball/vector" -RUNNER_LABELS="vector-hil" -RUNNER_VERSION="2.336.0" -RUNNER_ARCH="arm64" - -# Supplied by the config file. -REGISTRATION_TOKEN="" -WIFI_SSID="" -WIFI_PASSWORD="" - -EXIT_FAILURE=1 - -log() { echo "[hil-setup] $*"; } -fail() { echo "[hil-setup] ERROR: $*" >&2; exit "$EXIT_FAILURE"; } - -as_user() { runuser -u "$RUNNER_USER" -- "$@"; } - -# --- preflight ------------------------------------------------------------ - -[ "$(id -u)" -eq 0 ] || fail "must run as root" - -[ -f "$CONFIG_FILE" ] || fail "$CONFIG_FILE not found - create it before deploying (see vector-hil.conf.example)" - -# We are about to source this as root, so refuse it if anyone but root can -# write to it. -config_perms=$(stat -c '%a %U' "$CONFIG_FILE") -case "$config_perms" in - *[2367]" "*|*[2367][0-7]" "*) fail "$CONFIG_FILE is group- or world-writable ($config_perms) - chmod 600 it" ;; -esac -case "$config_perms" in - *" root") : ;; - *) fail "$CONFIG_FILE must be owned by root (currently ${config_perms#* })" ;; -esac - -# shellcheck source=/dev/null -. "$CONFIG_FILE" - -id "$RUNNER_USER" >/dev/null 2>&1 || fail "user '$RUNNER_USER' does not exist" - -USER_HOME=$(getent passwd "$RUNNER_USER" | cut -d: -f6) -[ -n "$USER_HOME" ] && [ -d "$USER_HOME" ] || fail "no home directory for '$RUNNER_USER'" - -REPO_DIR="$USER_HOME/vector" -RUNNER_DIR="$USER_HOME/actions-runner" -VENV_DIR="$REPO_DIR/.venv" - -[ -n "$WIFI_SSID" ] || fail "WIFI_SSID not set in $CONFIG_FILE" -[ -n "$WIFI_PASSWORD" ] || fail "WIFI_PASSWORD not set in $CONFIG_FILE" - -# The registration token is only needed the first time. Once the runner is -# registered it can be removed from the config file - it is single-use and -# expires an hour after you generate it anyway. -if [ ! -f "$RUNNER_DIR/.runner" ] && [ -z "$REGISTRATION_TOKEN" ]; then - fail "runner is not registered and REGISTRATION_TOKEN is not set in $CONFIG_FILE" -fi - -log "installing for user '$RUNNER_USER' (home: $USER_HOME)" - -# --- system packages ------------------------------------------------------ - -log "installing system packages" -export DEBIAN_FRONTEND=noninteractive -apt-get update >/dev/null 2>&1 || fail "apt-get update failed" -apt-get install -y git python3-venv curl >/dev/null 2>&1 \ - || fail "apt-get install failed" - -# Serial access to the Picos. Group membership is read by systemd when the -# runner service starts, and we start it below, so no reboot is needed. -if ! id -nG "$RUNNER_USER" | grep -qw dialout; then - log "adding $RUNNER_USER to dialout" - usermod -aG dialout "$RUNNER_USER" -fi - -# --- repo and dev pipeline ------------------------------------------------ - -if [ -d "$REPO_DIR/.git" ]; then - log "updating existing clone at $REPO_DIR" - as_user git -C "$REPO_DIR" fetch --quiet origin \ - || log "WARNING: git fetch failed, continuing with the existing checkout" -else - log "cloning $REPO_URL" - as_user git clone --quiet "$REPO_URL" "$REPO_DIR" || fail "git clone failed" -fi - -if [ ! -x "$VENV_DIR/bin/python" ]; then - log "creating virtualenv" - as_user python3 -m venv "$VENV_DIR" || fail "venv creation failed" -fi - -log "installing dev pipeline requirements (this takes a few minutes)" -as_user "$VENV_DIR/bin/pip" install --quiet --upgrade pip \ - || fail "pip self-upgrade failed" -as_user "$VENV_DIR/bin/pip" install --quiet -r "$REPO_DIR/dev/requirements.txt" \ - || fail "pip install of dev/requirements.txt failed" - -# --- board check ---------------------------------------------------------- -# Non-fatal: a board that is unplugged or mid-reset should not fail the whole -# deployment. It is logged loudly and the pre-job bench check owns it later. - -log "detecting boards" -if boards=$(as_user "$VENV_DIR/bin/python" "$REPO_DIR/dev/detect_boards.py" 2>&1); then - log "detected: $boards" - case "$boards" in - '{}'|'') log "WARNING: no boards detected - check the USB hub and power" ;; - esac -else - log "WARNING: board detection failed: $boards" -fi - -# --- actions runner ------------------------------------------------------- - -TARBALL="actions-runner-linux-${RUNNER_ARCH}-${RUNNER_VERSION}.tar.gz" -RUNNER_TARBALL_URL="https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/${TARBALL}" - -as_user mkdir -p "$RUNNER_DIR" - -if [ ! -x "$RUNNER_DIR/config.sh" ]; then - log "downloading actions runner $RUNNER_VERSION ($RUNNER_ARCH)" - as_user curl -fSL --retry 3 -o "$RUNNER_DIR/$TARBALL" "$RUNNER_TARBALL_URL" \ - || fail "runner download failed" - as_user tar -xzf "$RUNNER_DIR/$TARBALL" -C "$RUNNER_DIR" \ - || fail "runner extraction failed" - as_user rm -f "$RUNNER_DIR/$TARBALL" - - log "installing runner dependencies" - "$RUNNER_DIR/bin/installdependencies.sh" >/dev/null 2>&1 \ - || fail "installdependencies.sh failed" -else - log "runner already present, skipping download" -fi - -# config.sh refuses to run as root and errors if already configured, so -# re-registration is a deliberate manual step rather than something this -# script does behind your back. -if [ -f "$RUNNER_DIR/.runner" ]; then - log "runner already registered, leaving registration untouched" -else - log "registering runner with $REPO_URL" - as_user sh -c "cd '$RUNNER_DIR' && ./config.sh \ - --url '$REPO_URL' \ - --token '$REGISTRATION_TOKEN' \ - --labels '$RUNNER_LABELS' \ - --unattended --replace" >/dev/null 2>&1 \ - || fail "runner registration failed - the token may have expired (they last one hour)" -fi - -# --- bench environment ---------------------------------------------------- -# The runner reads .env at service start and exports it into every job. -# Rewrite only the lines this script owns so anything else set there survives. - -ENV_FILE="$RUNNER_DIR/.env" -log "writing bench environment to $ENV_FILE" - -TMP_ENV="${ENV_FILE}.new" -: > "$TMP_ENV" -if [ -f "$ENV_FILE" ]; then - grep -v '^VECTOR_HIL_' "$ENV_FILE" >> "$TMP_ENV" || true -fi -{ - echo "VECTOR_HIL_WIFI_SSID=$WIFI_SSID" - echo "VECTOR_HIL_WIFI_PASSWORD=$WIFI_PASSWORD" - echo "VECTOR_HIL_VENV=$VENV_DIR" - echo "VECTOR_HIL_REPO=$REPO_DIR" -} >> "$TMP_ENV" -mv "$TMP_ENV" "$ENV_FILE" -chown "$RUNNER_USER": "$ENV_FILE" -chmod 600 "$ENV_FILE" - -# --- service -------------------------------------------------------------- - -cd "$RUNNER_DIR" - -if [ -f "$RUNNER_DIR/.service" ]; then - log "restarting existing runner service" - ./svc.sh stop >/dev/null 2>&1 || true -else - log "installing runner service" - ./svc.sh install "$RUNNER_USER" >/dev/null 2>&1 || fail "svc.sh install failed" -fi - -./svc.sh start >/dev/null 2>&1 || fail "svc.sh start failed" - -# Give systemd a moment, then confirm the unit actually came up rather than -# reporting success for a service that immediately died. -sleep 5 -SERVICE_NAME=$(cat "$RUNNER_DIR/.service" 2>/dev/null || echo "") -if [ -n "$SERVICE_NAME" ] && ! systemctl is-active --quiet "$SERVICE_NAME"; then - log "service status:" - systemctl status "$SERVICE_NAME" --no-pager --lines=20 || true - fail "runner service is not active after start" -fi - -log "runner service '$SERVICE_NAME' is active" -log "setup complete - the runner should show Idle with label '$RUNNER_LABELS'" - -exit 0 diff --git a/dev/hil/ota/hil-runner-setup.yaml b/dev/hil/ota/hil-runner-setup.yaml deleted file mode 100644 index 34753e09..00000000 --- a/dev/hil/ota/hil-runner-setup.yaml +++ /dev/null @@ -1,8 +0,0 @@ -artefact: - name: vector-hil-runner-setup - version: 1.0 - device_type: rpi - -payloads: - - name: hil-runner-setup.sh - type: script diff --git a/dev/hil/ota/vector-hil-runner-setup.tar.zst b/dev/hil/ota/vector-hil-runner-setup.tar.zst deleted file mode 100644 index ee86626e8ab1aad989c0a69af73b063c797e4e73..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3233 zcmV;S3|{jnwJ-eyXr*KT%6I@4G(dCO!!QiPFbuv!m+baWj86h{GN0ZIY6p44ybHS5pz^T^~ky3AG!)vi+Z5ioSf>?Wy?HuHkSTI@V2tjS!upsx!DY@O{(q zDY4EZ_Ij%q>-hY(-`DuPqp#>0eYrc*_j21R?E3B4!&7EcSKJ7AKKaz6U)Up2V=CX% z3bE8YuH*dVsim9xEA2RQ)m}!W_NnV9Z;1EIwQpMcYrU9eQeRDJrD(~~se9dI@gwzX z{5pE7PU{ujm{$4l)258^@}gm0Vp@aLO|22aXpk1D1S$=0s5ET?1Qr?+I1dgF2@MSo z0M%Q0A@1(3H0*GEb*7%JDrv7xeaWFbdE{IP-Jw?>H%VA;RQm{OtQK*9i0jW+iW`6{%e*;gY;1vaezpbC9UNlfjT})B#=n-L8>m< z)MJ$Q`l-?f+plZ%(-;1kA68kG2c!|QC|!gsL?eo-%I^9dSJf3=!+tUU_xL(~>KmK4nM0{#a&yJ2fR1>dWt~(7mtjUC)P|`REOeLV0h{*sot&dvkrFK41PT zPky0_<#@XG5|yb$=v;_GBGM;>%1qwc?=q)2*BX4s+M;m%uyQJ=TWPyqJZHuDVxv-A zTff-_?SM*Ab)pdPU)N9_pIfQn>!!`OzUT{AC^=rwCojotw?M1*DcdizPPw+C`@d*& zzi004tNe8Rsic*9DnuX;(THSG5`97wpo;!mE32_*4+TEkn?FwB?ikGZ9#d(x#2{8M z*8x4!n5?eULtUqyb*`zt{@ClAtJd^2_A9N5dM%@Rjn(HT~I(Z4w&msbp0823ku_B-!st02&06t z*Vvg%LP3N=M?`s5b7s0ZdE9tVplfhW*RaN=4+K~SIyImAy%`!%RHro-eAp`ph_+m7 z0vw+))F{{fnG8q4lRfJy?cnUN4A58#|KLYd0#SrOs)n}i6U*rTWb|!&)ou^M*Dqr;6(j!)yX_Tx7@<(tvEzG>tCquKOkCX;dQ%49l1tD)0iu21vH z)Hk}xH+o%b>yLZBOy6db(?!1-lZ|h^*R7Q()VhS!BX#SQ>+RlGacspJsIC_}b>FXZ ztpHRnmN;(}IH8Kng1HJ);(oJg$$g3hW00>VD;HQs>dw*^|0d>wp>H;ERe!ALj#53MzA(B zxMrmaRpZt#O%>noZ}5oqykYI(sJr0Buz)!BQWK3|mh=+ktPIeW38JtUfGyhV3%%AI zpZ@+QgAm;x(V<=$%J$3+QXa%S^LVa$`ZGhfM(gEb5@P;L45~zSMMkx-lBN&tt_s&6 zJ-4P;pZkoaSTag)O3T&QxKV1pb8Qn2Bs#Qt!7m*)R6GjTANV14$d%B?Qgvb#?hz*t zX_Q-IGZmwZ%rbqEd(pmtu;Ki60d8pFxtH&xblXBNHa81Ds>ZRpZtJ=oF{aZx%g|3y zenfAizBhYpL>p?9FEX_^zC+ohv#Vat-uFdW2&kJ!r52#7-`RHwD=)KmvmM++&^T zieJkAMYnri1H)eqoA!E>jpz2n2vF8#KD39Qs8%9T-`6Gs0_j%Eea-L-0Z;3HMEJgj z1?wn8rWDYTzNy9ljCRw9vf^-uq``E9gax#f`v@RQhqChT*C`b5qdBM(jf9SLH*e)P z^K~;aPS7mqy#jGm{rXkRaE2^1YZb)s%~=L1j^xDhd!5i`QoA9c0F zDg_g5g5xtd@U@Sk7q0P;As zIQz7e&Hd$KaOugqpIr@MGKdZB;AZ+MM|745B#)c+5cfu&B zSBLFi2*?~vxQkGIfIeAOVV9kDREKVY(qXmFXFgp(tfxa|U^k*vgiF8C4=-?+U~;fHugONx28e?USK< zAvur&7|Mh9wxGjGAu~SyiA?L0N`Gx6%Ydx$D=XMJK^?nY0J9Dhvp9|^EMDg56dh9W zEQ8PVGznF|R#<)Rx$3@F`2+b?sVH?To63h!5 zR*rw-_-73f38dnT7~&u_>#^&V6xpa`Itk{c1Ay5~WBj)s-WJ{7o#;N@M>^uU1lzff zvId0*wE9qWqBI5gO!tI7&6MQJD%l}2QRx|z8>EEUgUpsi;*r~e5=tm23!Re2tf~}c z?E*`eD6@eDwaB58Lwrt?DTSiu_T==$5 z(>V!i8i#+irCNI<&Sfz!ct;$5n^mvsvN;nv2qCV@tTxDu5;QGMhxb@Ml|>E80@X?( z)oQfV1At8t*{LFi#Yr^Hj&JysH8oBoULdK^qw$T_juKGc-*< zo47Cm`k;_{OT%Z>UdmAbj$J~Db6PHhptN<6dKMc~dFqnopwM7~v651hgLc;EhT0B3 z%60ZBLr9KOb`#xYq%%TJMO7tlX#lYxRt80ly^lFg#?`pa?2=WG*$NPkLkE`zYBv$s zdZ#`$Lqz|j_eSYsWr|Ig`39tQ987wG6@WC8w8FyN`7?95V{A6#WQr<|;`)&Ch0Ovz zE*N7z{70-JGuks{7^*$M9I{7L;{|qHV@+o=f~1rIso$)G Settings -> Actions -> Runners -> New self-hosted runner -# -# Single-use and valid for one hour. Only needed for the FIRST deploy - once -# the runner is registered the setup script skips registration, and you can -# blank this out. Later redeploys need no token. -REGISTRATION_TOKEN="" - -# Optional overrides. Defaults shown; uncomment to change. -#RUNNER_USER="pi" -#REPO_URL="https://github.com/warped-pinball/vector" -#RUNNER_LABELS="vector-hil" -#RUNNER_VERSION="2.336.0" -#RUNNER_ARCH="arm64" diff --git a/dev/hil/setup-runner.sh b/dev/hil/setup-runner.sh new file mode 100755 index 00000000..0cff5f5a --- /dev/null +++ b/dev/hil/setup-runner.sh @@ -0,0 +1,173 @@ +#!/bin/sh +# +# Vector HIL bench — GitHub Actions runner setup. +# +# Run on the bench Pi as your normal login user (not root — the Actions +# runner refuses to be configured as root). Uses sudo for the parts that +# need it. +# +# export VECTOR_HIL_WIFI_SSID="bench-ssid" +# export VECTOR_HIL_WIFI_PASSWORD="bench-password" +# export RUNNER_TOKEN="from github, see below" +# ./setup-runner.sh +# +# RUNNER_TOKEN comes from the repo's +# Settings -> Actions -> Runners -> New self-hosted runner +# It is single-use and valid for one hour. It is only needed the first +# time; once the runner is registered you can re-run this without it. +# +# Safe to re-run: an existing clone is fetched rather than re-cloned, an +# existing runner download and registration are left alone, and only the +# VECTOR_HIL_* lines in the runner's .env are rewritten. + +set -eu + +REPO_URL="${REPO_URL:-https://github.com/warped-pinball/vector}" +RUNNER_LABELS="${RUNNER_LABELS:-vector-hil}" +RUNNER_VERSION="${RUNNER_VERSION:-2.336.0}" +RUNNER_ARCH="${RUNNER_ARCH:-arm64}" + +REPO_DIR="$HOME/vector" +RUNNER_DIR="$HOME/actions-runner" +VENV_DIR="$REPO_DIR/.venv" + +log() { echo "==> $*"; } +fail() { echo "ERROR: $*" >&2; exit 1; } + +# --- preflight ------------------------------------------------------------ + +[ "$(id -u)" -ne 0 ] || fail "run as your normal user, not root (the runner will not configure as root)" + +[ -n "${VECTOR_HIL_WIFI_SSID:-}" ] || fail "VECTOR_HIL_WIFI_SSID is not set" +[ -n "${VECTOR_HIL_WIFI_PASSWORD:-}" ] || fail "VECTOR_HIL_WIFI_PASSWORD is not set" + +if [ ! -f "$RUNNER_DIR/.runner" ] && [ -z "${RUNNER_TOKEN:-}" ]; then + fail "RUNNER_TOKEN is not set and the runner is not registered yet" +fi + +# Prompt for sudo once here rather than halfway through. +sudo -v || fail "sudo is required" + +# --- system packages ------------------------------------------------------ + +log "installing system packages" +sudo apt-get update -qq || fail "apt-get update failed" +sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq git python3-venv curl \ + || fail "apt-get install failed" + +# Serial access to the Picos. systemd reads group membership when it starts +# the runner service below, so no reboot is needed. +if ! id -nG | grep -qw dialout; then + log "adding $(whoami) to dialout" + sudo usermod -aG dialout "$(whoami)" +fi + +# --- repo and dev pipeline ------------------------------------------------ + +if [ -d "$REPO_DIR/.git" ]; then + log "updating existing clone at $REPO_DIR" + git -C "$REPO_DIR" fetch --quiet origin || echo "WARNING: git fetch failed, using existing checkout" +else + log "cloning $REPO_URL" + git clone --quiet "$REPO_URL" "$REPO_DIR" || fail "git clone failed" +fi + +if [ ! -x "$VENV_DIR/bin/python" ]; then + log "creating virtualenv" + python3 -m venv "$VENV_DIR" || fail "venv creation failed" +fi + +log "installing dev pipeline requirements (a few minutes)" +"$VENV_DIR/bin/pip" install --quiet --upgrade pip || fail "pip self-upgrade failed" +"$VENV_DIR/bin/pip" install --quiet -r "$REPO_DIR/dev/requirements.txt" \ + || fail "pip install of dev/requirements.txt failed" + +# --- board check ---------------------------------------------------------- +# Non-fatal: a board unplugged or mid-reset should not abort the setup. + +log "detecting boards" +if boards=$("$VENV_DIR/bin/python" "$REPO_DIR/dev/detect_boards.py" 2>&1); then + echo " $boards" + case "$boards" in + '{}'|'') echo " WARNING: no boards detected - check the USB hub and power" ;; + esac +else + echo " WARNING: board detection failed: $boards" +fi + +# --- actions runner ------------------------------------------------------- + +TARBALL="actions-runner-linux-${RUNNER_ARCH}-${RUNNER_VERSION}.tar.gz" + +mkdir -p "$RUNNER_DIR" + +if [ ! -x "$RUNNER_DIR/config.sh" ]; then + log "downloading actions runner $RUNNER_VERSION ($RUNNER_ARCH)" + curl -fSL --retry 3 -o "$RUNNER_DIR/$TARBALL" \ + "https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/${TARBALL}" \ + || fail "runner download failed" + tar -xzf "$RUNNER_DIR/$TARBALL" -C "$RUNNER_DIR" || fail "runner extraction failed" + rm -f "$RUNNER_DIR/$TARBALL" + + log "installing runner dependencies" + sudo "$RUNNER_DIR/bin/installdependencies.sh" >/dev/null \ + || fail "installdependencies.sh failed" +else + log "runner already downloaded, skipping" +fi + +if [ -f "$RUNNER_DIR/.runner" ]; then + log "runner already registered, leaving registration alone" +else + log "registering runner" + ( cd "$RUNNER_DIR" && ./config.sh \ + --url "$REPO_URL" \ + --token "$RUNNER_TOKEN" \ + --labels "$RUNNER_LABELS" \ + --unattended --replace >/dev/null ) \ + || fail "registration failed - the token may have expired (they last one hour)" +fi + +# --- bench environment ---------------------------------------------------- +# The runner reads .env at service start and exports it into every job. +# Only rewrite the lines we own so anything else set there survives. + +ENV_FILE="$RUNNER_DIR/.env" +log "writing bench environment" + +TMP_ENV="${ENV_FILE}.new" +: > "$TMP_ENV" +[ -f "$ENV_FILE" ] && { grep -v '^VECTOR_HIL_' "$ENV_FILE" >> "$TMP_ENV" || true; } +{ + echo "VECTOR_HIL_WIFI_SSID=$VECTOR_HIL_WIFI_SSID" + echo "VECTOR_HIL_WIFI_PASSWORD=$VECTOR_HIL_WIFI_PASSWORD" + echo "VECTOR_HIL_VENV=$VENV_DIR" + echo "VECTOR_HIL_REPO=$REPO_DIR" +} >> "$TMP_ENV" +mv "$TMP_ENV" "$ENV_FILE" +chmod 600 "$ENV_FILE" + +# --- service -------------------------------------------------------------- + +cd "$RUNNER_DIR" + +if [ -f "$RUNNER_DIR/.service" ]; then + log "restarting runner service" + sudo ./svc.sh stop >/dev/null 2>&1 || true +else + log "installing runner service" + sudo ./svc.sh install "$(whoami)" >/dev/null || fail "svc.sh install failed" +fi + +sudo ./svc.sh start >/dev/null || fail "svc.sh start failed" + +# Confirm the unit actually stayed up rather than reporting success for a +# service that immediately died. +sleep 5 +SERVICE_NAME=$(cat "$RUNNER_DIR/.service" 2>/dev/null || echo "") +if [ -n "$SERVICE_NAME" ] && ! systemctl is-active --quiet "$SERVICE_NAME"; then + systemctl status "$SERVICE_NAME" --no-pager --lines=20 || true + fail "runner service is not active after start" +fi + +log "done - runner '$SERVICE_NAME' is active with label '$RUNNER_LABELS'" From 962e25651aaa06b3f1ddae39b81a87f3c82aae0b Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 17 Aug 2026 19:06:39 +0000 Subject: [PATCH 10/22] ci: add HIL bench smoke workflow Nothing in the repo targeted the self-hosted runner yet, so there was no way to validate the bench. This checks that the runner picks up jobs, the bench environment reaches them, the serial devices are present and accessible, and every detected board answers over mpremote. It fails if no boards are found or if a detected board does not respond. Triggers on push to this branch as well as workflow_dispatch, because a workflow_dispatch workflow is not dispatchable until it exists on the default branch - without the push trigger the bench could not be validated before merging. Drop the push trigger once this is on main. No pull_request trigger: this targets a self-hosted runner on a private network, and pull_request would let any fork PR run against it. Fork gating is designed in DESIGN.md but not built. The job does no checkout, matching the trust model - the bench runs code from the clone it already has, not from a PR. Parses only the last line of detect_boards.py output, since it prints progress text before the JSON. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01WPbnmZNF6DFnLQBtpQpcW4 --- .github/workflows/hil-smoke.yml | 114 ++++++++++++++++++++++++++++++++ dev/hil/RUNNER_SETUP.md | 40 ++++++----- 2 files changed, 139 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/hil-smoke.yml diff --git a/.github/workflows/hil-smoke.yml b/.github/workflows/hil-smoke.yml new file mode 100644 index 00000000..57e4a3eb --- /dev/null +++ b/.github/workflows/hil-smoke.yml @@ -0,0 +1,114 @@ +name: HIL smoke + +# Proves the bench runner is wired up correctly: the runner picks up jobs, the +# bench environment reaches them, the boards enumerate, and each one answers +# over serial. It does not test firmware — see dev/hil/DESIGN.md for that. +# +# Deliberately no `pull_request` trigger. This targets a self-hosted runner on +# a private network, and a pull_request trigger would let any fork PR run code +# against it. Fork gating is designed in DESIGN.md §4 and is not built yet. +# +# The `push` trigger exists so this can be validated before merging, since a +# workflow_dispatch workflow is not dispatchable until it reaches the default +# branch. Drop the push trigger once this is on main. + +on: + workflow_dispatch: + push: + branches: + - claude/hil-testing-design-s564ln + +permissions: {} + +# The bench is a single set of physical boards. Queue, never interleave. +concurrency: + group: hil-bench + cancel-in-progress: false + +jobs: + smoke: + runs-on: [self-hosted, vector-hil] + timeout-minutes: 15 + + # No actions/checkout on purpose: under the design's trust model the bench + # runs harness code from a trusted checkout the runner already has, not + # from the PR. $VECTOR_HIL_REPO points at it. + + steps: + - name: Runner host + run: | + echo "user: $(whoami)" + echo "kernel: $(uname -srm)" + echo "arch: $(dpkg --print-architecture)" + free -h | awk '/Mem:/ {print "memory: " $2 " total, " $7 " available"}' + echo "uptime:$(uptime -p | sed 's/^up//')" + + - name: Bench environment + run: | + fail=0 + for var in VECTOR_HIL_REPO VECTOR_HIL_VENV VECTOR_HIL_WIFI_SSID VECTOR_HIL_WIFI_PASSWORD; do + eval "val=\${$var:-}" + if [ -z "$val" ]; then + echo "MISSING: $var" + fail=1 + else + echo "set: $var" + fi + done + [ "$fail" -eq 0 ] || { + echo "The runner's .env is incomplete. It is read at service start:" + echo " cd ~/actions-runner && sudo ./svc.sh stop && sudo ./svc.sh start" + exit 1 + } + test -x "$VECTOR_HIL_VENV/bin/python" || { echo "venv python missing at $VECTOR_HIL_VENV"; exit 1; } + test -d "$VECTOR_HIL_REPO/.git" || { echo "no clone at $VECTOR_HIL_REPO"; exit 1; } + + - name: Serial access + run: | + if ! ls /dev/ttyACM* >/dev/null 2>&1; then + echo "no /dev/ttyACM* devices - check the USB hub and power" + exit 1 + fi + ls -l /dev/ttyACM* + id -nG | grep -qw dialout || { + echo "runner user is not in the dialout group; reboot or restart the service" + exit 1 + } + + - name: Detect and query boards + run: | + # detect_boards.py prints progress text before the JSON, so the + # machine-readable part is the last line only. + raw=$("$VECTOR_HIL_VENV/bin/python" "$VECTOR_HIL_REPO/dev/detect_boards.py") + echo "$raw" + json=$(printf '%s\n' "$raw" | tail -1) + + "$VECTOR_HIL_VENV/bin/python" - "$json" <<'PY' + import json, os, subprocess, sys + + venv = os.environ["VECTOR_HIL_VENV"] + boards = json.loads(sys.argv[1]) + + if not boards: + sys.exit("no boards detected") + + rc = 0 + for kind, ports in sorted(boards.items()): + for port in ports: + probe = subprocess.run( + [f"{venv}/bin/mpremote", "connect", port, "exec", + "import systemConfig; print(systemConfig.vectorSystem, systemConfig.SystemVersion)"], + capture_output=True, text=True, timeout=60) + if probe.returncode: + print(f"FAIL {kind:12} {port:16} {probe.stderr.strip()}") + rc = 1 + else: + print(f"ok {kind:12} {port:16} {probe.stdout.strip()}") + + print(f"\n{sum(len(p) for p in boards.values())} board(s) responding") + sys.exit(rc) + PY + + - name: Summary + if: always() + run: echo "Bench smoke ${{ job.status }}" diff --git a/dev/hil/RUNNER_SETUP.md b/dev/hil/RUNNER_SETUP.md index 3bd30288..206a584c 100644 --- a/dev/hil/RUNNER_SETUP.md +++ b/dev/hil/RUNNER_SETUP.md @@ -72,23 +72,33 @@ cd ~/actions-runner && sudo ./svc.sh uninstall && ./config.sh remove --token Date: Mon, 17 Aug 2026 19:14:32 +0000 Subject: [PATCH 11/22] ci: put the venv on PATH for HIL smoke First bench run failed at board detection with an empty result while /dev/ttyACM0-2 were present and the runner user was in dialout. dev/detect_boards.py shells out to a bare `mpremote` resolved from PATH, and catches OSError - which covers FileNotFoundError - returning an empty list. Invoking the venv's python by absolute path does not put the venv's bin on PATH, so mpremote was missing and "tool not installed" surfaced as "no boards attached". Adds the venv to GITHUB_PATH and invokes python and mpremote by name so subprocesses resolve the same interpreter and tools. Also splits out an explicit mpremote check that prints `mpremote devs` directly, because detect_boards.py collapses "mpremote missing", "mpremote failed" and "no boards attached" into the same empty result - which is what made the first failure ambiguous. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01WPbnmZNF6DFnLQBtpQpcW4 --- .github/workflows/hil-smoke.yml | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/.github/workflows/hil-smoke.yml b/.github/workflows/hil-smoke.yml index 57e4a3eb..ca4c9cbc 100644 --- a/.github/workflows/hil-smoke.yml +++ b/.github/workflows/hil-smoke.yml @@ -63,6 +63,11 @@ jobs: test -x "$VECTOR_HIL_VENV/bin/python" || { echo "venv python missing at $VECTOR_HIL_VENV"; exit 1; } test -d "$VECTOR_HIL_REPO/.git" || { echo "no clone at $VECTOR_HIL_REPO"; exit 1; } + # dev/detect_boards.py shells out to a bare `mpremote` from PATH and + # silently returns {} when it is missing, so the venv must be ON PATH + # for later steps - invoking its python by absolute path is not enough. + echo "$VECTOR_HIL_VENV/bin" >> "$GITHUB_PATH" + - name: Serial access run: | if ! ls /dev/ttyACM* >/dev/null 2>&1; then @@ -75,18 +80,34 @@ jobs: exit 1 } + - name: mpremote sees the boards + run: | + # Checked separately because detect_boards.py collapses "mpremote is + # missing", "mpremote failed", and "no boards attached" into the same + # empty result, which makes a real failure indistinguishable from an + # empty bench. + command -v mpremote || { echo "mpremote not on PATH"; exit 1; } + mpremote version + + echo "--- mpremote devs ---" + mpremote devs | tee /tmp/devs.txt + test -s /tmp/devs.txt || { + echo "mpremote sees no devices, though /dev/ttyACM* exist." + echo "Boards may be busy or in an odd state; power-cycle the hub and retry." + exit 1 + } + - name: Detect and query boards run: | # detect_boards.py prints progress text before the JSON, so the # machine-readable part is the last line only. - raw=$("$VECTOR_HIL_VENV/bin/python" "$VECTOR_HIL_REPO/dev/detect_boards.py") + raw=$(python "$VECTOR_HIL_REPO/dev/detect_boards.py") echo "$raw" json=$(printf '%s\n' "$raw" | tail -1) - "$VECTOR_HIL_VENV/bin/python" - "$json" <<'PY' - import json, os, subprocess, sys + python - "$json" <<'PY' + import json, subprocess, sys - venv = os.environ["VECTOR_HIL_VENV"] boards = json.loads(sys.argv[1]) if not boards: @@ -96,7 +117,7 @@ jobs: for kind, ports in sorted(boards.items()): for port in ports: probe = subprocess.run( - [f"{venv}/bin/mpremote", "connect", port, "exec", + ["mpremote", "connect", port, "exec", "import systemConfig; print(systemConfig.vectorSystem, systemConfig.SystemVersion)"], capture_output=True, text=True, timeout=60) if probe.returncode: From cad038329ea847615651611efa4c8f58c4d4b022 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 17 Aug 2026 19:19:31 +0000 Subject: [PATCH 12/22] fix: pin the bench clone to a known ref, guard .env values Review feedback on #376. The setup script fetched but never checked anything out, so an existing clone stayed on whatever was last checked out. That contradicted both the trusted-checkout model and this repo's own docs, which claim re-running is how you pull a newer harness - it was not. Now checks out origin/$REPO_BRANCH (default main) detached after fetch, and refuses to run against a clone with uncommitted changes rather than discarding them. Rejects newlines in the wifi credentials. The runner parses .env by splitting each line on the first '=' and taking the rest verbatim (Runner.Listener/Program.cs) - there is no EnvironmentFile= in the generated unit and runsvc.sh does not source it - so spaces, quotes and backslashes are already safe and must not be escaped. A newline is the only value that cannot be represented. Documents why the runner is persistent rather than ephemeral, in both the design checklist and the setup guide, instead of leaving the two in silent disagreement. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01WPbnmZNF6DFnLQBtpQpcW4 --- dev/hil/DESIGN.md | 6 +++++- dev/hil/RUNNER_SETUP.md | 32 ++++++++++++++++++++++++++++++-- dev/hil/setup-runner.sh | 27 ++++++++++++++++++++++++++- 3 files changed, 61 insertions(+), 4 deletions(-) diff --git a/dev/hil/DESIGN.md b/dev/hil/DESIGN.md index 774abcdb..d9218c0d 100644 --- a/dev/hil/DESIGN.md +++ b/dev/hil/DESIGN.md @@ -400,7 +400,11 @@ So expect **roughly 30–50 minutes of bench occupancy per PR**, on a singleton ## 9. Runner hardening checklist -- **Ephemeral runner** (`--ephemeral`), registered at repo scope, label `vector-hil` +- **Ephemeral runner** (`--ephemeral`), registered at repo scope, label `vector-hil` — *not yet + done; the bench currently runs a persistent runner. Ephemeral registration needs a PAT with + `administration: write` stored on the Pi to mint a token per job, which is a worse secret to + hold than the runner's own credentials. Acceptable only because `workflow_run` means the Pi + never executes PR-authored code; see RUNNER_SETUP.md for the full reasoning.* - Unprivileged user, no sudo, no docker socket - `ACTIONS_RUNNER_HOOK_JOB_STARTED` / `_COMPLETED`: wipe `_work/`, run a bench-health precheck (all boards enumerate and answer), fail the job immediately if the bench is unhealthy rather than producing a confusing test failure - **No secrets on any self-hosted job.** In particular `WARPED_PINBALL_PRIVATE_KEY` must never be referenced by a job with a `self-hosted` label. Move signing into a dedicated environment restricted to `main` and tags so it is structurally unreachable from HIL. diff --git a/dev/hil/RUNNER_SETUP.md b/dev/hil/RUNNER_SETUP.md index 206a584c..d38ee627 100644 --- a/dev/hil/RUNNER_SETUP.md +++ b/dev/hil/RUNNER_SETUP.md @@ -59,8 +59,36 @@ download and registration are left alone, and `.env` is rewritten touching only a newer harness — and after the first time you don't need `RUNNER_TOKEN`, since registration is skipped once the runner exists. -Overridable via environment if you need them: `REPO_URL`, `RUNNER_LABELS`, `RUNNER_VERSION`, -`RUNNER_ARCH`. +**The clone is pinned, not floating.** After fetching, the script checks out `origin/$REPO_BRANCH` +(default `main`) as a detached HEAD, so the bench always runs a known ref rather than whatever +was last left checked out. It refuses to run if the clone has uncommitted changes rather than +discarding them — if you've been debugging by hand there, commit, stash, or delete the clone. + +Overridable via environment if you need them: `REPO_URL`, `REPO_BRANCH`, `RUNNER_LABELS`, +`RUNNER_VERSION`, `RUNNER_ARCH`. + +### On WiFi credentials with unusual characters + +The runner reads `.env` line by line, splits on the first `=`, and takes the rest of the line +verbatim ([`Runner.Listener/Program.cs`](https://github.com/actions/runner/blob/v2.336.0/src/Runner.Listener/Program.cs#L179-L197)) — +there is no `EnvironmentFile=` in the generated systemd unit and `runsvc.sh` doesn't source it +either. So spaces, quotes and backslashes in an SSID or password are safe and must **not** be +escaped; quoting would store literal quote characters. A newline is the one value that cannot +be represented, and the script rejects it up front. + +### Why not an ephemeral runner + +DESIGN.md §9 calls for `--ephemeral`. This script registers a persistent runner instead, which +is a deliberate deviation with a reason: ephemeral runners deregister after every job, so +something has to mint a fresh registration token each time — which means storing a PAT with +`administration: write` on the Pi, readable by the same user that runs job code. That PAT is a +much more valuable secret than the runner's own credentials, which only let you receive jobs. + +The trade works because of the trust model: under `workflow_run`, the Pi only ever executes +harness code from a trusted ref (DESIGN.md §4), so the "poison the workspace for the next job" +attack that ephemeral runners defend against has no foothold. Revisit this if the Pi ever +starts executing PR-authored code — at that point ephemeral runners via JIT config stop being +optional. To re-register against a different repo or token, remove the registration first — the script deliberately won't do this behind your back: diff --git a/dev/hil/setup-runner.sh b/dev/hil/setup-runner.sh index 0cff5f5a..28d25605 100755 --- a/dev/hil/setup-runner.sh +++ b/dev/hil/setup-runner.sh @@ -23,6 +23,7 @@ set -eu REPO_URL="${REPO_URL:-https://github.com/warped-pinball/vector}" +REPO_BRANCH="${REPO_BRANCH:-main}" RUNNER_LABELS="${RUNNER_LABELS:-vector-hil}" RUNNER_VERSION="${RUNNER_VERSION:-2.336.0}" RUNNER_ARCH="${RUNNER_ARCH:-arm64}" @@ -41,6 +42,18 @@ fail() { echo "ERROR: $*" >&2; exit 1; } [ -n "${VECTOR_HIL_WIFI_SSID:-}" ] || fail "VECTOR_HIL_WIFI_SSID is not set" [ -n "${VECTOR_HIL_WIFI_PASSWORD:-}" ] || fail "VECTOR_HIL_WIFI_PASSWORD is not set" +# The runner parses .env line by line, splitting on the first '=' and taking +# the rest of the line verbatim (Runner.Listener/Program.cs). Spaces, quotes +# and backslashes are therefore safe and must NOT be escaped - quoting would +# put literal quote characters into the value. A newline is the one thing that +# cannot be represented, so reject it rather than write a corrupt file. +for cred in VECTOR_HIL_WIFI_SSID VECTOR_HIL_WIFI_PASSWORD; do + eval "cred_value=\$$cred" + if [ "$(printf '%s' "$cred_value" | wc -l)" -ne 0 ]; then + fail "$cred contains a newline, which cannot be stored in the runner's .env" + fi +done + if [ ! -f "$RUNNER_DIR/.runner" ] && [ -z "${RUNNER_TOKEN:-}" ]; then fail "RUNNER_TOKEN is not set and the runner is not registered yet" fi @@ -66,12 +79,24 @@ fi if [ -d "$REPO_DIR/.git" ]; then log "updating existing clone at $REPO_DIR" - git -C "$REPO_DIR" fetch --quiet origin || echo "WARNING: git fetch failed, using existing checkout" + git -C "$REPO_DIR" fetch --quiet origin \ + || echo "WARNING: git fetch failed, falling back to the last fetched state" else log "cloning $REPO_URL" git clone --quiet "$REPO_URL" "$REPO_DIR" || fail "git clone failed" fi +# Fetching alone leaves the working tree wherever it was, so the bench could +# keep running stale or hand-modified harness code. Check out an explicit ref +# instead, detached, to make it obvious this is a pinned trusted checkout and +# not a branch anyone should be committing on. +if ! git -C "$REPO_DIR" diff --quiet HEAD 2>/dev/null; then + fail "$REPO_DIR has uncommitted changes - commit, stash, or delete the clone and re-run" +fi +git -C "$REPO_DIR" checkout --quiet --detach "origin/$REPO_BRANCH" \ + || fail "could not check out origin/$REPO_BRANCH" +log "harness at $(git -C "$REPO_DIR" rev-parse --short HEAD) (origin/$REPO_BRANCH)" + if [ ! -x "$VENV_DIR/bin/python" ]; then log "creating virtualenv" python3 -m venv "$VENV_DIR" || fail "venv creation failed" From 9761bf0fc763d301c6e15da7e0358153d857c8f9 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 17 Aug 2026 19:32:57 +0000 Subject: [PATCH 13/22] ci: flash every bench board and health-check its API Adds dev/hil/flash_and_check.py plus a workflow to drive it: inventory, resolve target per board, build, flash with bench wifi config, then health-check the API over USB and HTTP once the board boots. The identification problem is the interesting part. Nothing on a board reports what hardware it is - systemConfig.vectorSystem is a build-time constant baked into whatever was last flashed, and machine.unique_id() is the RP2040 chip id, stable per board but silent about which system it is wired for. The last smoke run had all three boards reporting sys11, so flashing from self-report would have perpetuated a mis-flash. So an explicit chip-id map (VECTOR_HIL_BOARD_MAP) is authoritative when set, and without one the harness falls back to self-report but refuses to flash when two boards claim the same system, printing the chip ids needed to build the map. --inventory-only prints them without touching anything. Health checks: version matches the built source, faults are only the HDWR02 expected on a bare bench, game status and config list are sane, the flashed config is the active one, leaders and the auth challenge route answer, then the same board over HTTP for version, index page and faults. HDWR01 warns rather than fails and suppresses the active-config assertion, since it means the board took the safe_mode path. The workflow uses actions/checkout, which is safe only because every trigger is repo-internal; noted inline that a fork-reachable trigger would have to move back to the runner's pinned clone. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01WPbnmZNF6DFnLQBtpQpcW4 --- .github/workflows/hil-flash-check.yml | 82 ++++ dev/hil/flash_and_check.py | 524 ++++++++++++++++++++++++++ 2 files changed, 606 insertions(+) create mode 100644 .github/workflows/hil-flash-check.yml create mode 100644 dev/hil/flash_and_check.py diff --git a/.github/workflows/hil-flash-check.yml b/.github/workflows/hil-flash-check.yml new file mode 100644 index 00000000..111946ca --- /dev/null +++ b/.github/workflows/hil-flash-check.yml @@ -0,0 +1,82 @@ +name: HIL flash and health check + +# Builds firmware for each attached board, flashes it, and health-checks the +# API over both USB and HTTP once the board boots. +# +# Deliberately no `pull_request` trigger — this targets a self-hosted runner on +# a private network and physically reflashes hardware. Fork gating is designed +# in dev/hil/DESIGN.md §4 and is not built yet. +# +# The `push` trigger exists so this can be validated before merging, since a +# workflow_dispatch workflow is not dispatchable until it reaches the default +# branch. Drop the push trigger once this is on main. + +on: + workflow_dispatch: + inputs: + skip_http: + description: "USB checks only (skip the network stack)" + type: boolean + default: false + skip_flash: + description: "Health-check what is already flashed, do not reflash" + type: boolean + default: false + inventory_only: + description: "Just print each board's chip id, for building VECTOR_HIL_BOARD_MAP" + type: boolean + default: false + push: + branches: + - claude/hil-testing-design-s564ln + paths: + - dev/hil/flash_and_check.py + - .github/workflows/hil-flash-check.yml + +permissions: + contents: read + +# Flashing is destructive and the bench is one set of boards. Never interleave. +concurrency: + group: hil-bench + cancel-in-progress: false + +jobs: + flash-and-check: + runs-on: [self-hosted, vector-hil] + timeout-minutes: 45 + + steps: + # actions/checkout is safe here ONLY because every trigger above is + # repo-internal. If a fork-reachable trigger is ever added, this must go + # back to running from the runner's pinned clone ($VECTOR_HIL_REPO) — + # otherwise a fork PR would execute its own harness code on the bench. + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + + - name: Prepare environment + run: | + test -x "$VECTOR_HIL_VENV/bin/python" || { echo "bench venv missing"; exit 1; } + echo "$VECTOR_HIL_VENV/bin" >> "$GITHUB_PATH" + + # VECTOR_HIL_BOARD_MAP, VECTOR_HIL_WIFI_* and friends arrive from the + # runner's .env, which the runner process exports into every job. They + # are deliberately NOT restated in an `env:` block here: the `env` + # context only covers workflow/job/step-level vars, so `${{ env.X }}` + # would evaluate to empty and shadow the real value. + - name: Flash and health-check every board + run: | + args="" + if [ "${{ inputs.skip_http }}" = "true" ]; then args="$args --skip-http"; fi + if [ "${{ inputs.skip_flash }}" = "true" ]; then args="$args --skip-flash"; fi + if [ "${{ inputs.inventory_only }}" = "true" ]; then args="$args --inventory-only"; fi + python dev/hil/flash_and_check.py $args + + - name: Board serial logs on failure + if: failure() + run: | + # A board that failed its health check may still be printing something + # useful; grab a few seconds of console from each. + for dev in /dev/ttyACM*; do + echo "--- $dev" + timeout 8 cat "$dev" || true + done diff --git a/dev/hil/flash_and_check.py b/dev/hil/flash_and_check.py new file mode 100644 index 00000000..dfbd456c --- /dev/null +++ b/dev/hil/flash_and_check.py @@ -0,0 +1,524 @@ +#!/usr/bin/env python3 +"""Flash every attached Vector board and health-check its API after boot. + +Run from the repo root on the bench Pi, with the dev venv on PATH: + + python dev/hil/flash_and_check.py + +Stages, in order: + + 1. inventory - probe every attached board for its RP2040 chip id and the + system its *current firmware* reports + 2. resolve - decide which target each board should be flashed with + 3. build - build each needed target once + 4. flash - wipe, copy, write bench config, reboot + 5. health - wait for boot, then exercise the API over USB and HTTP + +A note on identification, because it is the subtle part: nothing on the board +reports what *hardware* it is. ``systemConfig.vectorSystem`` is a build-time +constant baked into whatever was last flashed, and ``machine.unique_id()`` is +the RP2040 chip id - stable per board, but it says nothing about which system +the board is wired for. So "autodetection" can only tell you what a board is +currently *running*, which is exactly wrong after a mis-flash. See +resolve_targets() for how that is handled. +""" + +import argparse +import json +import os +import re +import subprocess +import sys +import time +import urllib.error +import urllib.request +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(REPO_ROOT / "dev")) + +from usb_coms_demo import UsbApiClient # noqa: E402 + +# A bare bench board has nothing driving the game bus, so this one is correct +# and expected rather than a regression. +EXPECTED_FAULTS = {"HDWR02"} + +# Floating data lines can trip the >250-transition check in main.py and send +# the board down the safe_mode path, where the game config is never loaded. +# Warned about rather than failed, but it invalidates the config assertions - +# see DESIGN.md §8 (G1). +BENCH_WARN_FAULTS = {"HDWR01"} + +# Config key is the config filename without .json (dev/build.py:253). +DEFAULT_GAMENAME = { + "sys11": "GenericSystem11_", + "wpc": "Generic_WPC", + "data_east": "GenericDE_", + "em": "EM_machine_", +} + +BOOT_TIMEOUT = 90 +HTTP_TIMEOUT = 10 + + +class CheckFailure(Exception): + pass + + +def log(msg): + print(msg, flush=True) + + +def group(title): + print(f"::group::{title}", flush=True) + + +def endgroup(): + print("::endgroup::", flush=True) + + +# -------------------------------------------------------------------------- +# 1. inventory +# -------------------------------------------------------------------------- + + +def mpremote(*args, timeout=60): + return subprocess.run(["mpremote", *args], capture_output=True, text=True, timeout=timeout) + + +def list_ports(): + result = mpremote("devs", timeout=30) + if result.returncode != 0: + raise CheckFailure(f"`mpremote devs` failed: {result.stderr.strip()}") + return [line.split()[0] for line in result.stdout.strip().splitlines() if line.strip()] + + +def probe(port): + """Return {port, chip_id, system, version} for one board. + + chip_id comes from the RP2040 itself so it survives any firmware state; + system/version come from the flashed firmware and may be missing if the + board is unflashed or broken. + """ + board = {"port": port, "chip_id": None, "system": None, "version": None} + + chip = mpremote( + "connect", port, "exec", + "from machine import unique_id;from binascii import hexlify;print(hexlify(unique_id()).decode())", + timeout=30, + ) + if chip.returncode == 0: + board["chip_id"] = chip.stdout.strip() + + info = mpremote( + "connect", port, "exec", + "import systemConfig;print(systemConfig.vectorSystem, systemConfig.SystemVersion)", + timeout=30, + ) + if info.returncode == 0 and info.stdout.strip(): + parts = info.stdout.split() + board["system"] = parts[0] + if len(parts) > 1: + board["version"] = parts[1] + + return board + + +def inventory(): + boards = [probe(port) for port in list_ports()] + if not boards: + raise CheckFailure("no boards found - check the USB hub and power") + + log(f"{'port':16} {'chip id':18} {'running':12} version") + for b in boards: + log(f"{b['port']:16} {b['chip_id'] or '?':18} {b['system'] or '(none)':12} {b['version'] or '-'}") + return boards + + +# -------------------------------------------------------------------------- +# 2. resolve +# -------------------------------------------------------------------------- + + +def parse_board_map(raw): + """Parse VECTOR_HIL_BOARD_MAP: 'chipid=target,chipid=target'.""" + mapping = {} + for entry in (raw or "").split(","): + entry = entry.strip() + if not entry: + continue + if "=" not in entry: + raise CheckFailure(f"bad VECTOR_HIL_BOARD_MAP entry {entry!r}, expected chipid=target") + chip, target = entry.split("=", 1) + mapping[chip.strip()] = target.strip() + return mapping + + +def resolve_targets(boards, board_map): + """Decide the target for each board, refusing to guess when it matters. + + An explicit chip-id map is authoritative. Without one we fall back to what + each board's firmware reports, which is only trustworthy when every board + reports something different - if two boards claim the same system, that is + the signature of a previous mis-flash rather than of the hardware, and + flashing on that basis would silently perpetuate it. + """ + if board_map: + unmapped = [b for b in boards if b["chip_id"] not in board_map] + if unmapped: + raise CheckFailure( + "VECTOR_HIL_BOARD_MAP is set but does not cover: " + + ", ".join(f"{b['port']} ({b['chip_id']})" for b in unmapped) + ) + for b in boards: + b["target"] = board_map[b["chip_id"]] + log("targets from VECTOR_HIL_BOARD_MAP") + return boards + + missing = [b for b in boards if not b["system"]] + if missing: + raise CheckFailure( + "cannot identify " + + ", ".join(b["port"] for b in missing) + + " - firmware did not report a system. Set VECTOR_HIL_BOARD_MAP." + ) + + systems = [b["system"] for b in boards] + duplicates = {s for s in systems if systems.count(s) > 1} + if duplicates: + raise CheckFailure( + "refusing to flash from autodetection: " + + ", ".join(sorted(duplicates)) + + " is reported by more than one board.\n" + "Detection reads the *flashed firmware*, not the hardware, so duplicates mean\n" + "at least one board is running firmware for a system it is not wired for.\n" + "Pin them explicitly instead, using the chip ids above:\n" + " VECTOR_HIL_BOARD_MAP=" + + ",".join(f"{b['chip_id']}=" for b in boards) + ) + + for b in boards: + b["target"] = b["system"] + log("targets from firmware self-report (all distinct)") + return boards + + +# -------------------------------------------------------------------------- +# 3. build +# -------------------------------------------------------------------------- + + +def source_version(target): + config = REPO_ROOT / "src" / target / "systemConfig.py" + match = re.search(r'SystemVersion\s*=\s*"([^"]+)"', config.read_text()) + if not match: + raise CheckFailure(f"could not read SystemVersion from {config}") + return match.group(1) + + +def build(target): + build_dir = REPO_ROOT / "build" / target + result = subprocess.run( + [sys.executable, "dev/build.py", "--target_hardware", target, "--build-dir", str(build_dir)], + cwd=REPO_ROOT, capture_output=True, text=True, timeout=900, + ) + if result.returncode != 0: + log(result.stdout[-3000:]) + log(result.stderr[-3000:]) + raise CheckFailure(f"build failed for {target}") + return build_dir + + +# -------------------------------------------------------------------------- +# 4. flash +# -------------------------------------------------------------------------- + + +def write_bench_config(target, workdir): + ssid = os.environ.get("VECTOR_HIL_WIFI_SSID", "") + password = os.environ.get("VECTOR_HIL_WIFI_PASSWORD", "") + game_password = os.environ.get("VECTOR_HIL_GAME_PASSWORD", "hiltest") + + if not ssid or not password: + raise CheckFailure("VECTOR_HIL_WIFI_SSID / VECTOR_HIL_WIFI_PASSWORD are not set") + + # dev/flash.py builds a MicroPython snippet with single-quoted values, so a + # single quote anywhere here would produce a syntax error on the board + # rather than an obvious failure here. + for name, value in (("ssid", ssid), ("password", password), ("game password", game_password)): + if "'" in value or "\\" in value: + raise CheckFailure(f"bench {name} contains a quote or backslash, which dev/flash.py cannot write") + + config = { + "ssid": ssid, + "password": password, + "gamename": DEFAULT_GAMENAME[target], + "Gpassword": game_password, + } + path = workdir / f"hil-config-{target}.json" + path.write_text(json.dumps(config)) + return path + + +def flash(target, port, build_dir, config_path): + result = subprocess.run( + [sys.executable, "dev/flash.py", str(build_dir), "--port", port, "--write-config", str(config_path)], + cwd=REPO_ROOT, capture_output=True, text=True, timeout=900, + ) + if result.returncode != 0: + log(result.stdout[-3000:]) + log(result.stderr[-3000:]) + raise CheckFailure(f"flash failed for {target} on {port}") + + +# -------------------------------------------------------------------------- +# 5. health +# -------------------------------------------------------------------------- + + +def wait_for_api(port, timeout=BOOT_TIMEOUT): + """Poll the USB API until the board answers, or give up.""" + deadline = time.monotonic() + timeout + last_error = None + while time.monotonic() < deadline: + client = None + try: + client = UsbApiClient.from_device(port=port, timeout=5) + response = client.send_and_receive(route="/api/version", payload=None, timeout=5) + if response.get("status") == 200: + return client + except Exception as exc: # serial not ready, board mid-boot, no response yet + last_error = exc + if client: + try: + client.close() + except Exception: + pass + time.sleep(3) + raise CheckFailure(f"{port} did not answer the USB API within {timeout}s (last error: {last_error})") + + +def get(client, route, expect=200): + response = client.send_and_receive(route=route, payload=None, timeout=15) + status = response.get("status") + if status != expect: + raise CheckFailure(f"{route} returned {status}, expected {expect}") + return response.get("body") + + +def check_faults(board): + faults = get(board["client"], "/api/fault") or [] + if isinstance(faults, dict): + faults = faults.get("faults", []) + codes = {str(f)[:6] for f in faults} + + log(f" faults: {faults if faults else 'none'}") + + unexpected = codes - EXPECTED_FAULTS - BENCH_WARN_FAULTS + if unexpected: + raise CheckFailure(f"unexpected fault(s): {sorted(unexpected)}") + + warned = codes & BENCH_WARN_FAULTS + if warned: + log(f"::warning::{board['port']} raised {sorted(warned)} - bare-board bus noise, " + "the board is in safe mode and the game config was NOT loaded") + return warned + + +def health_check_usb(board): + client = board["client"] + target = board["target"] + expected_version = source_version(target) + + version = get(client, "/api/version") + reported = version.get("version") if isinstance(version, dict) else version + log(f" version: {reported}") + if expected_version not in str(reported): + raise CheckFailure(f"version {reported!r} does not match built {expected_version!r}") + + safe_mode = check_faults(board) + + status = get(client, "/api/game/status") + if not isinstance(status, dict): + raise CheckFailure(f"/api/game/status returned {type(status).__name__}, expected an object") + log(f" game status keys: {sorted(status)[:6]}") + + configs = get(client, "/api/game/configs_list") + if not isinstance(configs, dict) or not configs: + raise CheckFailure("/api/game/configs_list is empty - config bundle missing from the build") + log(f" configs available: {len(configs)}") + + expected_config = DEFAULT_GAMENAME[target] + if safe_mode: + log(" skipping active-config check (board is in safe mode)") + else: + active = get(client, "/api/game/active_config") + active_name = active.get("name") if isinstance(active, dict) else active + log(f" active config: {active_name}") + if expected_config not in json.dumps(active): + raise CheckFailure(f"active config {active!r} is not the {expected_config!r} we flashed") + + leaders = get(client, "/api/leaders") + if leaders is None: + raise CheckFailure("/api/leaders returned no body") + + # Unauthenticated over USB by design (backend.py:280) - just prove it routes. + get(client, "/api/auth/challenge") + + wifi = get(client, "/api/wifi/status") + log(f" wifi: {wifi}") + + ip = None + try: + last_ip = get(client, "/api/last_ip") + ip = last_ip.get("ip") if isinstance(last_ip, dict) else None + except CheckFailure: + pass + return ip, wifi + + +def http_get(url): + request = urllib.request.Request(url, headers={"User-Agent": "vector-hil"}) + with urllib.request.urlopen(request, timeout=HTTP_TIMEOUT) as response: + return response.status, response.read() + + +def health_check_http(board): + ip = board.get("ip") + if not ip: + raise CheckFailure("board reported no IP address - it did not join the bench wifi") + + status, body = http_get(f"http://{ip}/api/version") + if status != 200: + raise CheckFailure(f"http /api/version returned {status}") + payload = json.loads(body) + log(f" http {ip} /api/version -> {payload}") + + usb_version = board["usb_version"] + if str(payload.get("version")) != str(usb_version): + raise CheckFailure(f"http version {payload.get('version')} disagrees with USB {usb_version}") + + status, body = http_get(f"http://{ip}/") + if status != 200: + raise CheckFailure(f"http / returned {status}") + if b" 200, {len(body)} bytes") + + status, _ = http_get(f"http://{ip}/api/fault") + if status != 200: + raise CheckFailure(f"http /api/fault returned {status}") + + +# -------------------------------------------------------------------------- + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--skip-http", action="store_true", help="USB checks only; do not exercise the network stack") + parser.add_argument("--skip-flash", action="store_true", help="health-check what is already on the boards") + parser.add_argument("--inventory-only", action="store_true", + help="print each board's chip id and stop - use this to build VECTOR_HIL_BOARD_MAP") + args = parser.parse_args() + + if args.inventory_only: + group("Inventory") + boards = inventory() + endgroup() + log("") + log("The boards are dedicated to the bench, so pin them by chip id once and") + log("autodetection stops mattering. Put this in the runner's .env, filling in") + log("the target for each (sys11, wpc, data_east, em, whitestar, classic):") + log("") + log(" VECTOR_HIL_BOARD_MAP=" + ",".join(f"{b['chip_id']}=" for b in boards)) + log("") + log("Then: cd ~/actions-runner && sudo ./svc.sh stop && sudo ./svc.sh start") + return 0 + + workdir = REPO_ROOT / "build" + workdir.mkdir(exist_ok=True) + failures = [] + + group("Inventory") + boards = inventory() + endgroup() + + group("Resolve targets") + boards = resolve_targets(boards, parse_board_map(os.environ.get("VECTOR_HIL_BOARD_MAP"))) + for b in boards: + log(f" {b['port']} -> {b['target']}") + endgroup() + + if not args.skip_flash: + for target in sorted({b["target"] for b in boards}): + group(f"Build {target}") + build(target) + log(f"built {target} at version {source_version(target)}") + endgroup() + + for b in boards: + group(f"Flash {b['target']} on {b['port']}") + try: + config_path = write_bench_config(b["target"], workdir) + flash(b["target"], b["port"], REPO_ROOT / "build" / b["target"], config_path) + log("flashed") + except CheckFailure as exc: + log(f"::error::{exc}") + failures.append(f"{b['port']} ({b['target']}): {exc}") + b["skip"] = True + endgroup() + + for b in boards: + if b.get("skip"): + continue + group(f"Health check {b['target']} on {b['port']}") + try: + b["client"] = wait_for_api(b["port"]) + ip, _wifi = health_check_usb(b) + b["ip"] = ip + b["usb_version"] = source_version(b["target"]) + log(" USB API OK") + + if args.skip_http: + log(" HTTP checks skipped") + else: + health_check_http(b) + log(" HTTP API OK") + except CheckFailure as exc: + log(f"::error::{exc}") + failures.append(f"{b['port']} ({b['target']}): {exc}") + except Exception as exc: + log(f"::error::unexpected error: {exc}") + failures.append(f"{b['port']} ({b['target']}): {exc}") + finally: + client = b.get("client") + if client: + try: + client.close() + except Exception: + pass + endgroup() + + log("") + log("=" * 60) + for b in boards: + state = "FAIL" if any(b["port"] in f for f in failures) else "ok" + log(f" {state:5} {b['port']:16} {b['target']:12} {b.get('ip') or ''}") + log("=" * 60) + + if failures: + log(f"\n{len(failures)} board(s) failed:") + for failure in failures: + log(f" - {failure}") + return 1 + + log(f"\nall {len(boards)} board(s) flashed and healthy") + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except CheckFailure as exc: + log(f"::error::{exc}") + sys.exit(1) From 25ae1ff39acd9427d0858b21c322a23a03a3d4c4 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 17 Aug 2026 19:39:30 +0000 Subject: [PATCH 14/22] feat: add board identify mode to the HIL harness [skip ci] The first flash run correctly refused to proceed: all three bench boards report sys11, so autodetection would have flashed sys11 firmware onto the WPC and Data East hardware. Resolving that needs a chip-id map, which needs a way to tell the physical boards apart. --identify blinks each board in turn, using the Pico W onboard LED (which works from the REPL regardless of what firmware is loaded) plus the Vector board's WS2812 in blue when the flashed firmware provides the driver. One pass over the bench identifies every board, rather than unplugging them one at a time and diffing the inventory. Documents the whole flow in RUNNER_SETUP.md, including why detection cannot identify hardware and where the map lives. Skipping CI on this commit: without the map set, a triggered run would fail at the same resolve step and add nothing. Identification is a hands-on task to run directly on the Pi while watching the bench. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01WPbnmZNF6DFnLQBtpQpcW4 --- .github/workflows/hil-flash-check.yml | 5 +++ dev/hil/RUNNER_SETUP.md | 29 ++++++++++++++ dev/hil/flash_and_check.py | 57 +++++++++++++++++++++++++++ 3 files changed, 91 insertions(+) diff --git a/.github/workflows/hil-flash-check.yml b/.github/workflows/hil-flash-check.yml index 111946ca..a5948dd3 100644 --- a/.github/workflows/hil-flash-check.yml +++ b/.github/workflows/hil-flash-check.yml @@ -26,6 +26,10 @@ on: description: "Just print each board's chip id, for building VECTOR_HIL_BOARD_MAP" type: boolean default: false + identify: + description: "Blink each board in turn so you can see which physical board is which" + type: boolean + default: false push: branches: - claude/hil-testing-design-s564ln @@ -69,6 +73,7 @@ jobs: if [ "${{ inputs.skip_http }}" = "true" ]; then args="$args --skip-http"; fi if [ "${{ inputs.skip_flash }}" = "true" ]; then args="$args --skip-flash"; fi if [ "${{ inputs.inventory_only }}" = "true" ]; then args="$args --inventory-only"; fi + if [ "${{ inputs.identify }}" = "true" ]; then args="$args --identify"; fi python dev/hil/flash_and_check.py $args - name: Board serial logs on failure diff --git a/dev/hil/RUNNER_SETUP.md b/dev/hil/RUNNER_SETUP.md index d38ee627..535b6245 100644 --- a/dev/hil/RUNNER_SETUP.md +++ b/dev/hil/RUNNER_SETUP.md @@ -97,6 +97,35 @@ deliberately won't do this behind your back: cd ~/actions-runner && sudo ./svc.sh uninstall && ./config.sh remove --token ``` +## Telling the boards apart + +Nothing on a board reports what hardware it is. `systemConfig.vectorSystem` is a build-time +constant baked into whatever was last flashed, so `dev/detect_boards.py` tells you what a board +is *running*, not what it is — which is exactly wrong after a mis-flash. `machine.unique_id()` +is the RP2040 chip id: stable per board and survives reflashing, but silent about the system. + +Since the bench boards are dedicated, pin them by chip id once and the question goes away. + +Blink each board in turn and watch the bench: + +```bash +cd ~/vector +PATH="$VECTOR_HIL_VENV/bin:$PATH" python dev/hil/flash_and_check.py --identify +``` + +Then record what you saw: + +```bash +echo 'VECTOR_HIL_BOARD_MAP==sys11,=wpc,=data_east' >> ~/actions-runner/.env +cd ~/actions-runner && sudo ./svc.sh stop && sudo ./svc.sh start +``` + +With the map set, `flash_and_check.py` uses it and ignores self-report entirely. Without it, +the harness falls back to self-report but **refuses to flash when two boards claim the same +system**, since that means at least one is running firmware for a system it isn't wired for. + +`--inventory-only` prints the chip ids without blinking or flashing anything. + ## Verify The runner should show **Idle** under Settings → Actions → Runners with the `vector-hil` label. diff --git a/dev/hil/flash_and_check.py b/dev/hil/flash_and_check.py index dfbd456c..0bb1a1f5 100644 --- a/dev/hil/flash_and_check.py +++ b/dev/hil/flash_and_check.py @@ -135,6 +135,52 @@ def inventory(): return boards +IDENTIFY_SNIPPET = """ +import machine, time +try: + import BoardLED as L + L.startUp() +except Exception: + L = None +led = machine.Pin("LED", machine.Pin.OUT) +for i in range({blinks}): + led.on() + if L: + L.ledColor(L.BLUE) + time.sleep(0.25) + led.off() + if L: + L.ledColor(L.BLACK) + time.sleep(0.25) +""" + + +def identify(boards, seconds=8): + """Blink each board in turn so a human can tell which is which. + + Uses the Pico W onboard LED, which works from the REPL no matter what + firmware is loaded, plus the Vector board's WS2812 in blue when the + flashed firmware happens to provide the driver. + """ + log(f"Blinking each board for ~{seconds}s. Watch the bench and note the order.") + log("") + for index, board in enumerate(boards, 1): + log(f" [{index}/{len(boards)}] BLINKING NOW: {board['port']} chip {board['chip_id']}") + result = mpremote( + "connect", board["port"], "exec", + IDENTIFY_SNIPPET.format(blinks=int(seconds / 0.5)), + timeout=seconds + 30, + ) + if result.returncode != 0: + log(f" could not blink this board: {result.stderr.strip()}") + else: + log(" done") + log("") + log("Now map what you saw to the chip ids, and put this in the runner's .env:") + log("") + log(" VECTOR_HIL_BOARD_MAP=" + ",".join(f"{b['chip_id']}=" for b in boards)) + + # -------------------------------------------------------------------------- # 2. resolve # -------------------------------------------------------------------------- @@ -419,8 +465,19 @@ def main(): parser.add_argument("--skip-flash", action="store_true", help="health-check what is already on the boards") parser.add_argument("--inventory-only", action="store_true", help="print each board's chip id and stop - use this to build VECTOR_HIL_BOARD_MAP") + parser.add_argument("--identify", action="store_true", + help="blink each board in turn so you can tell which physical board is which") args = parser.parse_args() + if args.identify: + group("Inventory") + boards = inventory() + endgroup() + group("Identify") + identify(boards) + endgroup() + return 0 + if args.inventory_only: group("Inventory") boards = inventory() From e6ef1e393ae0f1f37df707b8900918a6559c492e Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 17 Aug 2026 19:42:13 +0000 Subject: [PATCH 15/22] fix: make the HIL harness find its own tools [skip ci] Running the harness from a login shell failed with FileNotFoundError on mpremote. VECTOR_HIL_VENV is exported by the runner service into Actions jobs, not by an interactive shell, so the documented command prefixed PATH with an empty path and fell through to the system interpreter. The harness now locates the bench venv itself - VECTOR_HIL_VENV, then the directory of the running interpreter, then /.venv - prepends its bin to PATH, and uses its python for the build and flash subprocesses. That matters beyond our own calls: dev/build.py shells out to a bare mpy-cross and dev/flash.py to a bare mpremote, and the system interpreter would not have the build dependencies either. If mpremote still cannot be found it now says so and gives a working command, rather than raising a traceback from subprocess. Corrects the same bad command in RUNNER_SETUP.md and the module docstring. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01WPbnmZNF6DFnLQBtpQpcW4 --- dev/hil/RUNNER_SETUP.md | 6 +++-- dev/hil/flash_and_check.py | 49 ++++++++++++++++++++++++++++++++++---- 2 files changed, 49 insertions(+), 6 deletions(-) diff --git a/dev/hil/RUNNER_SETUP.md b/dev/hil/RUNNER_SETUP.md index 535b6245..34cf6149 100644 --- a/dev/hil/RUNNER_SETUP.md +++ b/dev/hil/RUNNER_SETUP.md @@ -109,10 +109,12 @@ Since the bench boards are dedicated, pin them by chip id once and the question Blink each board in turn and watch the bench: ```bash -cd ~/vector -PATH="$VECTOR_HIL_VENV/bin:$PATH" python dev/hil/flash_and_check.py --identify +cd ~/vector && PATH="$PWD/.venv/bin:$PATH" .venv/bin/python dev/hil/flash_and_check.py --identify ``` +(`$VECTOR_HIL_VENV` is exported by the runner *service*, so it is not set in a login shell — +hence the explicit `.venv` path here.) + Then record what you saw: ```bash diff --git a/dev/hil/flash_and_check.py b/dev/hil/flash_and_check.py index 0bb1a1f5..e9f1b9b1 100644 --- a/dev/hil/flash_and_check.py +++ b/dev/hil/flash_and_check.py @@ -1,9 +1,13 @@ #!/usr/bin/env python3 """Flash every attached Vector board and health-check its API after boot. -Run from the repo root on the bench Pi, with the dev venv on PATH: +Run from the repo root on the bench Pi: - python dev/hil/flash_and_check.py + cd ~/vector && PATH="$PWD/.venv/bin:$PATH" .venv/bin/python dev/hil/flash_and_check.py + +Inside an Actions job the runner's .env already provides VECTOR_HIL_VENV, so +plain `python dev/hil/flash_and_check.py` is enough there. In a login shell it +is not - .env is read by the runner service, not by your shell. Stages, in order: @@ -27,6 +31,7 @@ import json import os import re +import shutil import subprocess import sys import time @@ -82,6 +87,39 @@ def endgroup(): # -------------------------------------------------------------------------- +def ensure_tools_on_path(): + """Put the bench venv's bin dir on PATH and pick the interpreter to use. + + The harness gets run three ways - from an Actions job, from a login shell, + and by hand - and only the first has the runner's .env applied. dev/build.py + shells out to a bare `mpy-cross` and dev/flash.py to a bare `mpremote`, so + PATH has to be right for subprocesses too, not just for our own calls. + """ + candidates = [] + if os.environ.get("VECTOR_HIL_VENV"): + candidates.append(Path(os.environ["VECTOR_HIL_VENV"]) / "bin") + candidates.append(Path(sys.executable).parent) + candidates.append(REPO_ROOT / ".venv" / "bin") + + for bindir in candidates: + if (bindir / "mpremote").exists(): + os.environ["PATH"] = f"{bindir}{os.pathsep}{os.environ.get('PATH', '')}" + python = bindir / "python" + return str(python) if python.exists() else sys.executable + + if shutil.which("mpremote"): + return sys.executable + + raise CheckFailure( + "mpremote not found. Run with the bench venv, e.g.\n" + f" cd {REPO_ROOT} && PATH=\"$PWD/.venv/bin:$PATH\" .venv/bin/python dev/hil/flash_and_check.py ...\n" + "(VECTOR_HIL_VENV is exported by the runner service, so it is not set in a login shell.)" + ) + + +VENV_PYTHON = sys.executable + + def mpremote(*args, timeout=60): return subprocess.run(["mpremote", *args], capture_output=True, text=True, timeout=timeout) @@ -265,7 +303,7 @@ def source_version(target): def build(target): build_dir = REPO_ROOT / "build" / target result = subprocess.run( - [sys.executable, "dev/build.py", "--target_hardware", target, "--build-dir", str(build_dir)], + [VENV_PYTHON, "dev/build.py", "--target_hardware", target, "--build-dir", str(build_dir)], cwd=REPO_ROOT, capture_output=True, text=True, timeout=900, ) if result.returncode != 0: @@ -308,7 +346,7 @@ def write_bench_config(target, workdir): def flash(target, port, build_dir, config_path): result = subprocess.run( - [sys.executable, "dev/flash.py", str(build_dir), "--port", port, "--write-config", str(config_path)], + [VENV_PYTHON, "dev/flash.py", str(build_dir), "--port", port, "--write-config", str(config_path)], cwd=REPO_ROOT, capture_output=True, text=True, timeout=900, ) if result.returncode != 0: @@ -469,6 +507,9 @@ def main(): help="blink each board in turn so you can tell which physical board is which") args = parser.parse_args() + global VENV_PYTHON + VENV_PYTHON = ensure_tools_on_path() + if args.identify: group("Inventory") boards = inventory() From 3f18e6e2eb2e50908a71f00161dc99f720f5fc92 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 17 Aug 2026 19:46:20 +0000 Subject: [PATCH 16/22] ci: record per-stage timings in the HIL harness Build cost on the Zero 2 W is unmeasured - three targets of mpy-cross plus web minification, against a 45 minute job timeout. Each stage now reports its duration and the run ends with a breakdown, so if the timeout gets tight we know which stage to attack rather than guessing. Also triggers the first flash run now that VECTOR_HIL_BOARD_MAP is set on the bench. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01WPbnmZNF6DFnLQBtpQpcW4 --- dev/hil/flash_and_check.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/dev/hil/flash_and_check.py b/dev/hil/flash_and_check.py index e9f1b9b1..37e01037 100644 --- a/dev/hil/flash_and_check.py +++ b/dev/hil/flash_and_check.py @@ -74,11 +74,24 @@ def log(msg): print(msg, flush=True) +_TIMINGS = [] +_group = None + + def group(title): + global _group + _group = (title, time.monotonic()) print(f"::group::{title}", flush=True) def endgroup(): + global _group + if _group: + title, started = _group + elapsed = time.monotonic() - started + _TIMINGS.append((title, elapsed)) + print(f" [{elapsed:.1f}s]", flush=True) + _group = None print("::endgroup::", flush=True) @@ -597,6 +610,11 @@ def main(): pass endgroup() + log("") + log("stage timings (build cost on the Zero 2 W is the number to watch):") + for title, elapsed in _TIMINGS: + log(f" {elapsed:7.1f}s {title}") + log("") log("=" * 60) for b in boards: From f8962aa8b6f2cfe6e3d54e1513159dcc108af1c3 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 17 Aug 2026 19:51:53 +0000 Subject: [PATCH 17/22] fix: decompress gzip and retry in the HIL HTTP checks The first full flash run flashed all three boards, booted them, joined wifi and passed every USB API check - then failed on my own HTTP assertions. The board serves web assets pre-gzipped with Content-Encoding: gzip (backend.py:183) and urllib does not decompress automatically, so the index-page check was looking for " Claude-Session: https://claude.ai/code/session_01WPbnmZNF6DFnLQBtpQpcW4 --- dev/hil/flash_and_check.py | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/dev/hil/flash_and_check.py b/dev/hil/flash_and_check.py index 37e01037..617640a6 100644 --- a/dev/hil/flash_and_check.py +++ b/dev/hil/flash_and_check.py @@ -28,6 +28,7 @@ """ import argparse +import gzip import json import os import re @@ -450,8 +451,7 @@ def health_check_usb(board): log(" skipping active-config check (board is in safe mode)") else: active = get(client, "/api/game/active_config") - active_name = active.get("name") if isinstance(active, dict) else active - log(f" active config: {active_name}") + log(f" active config: {active}") if expected_config not in json.dumps(active): raise CheckFailure(f"active config {active!r} is not the {expected_config!r} we flashed") @@ -474,10 +474,29 @@ def health_check_usb(board): return ip, wifi -def http_get(url): - request = urllib.request.Request(url, headers={"User-Agent": "vector-hil"}) - with urllib.request.urlopen(request, timeout=HTTP_TIMEOUT) as response: - return response.status, response.read() +def http_get(url, attempts=3): + """GET a URL, decompressing gzip and retrying transient failures. + + The board serves its web assets pre-gzipped with Content-Encoding: gzip + (backend.py:183) and urllib does not decompress automatically. Retries + exist because phew is a single-threaded server on a microcontroller that + is also fielding discovery broadcasts - an occasional dropped body is not + a regression worth failing a bench run over. + """ + last_error = None + for attempt in range(attempts): + try: + request = urllib.request.Request(url, headers={"User-Agent": "vector-hil"}) + with urllib.request.urlopen(request, timeout=HTTP_TIMEOUT) as response: + body = response.read() + if response.headers.get("Content-Encoding", "").lower() == "gzip": + body = gzip.decompress(body) + return response.status, body + except Exception as exc: + last_error = exc + if attempt + 1 < attempts: + time.sleep(2) + raise CheckFailure(f"GET {url} failed after {attempts} attempts: {last_error!r}") def health_check_http(board): @@ -498,8 +517,9 @@ def health_check_http(board): status, body = http_get(f"http://{ip}/") if status != 200: raise CheckFailure(f"http / returned {status}") - if b" 200, {len(body)} bytes") status, _ = http_get(f"http://{ip}/api/fault") From a3895239a3604d7cdbada5f93f9d06be7459d9b8 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 17 Aug 2026 19:53:03 +0000 Subject: [PATCH 18/22] docs: correct the bare-board fault predictions with measured results [skip ci] The design predicted HDWR02 would always fire on a bare bench and that HDWR01 might fire nondeterministically and silently invalidate the config coverage. Both were wrong, and the first full flash run measured it: three boards freshly flashed for their real targets all booted with no faults at all. HDWR01 simply did not appear, so the floating-pin concern does not need a resistor pack and the config assertions are not vacuous. HDWR02 cannot appear: adr_activity_ok() is defined but never called, in either src/common/main.py or src/data_east/main.py. That is a firmware finding rather than a bench one - "No Bus Activity" is precisely the diagnostic a customer with a dead bus needs, and nothing can currently raise it. Worth its own issue: reconnect the check or drop the code, but a dead diagnostic in faults.py is the worst of both. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01WPbnmZNF6DFnLQBtpQpcW4 --- dev/hil/DESIGN.md | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/dev/hil/DESIGN.md b/dev/hil/DESIGN.md index d9218c0d..222e09c5 100644 --- a/dev/hil/DESIGN.md +++ b/dev/hil/DESIGN.md @@ -1,6 +1,6 @@ # Hardware-in-the-Loop (HIL) Testing — Design -**Status:** proposal, not yet implemented +**Status:** design proposal; bench bring-up and the flash/health-check harness are implemented and running **Scope:** a self-hosted GitHub Actions runner driving real Vector boards, safely, from a public repository. --- @@ -261,10 +261,22 @@ This is the most likely source of a flaky suite, and it needs measuring during b Separately, `adr_activity_ok()` raises **`HDWR02: No Bus Activity`** when shadow RAM lamp columns don't change, which on a bare board is the correct and permanent state. -So: +**Measured on the bench (2026-08-17), and both predictions were wrong:** -- `HDWR02` is **expected** on the bench and belongs in the allowlist. -- `HDWR01` must be **deterministic**. If it appears intermittently, the fix is a resistor pack tying those eight lines to a known level on the test bench — a bench fixture, not a firmware change. Do not paper over it with a retry; a test suite that retries its way past nondeterministic boot paths will hide real regressions. +- `HDWR01` did **not** fire. Three boards, freshly flashed for their real targets, all booted + with `faults: none`. The floating-pin worry did not materialise — no resistor pack needed, + and the config assertions are not silently vacuous. +- `HDWR02` did not fire either, and cannot: **`adr_activity_ok()` is defined but never called** + (`src/common/main.py:69`, `src/data_east/main.py:82`). The fault is unreachable code. + +That second point is a finding about the firmware rather than the bench. `HDWR02: No Bus +Activity` is exactly the diagnostic a customer with a dead bus needs, and today nothing can +raise it. Worth a separate issue: either wire the check into the boot path or drop the fault +code, but leaving a dead diagnostic in `faults.py` is the worst of both. + +The allowlist keeps tolerating `HDWR02` for now — harmless, and it costs nothing if the check +is ever reconnected. `HDWR01` stays a warning that suppresses the active-config assertion, +since if it ever does fire the board is in safe mode and the config genuinely was not loaded. Bring-up task: boot each bare board 50 times, record the fault set each time, and confirm it is identical every time. Until that holds, nothing else is worth automating. From bbdbb31f59626cbd620ef4563743877bdffdaed7 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 17 Aug 2026 19:59:16 +0000 Subject: [PATCH 19/22] docs: replace HIL throughput estimates with measured numbers The flash/health-check harness now reports per-stage timings, so the design can carry real figures instead of guesses: builds are 14-22s per target, flashing 22-28s per board, and boot to a fully health-checked API 10-15s. A full flash-and-verify of all three boards is about 3.5 minutes. Build cost was called out as a risk against the 45 minute job timeout and a candidate for caching. It is not - that concern is retired. The per-config reboot loop that drives the config matrix is still an estimate and is now labelled as one, since only flash-and-verify has been measured. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01WPbnmZNF6DFnLQBtpQpcW4 --- dev/hil/DESIGN.md | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/dev/hil/DESIGN.md b/dev/hil/DESIGN.md index 222e09c5..cf0466b9 100644 --- a/dev/hil/DESIGN.md +++ b/dev/hil/DESIGN.md @@ -313,7 +313,24 @@ Full matrix on every PR, per the decision above. Per board, loop over that hardw That last assertion is the one that earns its keep. `sys11_tiny` exists because RAM is tight; a config that parses fine but leaves too little heap is the failure that actually reaches customers. -**Throughput.** Boot cycle is roughly 15–25s (`main.py` alone has an 0.8s bus check plus ~4.5s of sleeps before WiFi comes up), times config count: +**Throughput.** Measured on the bench, per board, from the flash/health-check harness: + +| Stage | sys11 | wpc | data_east | +|---|---|---|---| +| build | 13.7s | 21.7s | 19.8s | +| flash (wipe + copy + config + reset) | 26.4s | 28.0s | 22.5s | +| boot → API answering + full health check | 11.7s | 14.8s | 10.7s | + +So a full flash-and-verify of all three boards is **~3.5 min end to end**, and build cost is +not a concern on the Zero 2 W — the caching this design worried about is unnecessary. + +The number that drives the config matrix is the last row: a board answers its API within +roughly 10s of reset, and the health check itself accounts for most of that 10–15s. The +15–25s per-config estimate below therefore still holds, but is an estimate — a per-config +reboot loop has not been measured yet. + +Boot cycle is roughly 15–25s (`main.py` alone has an 0.8s bus check plus ~4.5s of sleeps +before WiFi comes up), times config count: | Board | Configs | Serial estimate | |---|---|---| From 17316c557c077a6cd1463682c8bafa19699bb5cf Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 18 Aug 2026 00:53:00 +0000 Subject: [PATCH 20/22] feat: wait on the boot console and broaden the HTTP API checks Boot is slow and variable, so the harness now watches the serial console for the firmware announcing readiness instead of polling an API that is not listening yet. Polling told us nothing about why a board was late and burned the whole timeout when one failed to boot; reading the console gives an exact ready signal and, on failure, the boot log that explains it. The transcript is now printed whenever a health check fails. The marker is "Server: Loop Forever", printed immediately before loop.run_forever() (phew/server.py:381). Deliberately not the earlier "> starting web server on port 80" line - that is printed before start_server is even scheduled, so matching it returns while the socket is still closed. A pty-based test covers exactly that distinction. Since the marker still precedes run_forever(), a short settle follows it, with http_get's retries covering the remainder. backend.go() runs connect_to_wifi() before either line, so the marker covers both transports. HTTP coverage grows from three requests to the index page plus twelve read-only routes, all required to return JSON, and adds checks only this transport can make: - authentication is enforced (password_check must 401 without credentials) - USB bypasses auth by design (backend.py:280), so HTTP is the only place the gate can be proven - challenges are single-use, not a repeated nonce - version and config count agree between USB and HTTP, since the two bridges share a route table but not their plumbing Boot timeout raised to 150s. Verified against a stub server reproducing the board's gzip, 401 and nonce behaviour, including the mismatch and unenforced-auth failure paths. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01WPbnmZNF6DFnLQBtpQpcW4 --- dev/hil/flash_and_check.py | 201 +++++++++++++++++++++++++++++++------ 1 file changed, 170 insertions(+), 31 deletions(-) diff --git a/dev/hil/flash_and_check.py b/dev/hil/flash_and_check.py index 617640a6..d221a12b 100644 --- a/dev/hil/flash_and_check.py +++ b/dev/hil/flash_and_check.py @@ -43,6 +43,7 @@ REPO_ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(REPO_ROOT / "dev")) +import serial # noqa: E402 (ships with mpremote) from usb_coms_demo import UsbApiClient # noqa: E402 # A bare bench board has nothing driving the game bus, so this one is correct @@ -63,9 +64,42 @@ "em": "EM_machine_", } -BOOT_TIMEOUT = 90 +# Boot is slow and variable, so we watch the console for the firmware saying +# it is ready rather than guessing at a delay. +# +# "Server: Loop Forever" is the correct marker and the only one: phew prints +# it immediately before loop.run_forever() (phew/server.py:381). The earlier +# "> starting web server on port 80" line is NOT a ready signal - it is +# printed before start_server is even scheduled, let alone bound, so matching +# it returns while the socket is still closed. backend.go() has already run +# connect_to_wifi() by this point, so the marker covers both transports. +READY_MARKER = "Server: Loop Forever" + +# The marker is printed just *before* run_forever(), so give the event loop a +# moment to actually accept the listening socket. http_get's retries cover any +# remainder. +SERVER_SETTLE_SECONDS = 2 + +BOOT_TIMEOUT = 150 HTTP_TIMEOUT = 10 +# Read-only routes exercised over HTTP. Kept side-effect free so the check can +# run against a board repeatedly without changing its state. +HTTP_ROUTES = ( + "/api/version", + "/api/fault", + "/api/game/name", + "/api/game/status", + "/api/game/active_config", + "/api/game/configs_list", + "/api/leaders", + "/api/players", + "/api/machine_id", + "/api/wifi/status", + "/api/settings/get_tournament_mode", + "/api/auth/challenge", +) + class CheckFailure(Exception): pass @@ -374,26 +408,63 @@ def flash(target, port, build_dir, config_path): # -------------------------------------------------------------------------- -def wait_for_api(port, timeout=BOOT_TIMEOUT): - """Poll the USB API until the board answers, or give up.""" +def wait_for_server(port, timeout=BOOT_TIMEOUT): + """Watch the boot console until the firmware reports its web server is up. + + Polling an API that is not listening yet tells you nothing about why, and + burns the whole timeout when a board fails to boot. Reading the console + instead gives an exact ready signal and, on failure, the boot log that + explains it. + + Returns the open serial connection so the USB API can reuse it - the + Pico exposes one CDC endpoint, so a second connection would fight this one. + """ deadline = time.monotonic() + timeout - last_error = None + transcript = [] + connection = None + while time.monotonic() < deadline: - client = None + if connection is None: + try: + # The port disappears and re-enumerates across the reset, so a + # failure to open here is expected for the first second or two. + connection = serial.Serial(port=port, baudrate=115200, timeout=1) + except Exception: + time.sleep(1) + continue try: - client = UsbApiClient.from_device(port=port, timeout=5) - response = client.send_and_receive(route="/api/version", payload=None, timeout=5) - if response.get("status") == 200: - return client - except Exception as exc: # serial not ready, board mid-boot, no response yet - last_error = exc - if client: + raw = connection.readline() + except Exception: try: - client.close() + connection.close() except Exception: pass - time.sleep(3) - raise CheckFailure(f"{port} did not answer the USB API within {timeout}s (last error: {last_error})") + connection = None + continue + + if not raw: + continue + text = raw.decode(errors="replace").rstrip("\r\n") + if not text: + continue + transcript.append(text) + + if READY_MARKER in text: + elapsed = timeout - (deadline - time.monotonic()) + log(f" server up after {elapsed:.1f}s ({text.strip()!r})") + time.sleep(SERVER_SETTLE_SECONDS) + return connection, transcript + + if connection is not None: + try: + connection.close() + except Exception: + pass + + tail = "\n ".join(transcript[-20:]) or "(nothing on the console)" + raise CheckFailure( + f"{port} never reported its web server within {timeout}s. Last console output:\n {tail}" + ) def get(client, route, expect=200): @@ -445,6 +516,7 @@ def health_check_usb(board): if not isinstance(configs, dict) or not configs: raise CheckFailure("/api/game/configs_list is empty - config bundle missing from the build") log(f" configs available: {len(configs)}") + board["usb_config_count"] = len(configs) expected_config = DEFAULT_GAMENAME[target] if safe_mode: @@ -499,37 +571,100 @@ def http_get(url, attempts=3): raise CheckFailure(f"GET {url} failed after {attempts} attempts: {last_error!r}") +def http_status(url): + """Return the status code, including for responses urllib treats as errors.""" + try: + request = urllib.request.Request(url, headers={"User-Agent": "vector-hil"}) + with urllib.request.urlopen(request, timeout=HTTP_TIMEOUT) as response: + return response.status + except urllib.error.HTTPError as exc: + return exc.code + + def health_check_http(board): ip = board.get("ip") if not ip: raise CheckFailure("board reported no IP address - it did not join the bench wifi") + base = f"http://{ip}" - status, body = http_get(f"http://{ip}/api/version") - if status != 200: - raise CheckFailure(f"http /api/version returned {status}") - payload = json.loads(body) - log(f" http {ip} /api/version -> {payload}") - - usb_version = board["usb_version"] - if str(payload.get("version")) != str(usb_version): - raise CheckFailure(f"http version {payload.get('version')} disagrees with USB {usb_version}") - - status, body = http_get(f"http://{ip}/") + # The index page is served pre-gzipped; http_get transparently inflates it. + status, body = http_get(f"{base}/") if status != 200: raise CheckFailure(f"http / returned {status}") head = body[:2000].lower() if b" 200, {len(body)} bytes") + log(f" GET {'/':34} 200 {len(body)} bytes (html)") - status, _ = http_get(f"http://{ip}/api/fault") - if status != 200: - raise CheckFailure(f"http /api/fault returned {status}") + payloads = {} + for route in HTTP_ROUTES: + status, body = http_get(f"{base}{route}") + if status != 200: + raise CheckFailure(f"http {route} returned {status}") + try: + payloads[route] = json.loads(body) + except json.JSONDecodeError: + raise CheckFailure(f"http {route} did not return JSON: {body[:120]!r}") + log(f" GET {route:34} 200 {_summarise(payloads[route])}") + + # Both transports must agree. They share the route table but not the + # plumbing, so a mismatch means one of the two bridges is misbehaving. + http_version = str(payloads["/api/version"].get("version")) + if http_version != str(board["usb_version"]): + raise CheckFailure(f"http version {http_version} disagrees with USB {board['usb_version']}") + + http_configs = payloads["/api/game/configs_list"] + if len(http_configs) != board["usb_config_count"]: + raise CheckFailure( + f"http lists {len(http_configs)} configs, USB lists {board['usb_config_count']}" + ) + + # Authentication is enforced over HTTP and deliberately bypassed over USB + # (backend.py:280), so this is the only transport that can prove the gate + # works. password_check is the one auth route with no side effects. + status = http_status(f"{base}/api/auth/password_check") + if status != 401: + raise CheckFailure( + f"/api/auth/password_check returned {status} without credentials, expected 401 - " + "HTTP authentication is not being enforced" + ) + log(f" GET {'/api/auth/password_check':34} 401 (auth enforced, as expected)") + + # A challenge must not be reusable: the handler deletes it on use. + first = http_get(f"{base}/api/auth/challenge")[1] + second = http_get(f"{base}/api/auth/challenge")[1] + if json.loads(first).get("challenge") == json.loads(second).get("challenge"): + raise CheckFailure("/api/auth/challenge issued the same nonce twice") + + log(f" {len(HTTP_ROUTES)} routes + index + auth checks OK over HTTP") + + +def _summarise(payload): + """One-line rendering of a response body for the log.""" + if isinstance(payload, dict): + if len(payload) == 1: + key, value = next(iter(payload.items())) + return f"{key}={value!r}" + return f"{len(payload)} keys" + if isinstance(payload, list): + return f"{len(payload)} items" + return repr(payload)[:60] # -------------------------------------------------------------------------- +def _dump_boot_log(board, lines=25): + """Show what the board actually said. A failed health check is usually + explained by the boot output, and by this point we already have it.""" + transcript = board.get("boot_log") + if not transcript: + return + log(f" last {min(lines, len(transcript))} lines of {board['port']} boot console:") + for line in transcript[-lines:]: + log(f" {line}") + + def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--skip-http", action="store_true", help="USB checks only; do not exercise the network stack") @@ -604,7 +739,9 @@ def main(): continue group(f"Health check {b['target']} on {b['port']}") try: - b["client"] = wait_for_api(b["port"]) + connection, boot_log = wait_for_server(b["port"]) + b["boot_log"] = boot_log + b["client"] = UsbApiClient(connection) ip, _wifi = health_check_usb(b) b["ip"] = ip b["usb_version"] = source_version(b["target"]) @@ -617,9 +754,11 @@ def main(): log(" HTTP API OK") except CheckFailure as exc: log(f"::error::{exc}") + _dump_boot_log(b) failures.append(f"{b['port']} ({b['target']}): {exc}") except Exception as exc: log(f"::error::unexpected error: {exc}") + _dump_boot_log(b) failures.append(f"{b['port']} ({b['target']}): {exc}") finally: client = b.get("client") From be59fc6bb63e8d6f979d5ef3148ac1c137a40de9 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 18 Aug 2026 01:06:00 +0000 Subject: [PATCH 21/22] fix: reset each board before watching for its ready marker The console watch timed out on all three boards while they were in fact healthy - the boot log it dumped showed steady-state runtime chatter (RESOURCE, DISCOVERY PING/PONG between the boards' own IPs, FRAM cycles), not boot output. The marker prints exactly once per boot. dev/flash.py resets at the end of flashing, but flashing loops over every board before any health check starts, so the first board had booted 50-75 seconds earlier and its marker was long past. The previous polling approach was immune to this because it did not care when the board became ready; watching a one-shot line is not. So the harness now issues the reset itself at the start of each health check, immediately before opening the console. The wait becomes deterministic and the reported boot time actually measures a boot. A failed reset is now its own clear error rather than a 150s timeout. Worth noting the boot-log dump added in the same series is what made this diagnosable at a glance - without it this looked like three boards failing to boot. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01WPbnmZNF6DFnLQBtpQpcW4 --- dev/hil/flash_and_check.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/dev/hil/flash_and_check.py b/dev/hil/flash_and_check.py index d221a12b..cef9836f 100644 --- a/dev/hil/flash_and_check.py +++ b/dev/hil/flash_and_check.py @@ -408,6 +408,23 @@ def flash(target, port, build_dir, config_path): # -------------------------------------------------------------------------- +def reset_board(port): + """Reset the board so we own the boot we are about to watch. + + The ready marker is printed exactly once per boot. dev/flash.py already + resets at the end of flashing, but flashing runs over every board before + any health check starts, so by the time we open a console the board booted + a minute ago and the marker is long gone. Resetting here makes the wait + deterministic and the reported boot time meaningful. + """ + result = mpremote( + "connect", port, "exec", "--no-follow", "import machine; machine.reset()", + timeout=30, + ) + if result.returncode != 0: + raise CheckFailure(f"could not reset {port} before the health check: {result.stderr.strip()}") + + def wait_for_server(port, timeout=BOOT_TIMEOUT): """Watch the boot console until the firmware reports its web server is up. @@ -739,6 +756,7 @@ def main(): continue group(f"Health check {b['target']} on {b['port']}") try: + reset_board(b["port"]) connection, boot_log = wait_for_server(b["port"]) b["boot_log"] = boot_log b["client"] = UsbApiClient(connection) From 42f91e5fcc4fa246203c8296eebdf62208d47acf Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 18 Aug 2026 01:14:15 +0000 Subject: [PATCH 22/22] fix: allow plain-text routes and prime the USB link before use The reset fix worked - all three boards reported their ready marker (12.5s, 13.9s, 23.6s) - which exposed two further problems. sys11 failed on my own over-strict assertion. /api/game/name is documented as "Plain-text game name" (backend.py:451) and returns a bare string, so requiring JSON from every route was wrong. Routes now carry their expected body kind; text routes are checked for a non-empty body instead. Worth noting separately that route_wrapper still labels that response application/json, which is a small inconsistency in the firmware rather than in the test. wpc and data_east answered 404 to the very first USB request for /api/version. The route table cannot be incomplete - the server only starts if backend.py imports fully - so the board must have parsed a different path. usb_comms accumulates stdin into a module-level buffer until it sees a newline (usb_comms.py:132), so any untermined bytes left over from the raw-REPL session that issued the reset prefix the next request and break the _routes lookup. The harness now sends a lone newline to flush that buffer, waits one scheduler turn for it to be consumed, and clears both ends of the line first. If a USB request still fails, the error now includes the board's own narration. usb_comms prints "USB REQ: route not found: " but UsbApiClient discards every line that is not a response, which is why the last run could not say which path the board actually saw. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01WPbnmZNF6DFnLQBtpQpcW4 --- dev/hil/flash_and_check.py | 97 ++++++++++++++++++++++++++++++-------- 1 file changed, 78 insertions(+), 19 deletions(-) diff --git a/dev/hil/flash_and_check.py b/dev/hil/flash_and_check.py index cef9836f..52a7af9e 100644 --- a/dev/hil/flash_and_check.py +++ b/dev/hil/flash_and_check.py @@ -85,19 +85,22 @@ # Read-only routes exercised over HTTP. Kept side-effect free so the check can # run against a board repeatedly without changing its state. +# Route -> expected body kind. Not everything is JSON: /api/game/name is +# documented as "Plain-text game name" (backend.py:451) and returns a bare +# string, even though route_wrapper still labels it application/json. HTTP_ROUTES = ( - "/api/version", - "/api/fault", - "/api/game/name", - "/api/game/status", - "/api/game/active_config", - "/api/game/configs_list", - "/api/leaders", - "/api/players", - "/api/machine_id", - "/api/wifi/status", - "/api/settings/get_tournament_mode", - "/api/auth/challenge", + ("/api/version", "json"), + ("/api/fault", "json"), + ("/api/game/name", "text"), + ("/api/game/status", "json"), + ("/api/game/active_config", "json"), + ("/api/game/configs_list", "json"), + ("/api/leaders", "json"), + ("/api/players", "json"), + ("/api/machine_id", "json"), + ("/api/wifi/status", "json"), + ("/api/settings/get_tournament_mode", "json"), + ("/api/auth/challenge", "json"), ) @@ -484,14 +487,62 @@ def wait_for_server(port, timeout=BOOT_TIMEOUT): ) +def prime_usb(connection): + """Clear both ends of the serial line before the first API request. + + usb_comms accumulates stdin characters into a module-level `buffer` until + it sees a newline (usb_comms.py:132). Anything left there without a + terminator - a partial line, stray bytes from the raw-REPL session that + issued the reset - silently prefixes the next request, so the board parses + a route like "\x02/api/version", fails the `_routes` lookup and answers + 404. A lone newline flushes whatever is pending into a discarded request. + """ + try: + connection.reset_input_buffer() + connection.reset_output_buffer() + connection.write(b"\n") + connection.flush() + except Exception as exc: + log(f" warning: could not prime the USB link: {exc}") + return + # usb_request_handler is scheduled every 1000ms (phew/server.py:342), so + # give it a turn to consume the flush before the first real request. + time.sleep(1.5) + try: + connection.reset_input_buffer() + except Exception: + pass + + def get(client, route, expect=200): response = client.send_and_receive(route=route, payload=None, timeout=15) status = response.get("status") if status != expect: - raise CheckFailure(f"{route} returned {status}, expected {expect}") + # The board narrates its own routing failures ("USB REQ: route not + # found: ..."), but the client discards every line that is not a + # response. Drain whatever is pending so the reason is visible. + raise CheckFailure( + f"{route} returned {status}, expected {expect}" + f"{_drain_serial(client.ser)}" + ) return response.get("body") +def _drain_serial(connection, limit=12): + """Return any pending board chatter, formatted for an error message.""" + try: + time.sleep(0.5) + pending = connection.read(connection.in_waiting or 0) + except Exception: + return "" + if not pending: + return "" + lines = [line for line in pending.decode(errors="replace").splitlines() if line.strip()] + if not lines: + return "" + return "\n board said: " + "\n board said: ".join(lines[:limit]) + + def check_faults(board): faults = get(board["client"], "/api/fault") or [] if isinstance(faults, dict): @@ -614,15 +665,22 @@ def health_check_http(board): log(f" GET {'/':34} 200 {len(body)} bytes (html)") payloads = {} - for route in HTTP_ROUTES: + for route, kind in HTTP_ROUTES: status, body = http_get(f"{base}{route}") if status != 200: raise CheckFailure(f"http {route} returned {status}") - try: - payloads[route] = json.loads(body) - except json.JSONDecodeError: - raise CheckFailure(f"http {route} did not return JSON: {body[:120]!r}") - log(f" GET {route:34} 200 {_summarise(payloads[route])}") + if kind == "json": + try: + payloads[route] = json.loads(body) + except json.JSONDecodeError: + raise CheckFailure(f"http {route} did not return JSON: {body[:120]!r}") + rendered = _summarise(payloads[route]) + else: + if not body.strip(): + raise CheckFailure(f"http {route} returned an empty body") + payloads[route] = body.decode(errors="replace").strip() + rendered = repr(payloads[route])[:60] + log(f" GET {route:34} 200 {rendered}") # Both transports must agree. They share the route table but not the # plumbing, so a mismatch means one of the two bridges is misbehaving. @@ -759,6 +817,7 @@ def main(): reset_board(b["port"]) connection, boot_log = wait_for_server(b["port"]) b["boot_log"] = boot_log + prime_usb(connection) b["client"] = UsbApiClient(connection) ip, _wifi = health_check_usb(b) b["ip"] = ip